diff --git a/.editorconfig b/.editorconfig index 86360e658..e1c8bae39 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,20 +4,20 @@ # Top-most EditorConfig file root = true -# Unix-style newlines with a newline ending every file, utf-8 charset +# Unix-style newlines with a newline ending every file, UTF-8 charset [*] end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true charset = utf-8 -# Match nix files, set indent to spaces with width of two +# Match Nix files, set indent to spaces with width of two [*.nix] indent_style = space indent_size = 2 -# Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl,xs}] +# Match C++/C/shell/Perl, set indent to spaces with width of four +[*.{hpp,cc,hh,c,h,sh,pl,xs}] indent_style = space indent_size = 4 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 99ca670e0..a9ca74c17 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -23,4 +23,4 @@ maintainers/*.md @fricklerhandwerk src/**/*.md @fricklerhandwerk # Libstore layer -/src/libstore @thufschmitt @ericson2314 +/src/libstore @ericson2314 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aa3b776e..ca94ff956 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,23 @@ jobs: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - if: matrix.os == 'ubuntu-latest' + run: | + free -h + swapon --show + swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}') + echo "Found swap: $swap" + sudo swapoff $swap + # resize it (fallocate) + sudo fallocate -l 10G $swap + sudo mkswap $swap + sudo swapon $swap + free -h + ( + while sleep 60; do + free -h + done + ) & - run: nix --experimental-features 'nix-command flakes' flake check -L # Steps to test CI automation in your own fork. @@ -79,6 +96,7 @@ jobs: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + cachixArgs: '-v' - id: prepare-installer run: scripts/prepare-installer-for-github-actions @@ -175,4 +193,18 @@ jobs: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes + - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user + + meson_build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + # Only meson packages that don't have a tests.run derivation. + # Those that have it are already built and tested as part of nix flake check. + - run: nix build -L .#hydraJobs.build.{nix-cmd,nix-main}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g') diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml deleted file mode 100644 index 2a7574747..000000000 --- a/.github/workflows/hydra_status.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Hydra status - -permissions: read-all - -on: - schedule: - - cron: "12,42 * * * *" - workflow_dispatch: - -jobs: - check_hydra_status: - name: Check Hydra status - if: github.repository_owner == 'NixOS' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: bash scripts/check-hydra-status.sh - diff --git a/.gitignore b/.gitignore index 28f853715..a17b627f4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ perl/Makefile.config /svn-revision /libtool /config/config.* +# Default meson build dir +/build # /doc/manual/ /doc/manual/*.1 @@ -52,6 +54,9 @@ perl/Makefile.config # /src/libfetchers /tests/unit/libfetchers/libnixfetchers-tests +# /src/libflake +/tests/unit/libflake/libnixflake-tests + # /src/libstore/ *.gen.* /src/libstore/tests diff --git a/.shellcheckrc b/.shellcheckrc index 662e2045c..de98055f7 100644 --- a/.shellcheckrc +++ b/.shellcheckrc @@ -1,2 +1,4 @@ external-sources=true source-path=SCRIPTDIR +# Hack for scripts in e.g. tests/functional/ca +source-path=SCRIPTDIR/.. diff --git a/.version b/.version index e9763f6bf..ad2261920 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.23.0 +2.24.0 diff --git a/Makefile b/Makefile index 6c96ef5db..bb64a104e 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ makefiles = \ src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ + src/libflake/local.mk \ src/libcmd/local.mk \ src/nix/local.mk \ src/libutil-c/local.mk \ @@ -45,7 +46,8 @@ makefiles += \ tests/unit/libstore-support/local.mk \ tests/unit/libfetchers/local.mk \ tests/unit/libexpr/local.mk \ - tests/unit/libexpr-support/local.mk + tests/unit/libexpr-support/local.mk \ + tests/unit/libflake/local.mk endif ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) @@ -68,14 +70,6 @@ ifeq ($(ENABLE_DOC_GEN), yes) makefiles-late += doc/manual/local.mk endif -ifeq ($(ENABLE_INTERNAL_API_DOCS), yes) -makefiles-late += doc/internal-api/local.mk -endif - -ifeq ($(ENABLE_EXTERNAL_API_DOCS), yes) -makefiles-late += doc/external-api/local.mk -endif - # Miscellaneous global Flags OPTIMIZE = 1 @@ -98,7 +92,7 @@ ifdef HOST_WINDOWS GLOBAL_LDFLAGS += -Wl,--export-all-symbols endif -GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src +GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -include $(buildprefix)config.h -std=c++2a -I src # Include the main lib, causing rules to be defined @@ -131,17 +125,3 @@ manual-html manpages: @echo "Generated docs are disabled. Configure without '--disable-doc-gen', or avoid calling 'make manpages' and 'make manual-html'." @exit 1 endif - -ifneq ($(ENABLE_INTERNAL_API_DOCS), yes) -.PHONY: internal-api-html -internal-api-html: - @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." - @exit 1 -endif - -ifneq ($(ENABLE_EXTERNAL_API_DOCS), yes) -.PHONY: external-api-html -external-api-html: - @echo "External API docs are disabled. Configure with '--enable-external-api-docs', or avoid calling 'make external-api-html'." - @exit 1 -endif diff --git a/Makefile.config.in b/Makefile.config.in index 7f517898c..3100d2073 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -11,8 +11,6 @@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ -ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@ -ENABLE_EXTERNAL_API_DOCS = @ENABLE_EXTERNAL_API_DOCS@ ENABLE_S3 = @ENABLE_S3@ ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ diff --git a/README.md b/README.md index e1cace3b4..931a60bba 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). -## Building And Developing +## Building and developing See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to set up a development environment and build Nix from source. @@ -22,12 +22,17 @@ See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacki Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. -## Additional Resources +## Additional resources -- [Nix manual](https://nixos.org/nix/manual) -- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) -- [NixOS Discourse](https://discourse.nixos.org/) -- [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) +Nix was created by Eelco Dolstra and developed as the subject of his PhD thesis [The Purely Functional Software Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf), published 2006. +Today, a world-wide developer community contributes to Nix and the ecosystem that has grown around it. + +- [The Nix, Nixpkgs, NixOS Community on nixos.org](https://nixos.org/) +- [Official documentation on nix.dev](https://nix.dev) +- [Nixpkgs](https://github.com/NixOS/nixpkgs) is [the largest, most up-to-date free software repository in the world](https://repology.org/repositories/graphs) +- [NixOS](https://github.com/NixOS/nixpkgs/tree/master/nixos) is a Linux distribution that can be configured fully declaratively +- [Discourse](https://discourse.nixos.org/) +- [Matrix](https://matrix.to/#/#nix:nixos.org) ## License diff --git a/build-utils-meson/deps-lists/meson.build b/build-utils-meson/deps-lists/meson.build new file mode 100644 index 000000000..237eac545 --- /dev/null +++ b/build-utils-meson/deps-lists/meson.build @@ -0,0 +1,36 @@ +# These are private dependencies with pkg-config files. What private +# means is that the dependencies are used by the library but they are +# *not* used (e.g. `#include`-ed) in any installed header file, and only +# in regular source code (`*.cc`) or private, uninstalled headers. They +# are thus part of the *implementation* of the library, but not its +# *interface*. +# +# See `man pkg-config` for some details. +deps_private = [ ] + +# These are public dependencies with pkg-config files. Public is the +# opposite of private: these dependencies are used in installed header +# files. They are part of the interface (and implementation) of the +# library. +# +# N.B. This concept is mostly unrelated to our own concept of a public +# (stable) API, for consumption outside of the Nix repository. +# `libnixutil` is an unstable C++ library, whose public interface is +# likewise unstable. `libutilc` conversely is a hopefully-soon stable +# C library, whose public interface --- including public but not private +# dependencies --- will also likewise soon be stable. +# +# N.B. For distributions that care about "ABI" stability and not just +# "API" stability, the private dependencies also matter as they can +# potentially affect the public ABI. +deps_public = [ ] + +# These are subproject deps (type == "internal"). They are other +# packages in `/src` in this repo. The private vs public distinction is +# the same as above. +deps_private_subproject = [ ] +deps_public_subproject = [ ] + +# These are dependencencies without pkg-config files. Ideally they are +# just private, but they may also be public (e.g. boost). +deps_other = [ ] diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build new file mode 100644 index 000000000..2b79f6566 --- /dev/null +++ b/build-utils-meson/diagnostics/meson.build @@ -0,0 +1,13 @@ +add_project_arguments( + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Werror=unused-result', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + language : 'cpp', +) diff --git a/build-utils-meson/export-all-symbols/meson.build b/build-utils-meson/export-all-symbols/meson.build new file mode 100644 index 000000000..d7c086749 --- /dev/null +++ b/build-utils-meson/export-all-symbols/meson.build @@ -0,0 +1,11 @@ +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif diff --git a/build-utils-meson/export/meson.build b/build-utils-meson/export/meson.build new file mode 100644 index 000000000..40f6dcd59 --- /dev/null +++ b/build-utils-meson/export/meson.build @@ -0,0 +1,30 @@ +requires_private = [] +foreach dep : deps_private_subproject + requires_private += dep.name() +endforeach +requires_private += deps_private + +requires_public = [] +foreach dep : deps_public_subproject + requires_public += dep.name() +endforeach +requires_public += deps_public + +import('pkgconfig').generate( + this_library, + filebase : meson.project_name(), + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : requires_public, + requires_private : requires_private, + libraries_private : libraries_private, +) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_library, + compile_args : ['-std=c++2a'], + dependencies : deps_public_subproject + deps_public, +)) diff --git a/build-utils-meson/generate-header/meson.build b/build-utils-meson/generate-header/meson.build new file mode 100644 index 000000000..dfbe1375f --- /dev/null +++ b/build-utils-meson/generate-header/meson.build @@ -0,0 +1,7 @@ +bash = find_program('bash', native: true) + +gen_header = generator( + bash, + arguments : [ '-c', '{ echo \'R"__NIX_STR(\' && cat @INPUT@ && echo \')__NIX_STR"\'; } > "$1"', '_ignored_argv0', '@OUTPUT@' ], + output : '@PLAINNAME@.gen.hh', +) diff --git a/build-utils-meson/subprojects/meson.build b/build-utils-meson/subprojects/meson.build new file mode 100644 index 000000000..30a54ed91 --- /dev/null +++ b/build-utils-meson/subprojects/meson.build @@ -0,0 +1,19 @@ +foreach maybe_subproject_dep : deps_private_maybe_subproject + if maybe_subproject_dep.type_name() == 'internal' + deps_private_subproject += maybe_subproject_dep + # subproject sadly no good for pkg-config module + deps_other += maybe_subproject_dep + else + deps_private += maybe_subproject_dep + endif +endforeach + +foreach maybe_subproject_dep : deps_public_maybe_subproject + if maybe_subproject_dep.type_name() == 'internal' + deps_public_subproject += maybe_subproject_dep + # subproject sadly no good for pkg-config module + deps_other += maybe_subproject_dep + else + deps_public += maybe_subproject_dep + endif +endforeach diff --git a/build-utils-meson/threads/meson.build b/build-utils-meson/threads/meson.build new file mode 100644 index 000000000..294160de1 --- /dev/null +++ b/build-utils-meson/threads/meson.build @@ -0,0 +1,6 @@ +# This is only conditional to work around +# https://github.com/mesonbuild/meson/issues/13293. It should be +# unconditional. +if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') + deps_private += dependency('threads') +endif diff --git a/configure.ac b/configure.ac index 90a6d45d5..4f66a3efc 100644 --- a/configure.ac +++ b/configure.ac @@ -149,11 +149,6 @@ AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build th ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) AC_SUBST(ENABLE_UNIT_TESTS) -# Build external API docs by default -AC_ARG_ENABLE(external_api_docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's C interface]), - external_api_docs=$enableval, external_api_docs=yes) -AC_SUBST(external_api_docs) - AS_IF( [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) @@ -171,15 +166,6 @@ AS_IF( [test "$ENABLE_BUILD" == "no" && test "$ENABLE_DOC_GEN" == "yes"], [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])]) -# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. -AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), - ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no) -AC_SUBST(ENABLE_INTERNAL_API_DOCS) - -AC_ARG_ENABLE(external-api-docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's external unstable C interfaces]), - ENABLE_EXTERNAL_API_DOCS=$enableval, ENABLE_EXTERNAL_API_DOCS=no) -AC_SUBST(ENABLE_EXTERNAL_API_DOCS) - AS_IF( [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"], [NEED_PROG(jq, jq)]) @@ -369,6 +355,14 @@ if test "$gc" = yes; then PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS" AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.]) + + # See `fixupBoehmStackPointer`, for the integration between Boehm GC + # and Boost coroutines. + old_CFLAGS="$CFLAGS" + # Temporary set `-pthread` just for the next check + CFLAGS="$CFLAGS -pthread" + AC_CHECK_FUNCS([pthread_attr_get_np pthread_getattr_np]) + CFLAGS="$old_CFLAGS" fi AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ @@ -406,6 +400,11 @@ AS_CASE(["$enable_markdown"], PKG_CHECK_MODULES([LIBGIT2], [libgit2]) +# Look for toml11, a required dependency. +AC_LANG_PUSH(C++) +AC_CHECK_HEADER([toml.hpp], [], [AC_MSG_ERROR([toml11 is not found.])]) +AC_LANG_POP(C++) + # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) diff --git a/dep-patches/boehmgc-traceable_allocator-public.diff b/dep-patches/boehmgc-traceable_allocator-public.diff deleted file mode 100644 index 903c707a6..000000000 --- a/dep-patches/boehmgc-traceable_allocator-public.diff +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/include/gc_allocator.h b/include/gc_allocator.h -index 597c7f13..587286be 100644 ---- a/include/gc_allocator.h -+++ b/include/gc_allocator.h -@@ -312,6 +312,7 @@ public: - - template<> - class traceable_allocator { -+public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef void* pointer; diff --git a/doc/external-api/local.mk b/doc/external-api/local.mk deleted file mode 100644 index ae2b44db8..000000000 --- a/doc/external-api/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -$(docdir)/external-api/html/index.html $(docdir)/external-api/latex: $(d)/doxygen.cfg src/lib*-c/*.h - mkdir -p $(docdir)/external-api - { cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/external-api" ; } | doxygen - - -# Generate the HTML API docs for Nix's unstable C bindings -.PHONY: external-api-html -external-api-html: $(docdir)/external-api/html/index.html diff --git a/doc/internal-api/local.mk b/doc/internal-api/local.mk deleted file mode 100644 index be9b7bb55..000000000 --- a/doc/internal-api/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -$(docdir)/internal-api/html/index.html $(docdir)/internal-api/latex: $(d)/doxygen.cfg src/**/*.hh - mkdir -p $(docdir)/internal-api - { cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/internal-api" ; } | doxygen - - -# Generate the HTML API docs for Nix's unstable internal interfaces. -.PHONY: internal-api-html -internal-api-html: $(docdir)/internal-api/html/index.html diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix deleted file mode 100644 index cccd1e279..000000000 --- a/doc/manual/generate-builtin-constants.nix +++ /dev/null @@ -1,31 +0,0 @@ -let - inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ) optionalString squash; -in - -builtinsInfo: -let - showBuiltin = name: { doc, type, impure-only }: - let - type' = optionalString (type != null) " (${type})"; - - impureNotice = optionalString impure-only '' - > **Note** - > - > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). - ''; - in - squash '' -
- ${name}${type'} -
-
- - ${doc} - - ${impureNotice} - -
- ''; -in -concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 007b698f1..13de6c397 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -5,8 +5,10 @@ in builtinsInfo: let - showBuiltin = name: { doc, args, arity, experimental-feature }: + showBuiltin = name: { doc, type ? null, args ? [ ], experimental-feature ? null, impure-only ? false }: let + type' = optionalString (type != null) " (${type})"; + experimentalNotice = optionalString (experimental-feature != null) '' > **Note** > @@ -18,18 +20,26 @@ let > extra-experimental-features = ${experimental-feature} > ``` ''; + + impureNotice = optionalString impure-only '' + > **Note** + > + > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + ''; in squash ''
- ${name} ${listArgs args} + ${name}${listArgs args}${type'}
${experimentalNotice} ${doc} + + ${impureNotice}
''; - listArgs = args: concatStringsSep " " (map (s: "${s}") args); + listArgs = args: concatStringsSep "" (map (s: " ${s}") args); in concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 71ad5c8e6..0cec52885 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -140,16 +140,10 @@ $(d)/xp-features.json: $(doc_nix) $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix) @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix) - @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; - @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp - @mv $@.tmp $@ - $(d)/language.json: $(doc_nix) $(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp @mv $@.tmp $@ @@ -217,7 +211,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 633932e55..0f9f91b03 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,7 @@ // redirect rules for URL fragments (client-side) to prevent link rot. // this must be done on the client side, as web servers do not see the fragment part of the URL. // it will only work with JavaScript enabled in the browser, but this is the best we can do here. -// see ./_redirects for path redirects (client-side) +// see src/_redirects for path redirects (server-side) // redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. @@ -238,12 +238,12 @@ const redirects = { "attr-system": "language/derivations.html#attr-system", "ssec-derivation": "language/derivations.html", "ch-expression-language": "language/index.html", - "sec-constructs": "language/constructs.html", - "sect-let-language": "language/constructs.html#let-language", - "ss-functions": "language/constructs.html#functions", + "sec-constructs": "language/syntax.html", + "sect-let-language": "language/syntax.html#let-expressions", + "ss-functions": "language/syntax.html#functions", "sec-language-operators": "language/operators.html", "table-operators": "language/operators.html", - "ssec-values": "language/values.html", + "ssec-values": "language/types.html", "gloss-closure": "glossary.html#gloss-closure", "gloss-derivation": "glossary.html#gloss-derivation", "gloss-deriver": "glossary.html#gloss-deriver", @@ -335,11 +335,15 @@ const redirects = { "ssec-relnotes-2.2": "release-notes/rl-2.2.html", "ssec-relnotes-2.3": "release-notes/rl-2.3.html", }, - "language/values.html": { + "language/types.html": { "simple-values": "#primitives", "lists": "#list", "strings": "#string", "attribute-sets": "#attribute-set", + "type-number": "#type-int", + }, + "language/syntax.html": { + "scoping-rules": "scoping.html", }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", diff --git a/doc/manual/rl-next/drop-vendored-toml11.md b/doc/manual/rl-next/drop-vendored-toml11.md new file mode 100644 index 000000000..d1feeb703 --- /dev/null +++ b/doc/manual/rl-next/drop-vendored-toml11.md @@ -0,0 +1,6 @@ +--- +synopsis: Stop vendoring toml11 +--- + +We don't apply any patches to it, and vendoring it locks users into +bugs (it hasn't been updated since its introduction in late 2021). diff --git a/doc/manual/rl-next/harden-user-sandboxing.md b/doc/manual/rl-next/harden-user-sandboxing.md new file mode 100644 index 000000000..a647acf25 --- /dev/null +++ b/doc/manual/rl-next/harden-user-sandboxing.md @@ -0,0 +1,7 @@ +--- +synopsis: Harden the user sandboxing +significance: significant +issues: +--- + +The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. diff --git a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md new file mode 100644 index 000000000..99be4148b --- /dev/null +++ b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md @@ -0,0 +1,28 @@ +--- +synopsis: "`nix-shell ` looks for `shell.nix`" +significance: significant +issues: +- 496 +- 2279 +- 4529 +- 5431 +- 11053 +prs: +- 11057 +--- + +`nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + +Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. +This adjustment addresses a commonly reported problem. + +This also applies to `nix-shell` shebang scripts. Consider the following example: + +```shell +#!/usr/bin/env nix-shell +#!nix-shell -i bash +``` + +This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + +The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index cb54a3822..a6a2101e9 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -25,17 +25,17 @@ - [Store Types](store/types/index.md) {{#include ./store/types/SUMMARY.md}} - [Nix Language](language/index.md) - - [Data Types](language/values.md) - - [Language Constructs](language/constructs.md) + - [Data Types](language/types.md) + - [String context](language/string-context.md) + - [Syntax and semantics](language/syntax.md) + - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) - [Lookup path](language/constructs/lookup-path.md) - - [String context](language/string-context.md) - [Operators](language/operators.md) - - [Derivations](language/derivations.md) - - [Advanced Attributes](language/advanced-attributes.md) - - [Import From Derivation](language/import-from-derivation.md) - - [Built-in Constants](language/builtin-constants.md) - - [Built-in Functions](language/builtins.md) + - [Built-ins](language/builtins.md) + - [Derivations](language/derivations.md) + - [Advanced Attributes](language/advanced-attributes.md) + - [Import From Derivation](language/import-from-derivation.md) - [Package Management](package-management/index.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) diff --git a/doc/manual/src/_redirects b/doc/manual/src/_redirects index a04a36f1e..578c48f06 100644 --- a/doc/manual/src/_redirects +++ b/doc/manual/src/_redirects @@ -1,5 +1,5 @@ # redirect rules for paths (server-side) to prevent link rot. -# see ./redirects.js for redirects based on URL fragments (client-side) +# see ../redirects.js for redirects based on URL fragments (client-side) # # concrete user story this supports: # - user finds URL to the manual for Nix x.y @@ -27,6 +27,9 @@ /expressions/language-operators /language/operators 301! /expressions/language-values /language/values 301! /expressions/* /language/:splat 301! +/language/values /language/types 301! +/language/constructs /language/syntax 301! +/language/builtin-constants /language/builtins 301! /installation/installation /installation 301! diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 34e0dbfbd..d3f5f9c14 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -10,7 +10,7 @@ Most Nix commands interpret the following environment variables: - [`NIX_PATH`](#env-NIX_PATH) A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/values.md#type-path) + expressions using [paths](@docroot@/language/types.md#type-path) enclosed in angle brackets (i.e., ``), e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index e4223b542..3bb59cbed 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -55,20 +55,20 @@ All options not listed here are passed to [`nix-store --realise`](nix-store/realise.md), except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md). - - [`--no-out-link`](#opt-no-out-link) +- [`--no-out-link`](#opt-no-out-link) - Do not create a symlink to the output path. Note that as a result - the output does not become a root of the garbage collector, and so - might be deleted by `nix-store --gc`. + Do not create a symlink to the output path. Note that as a result + the output does not become a root of the garbage collector, and so + might be deleted by `nix-store --gc`. - - [`--dry-run`](#opt-dry-run) +- [`--dry-run`](#opt-dry-run) - Show what store paths would be built or downloaded. + Show what store paths would be built or downloaded. - - [`--out-link`](#opt-out-link) / `-o` *outlink* +- [`--out-link`](#opt-out-link) / `-o` *outlink* - Change the name of the symlink to the output path created from - `result` to *outlink*. + Change the name of the symlink to the output path created from + `result` to *outlink*. {{#include ./status-build-failure.md}} diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index cebbc7b00..8b58392b7 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -27,40 +27,46 @@ The moving parts of channels are: This command has the following operations: - - `--add` *url* \[*name*\]\ - Add a channel *name* located at *url* to the list of subscribed channels. - If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. +- `--add` *url* \[*name*\] - > **Note** - > - > `--add` does not automatically perform an update. - > Use `--update` explicitly. + Add a channel *name* located at *url* to the list of subscribed channels. + If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. - A channel URL must point to a directory containing a file `nixexprs.tar.gz`. - At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. + > **Note** + > + > `--add` does not automatically perform an update. + > Use `--update` explicitly. - - `--remove` *name*\ - Remove the channel *name* from the list of subscribed channels. + A channel URL must point to a directory containing a file `nixexprs.tar.gz`. + At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. - - `--list`\ - Print the names and URLs of all subscribed channels on standard output. +- `--remove` *name* - - `--update` \[*names*…\]\ - Download the Nix expressions of subscribed channels and create a new generation. - Update all channels if none is specified, and only those included in *names* otherwise. + Remove the channel *name* from the list of subscribed channels. - - `--list-generations`\ - Prints a list of all the current existing generations for the - channel profile. +- `--list` - Works the same way as - ``` - nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations - ``` + Print the names and URLs of all subscribed channels on standard output. - - `--rollback` \[*generation*\]\ - Revert channels to the state before the last call to `nix-channel --update`. - Optionally, you can specify a specific channel *generation* number to restore. +- `--update` \[*names*…\] + + Download the Nix expressions of subscribed channels and create a new generation. + Update all channels if none is specified, and only those included in *names* otherwise. + +- `--list-generations` + + Prints a list of all the current existing generations for the + channel profile. + + Works the same way as + ``` + nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations + ``` + +- `--rollback` \[*generation*\] + + Revert channels to the state before the last call to `nix-channel --update`. + Optionally, you can specify a specific channel *generation* number to restore. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 8e1307c48..2136d28e9 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -48,12 +48,14 @@ Instead, it looks in a few locations, and acts on all profiles it finds there: These options are for deleting old [profiles] prior to deleting unreachable [store objects]. -- [`--delete-old`](#opt-delete-old) / `-d`\ +- [`--delete-old`](#opt-delete-old) / `-d` + Delete all old generations of profiles. This is the equivalent of invoking [`nix-env --delete-generations old`](@docroot@/command-ref/nix-env/delete-generations.md#generations-old) on each found profile. -- [`--delete-older-than`](#opt-delete-older-than) *period*\ +- [`--delete-older-than`](#opt-delete-older-than) *period* + Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). *period* is a value such as `30d`, which would mean 30 days. diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index d94bde3a3..8cfd6ebad 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -27,38 +27,38 @@ When using public key authentication, you can avoid typing the passphrase with ` # Options - - `--to` +- `--to` - Copy the closure of _paths_ from a Nix store accessible from the local machine to the Nix store on the remote _machine_. - This is the default behavior. + Copy the closure of _paths_ from a Nix store accessible from the local machine to the Nix store on the remote _machine_. + This is the default behavior. - - `--from` +- `--from` - Copy the closure of _paths_ from the Nix store on the remote _machine_ to the local machine's specified Nix store. + Copy the closure of _paths_ from the Nix store on the remote _machine_ to the local machine's specified Nix store. - - `--gzip` +- `--gzip` - Enable compression of the SSH connection. + Enable compression of the SSH connection. - - `--include-outputs` +- `--include-outputs` - Also copy the outputs of [store derivation]s included in the closure. + Also copy the outputs of [store derivation]s included in the closure. - [store derivation]: @docroot@/glossary.md#gloss-store-derivation + [store derivation]: @docroot@/glossary.md#gloss-store-derivation - - `--use-substitutes` / `-s` +- `--use-substitutes` / `-s` - Attempt to download missing store objects on the target from [substituters](@docroot@/command-ref/conf-file.md#conf-substituters). - Any store objects that cannot be substituted on the target are still copied normally from the source. - This is useful, for instance, if the connection between the source and target machine is slow, but the connection between the target machine and `cache.nixos.org` (the default binary cache server) is fast. + Attempt to download missing store objects on the target from [substituters](@docroot@/command-ref/conf-file.md#conf-substituters). + Any store objects that cannot be substituted on the target are still copied normally from the source. + This is useful, for instance, if the connection between the source and target machine is slow, but the connection between the target machine and `cache.nixos.org` (the default binary cache server) is fast. {{#include ./opt-common.md}} # Environment variables - - `NIX_SSHOPTS` +- `NIX_SSHOPTS` - Additional options to be passed to `ssh` on the command line. + Additional options to be passed to `ssh` on the command line. {{#include ./env-common.md}} diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index ae618b2c6..b1ff0bb69 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -12,7 +12,8 @@ This operation deletes the specified generations of the current profile. *generations* can be a one of the following: -- [`...`](#generations-list):\ +- [`...`](#generations-list) + A list of generation numbers, each one a separate command-line argument. Delete exactly the profile generations given by their generation number. @@ -30,7 +31,8 @@ This operation deletes the specified generations of the current profile. > Because one can roll back to a previous generation, it is possible to have generations newer than the current one. > They will also be deleted. -- [`d`](#generations-time):\ +- [`d`](#generations-time) + The last *number* days *Example*: `30d` @@ -38,7 +40,8 @@ This operation deletes the specified generations of the current profile. Delete all generations created more than *number* days ago, except the most recent one of them. This allows rolling back to generations that were available within the specified period. -- [`+`](#generations-count):\ +- [`+`](#generations-count) + The last *number* generations up to the present *Example*: `+5` diff --git a/doc/manual/src/command-ref/nix-env/env-common.md b/doc/manual/src/command-ref/nix-env/env-common.md index 735817959..200da7219 100644 --- a/doc/manual/src/command-ref/nix-env/env-common.md +++ b/doc/manual/src/command-ref/nix-env/env-common.md @@ -1,6 +1,7 @@ # Environment variables -- `NIX_PROFILE`\ +- `NIX_PROFILE` + Location of the Nix profile. Defaults to the target of the symlink `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default` otherwise. diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index 85f37904f..748dd1e7a 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -21,125 +21,125 @@ It is based on the current generation of the active [profile](@docroot@/command- The arguments *args* map to store paths in a number of possible ways: - - By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression]. - These are [realised], and the resulting output paths are installed. - Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. +- By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression]. + These are [realised], and the resulting output paths are installed. + Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. - [derivation]: @docroot@/glossary.md#gloss-derivation - [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md - [realised]: @docroot@/glossary.md#gloss-realise + [derivation]: @docroot@/glossary.md#gloss-derivation + [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md + [realised]: @docroot@/glossary.md#gloss-realise - If there are multiple derivations matching a name in *args* that - have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the - derivation with the highest *priority* is used. A derivation can - define a priority by declaring the `meta.priority` attribute. This - attribute should be a number, with a higher value denoting a lower - priority. The default priority is `5`. + If there are multiple derivations matching a name in *args* that + have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the + derivation with the highest *priority* is used. A derivation can + define a priority by declaring the `meta.priority` attribute. This + attribute should be a number, with a higher value denoting a lower + priority. The default priority is `5`. - If there are multiple matching derivations with the same priority, - then the derivation with the highest version will be installed. + If there are multiple matching derivations with the same priority, + then the derivation with the highest version will be installed. - You can force the installation of multiple derivations with the same - name by being specific about the versions. For instance, `nix-env --install - gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will - probably cause a user environment conflict\!). + You can force the installation of multiple derivations with the same + name by being specific about the versions. For instance, `nix-env --install + gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will + probably cause a user environment conflict\!). - - If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. - This is faster than using derivation names and unambiguous. - Show the attribute paths of available packages with [`nix-env --query`](./query.md): +- If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. + This is faster than using derivation names and unambiguous. + Show the attribute paths of available packages with [`nix-env --query`](./query.md): - ```console - nix-env --query --available --attr-path - ``` + ```console + nix-env --query --available --attr-path + ``` - - If `--from-profile` *path* is given, *args* is a set of names - denoting installed [store paths] in the profile *path*. This is an - easy way to copy user environment elements from one profile to - another. +- If `--from-profile` *path* is given, *args* is a set of names + denoting installed [store paths] in the profile *path*. This is an + easy way to copy user environment elements from one profile to + another. - - If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/constructs.md#functions) that are called with the [default Nix expression] as their single argument. - The derivations returned by those function calls are installed. - This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. +- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the [default Nix expression] as their single argument. + The derivations returned by those function calls are installed. + This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. - - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. +- If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. - - If *args* are [store paths] that are not store derivations, then these are [realised] and installed. +- If *args* are [store paths] that are not store derivations, then these are [realised] and installed. - - By default all [outputs](@docroot@/language/derivations.md#attr-outputs) are installed for each [derivation]. - This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. +- By default all [outputs](@docroot@/language/derivations.md#attr-outputs) are installed for each [derivation]. + This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. - Example: + Example: - The file `example.nix` defines a derivation with two outputs `foo` and `bar`, each containing a file. + The file `example.nix` defines a derivation with two outputs `foo` and `bar`, each containing a file. - ```nix - # example.nix - let - pkgs = import {}; - command = '' - ${pkgs.coreutils}/bin/mkdir -p $foo $bar - echo foo > $foo/foo-file - echo bar > $bar/bar-file - ''; - in - derivation { - name = "example"; - builder = "${pkgs.bash}/bin/bash"; - args = [ "-c" command ]; - outputs = [ "foo" "bar" ]; - system = builtins.currentSystem; - } - ``` + ```nix + # example.nix + let + pkgs = import {}; + command = '' + ${pkgs.coreutils}/bin/mkdir -p $foo $bar + echo foo > $foo/foo-file + echo bar > $bar/bar-file + ''; + in + derivation { + name = "example"; + builder = "${pkgs.bash}/bin/bash"; + args = [ "-c" command ]; + outputs = [ "foo" "bar" ]; + system = builtins.currentSystem; + } + ``` - Installing from this Nix expression will make files from both outputs appear in the current profile. + Installing from this Nix expression will make files from both outputs appear in the current profile. - ```console - $ nix-env --install --file example.nix - installing 'example' - $ ls ~/.nix-profile - foo-file - bar-file - manifest.nix - ``` + ```console + $ nix-env --install --file example.nix + installing 'example' + $ ls ~/.nix-profile + foo-file + bar-file + manifest.nix + ``` - Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. + Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. - ```nix - # example-outputs.nix - import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } - ``` + ```nix + # example-outputs.nix + import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } + ``` - ```console - $ nix-env --install --file example-outputs.nix - installing 'example' - $ ls ~/.nix-profile - bar-file - manifest.nix - ``` + ```console + $ nix-env --install --file example-outputs.nix + installing 'example' + $ ls ~/.nix-profile + bar-file + manifest.nix + ``` # Options - - `--prebuilt-only` / `-b` +- `--prebuilt-only` / `-b` - Use only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, no packages will be built from - source. + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. - - `--preserve-installed` / `-P` +- `--preserve-installed` / `-P` - Do not remove derivations with a name matching one of the - derivations being installed. Usually, trying to have two versions of - the same package installed in the same generation of a profile will - lead to an error in building the generation, due to file name - clashes between the two versions. However, this is not the case for - all packages. + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. - - `--remove-all` / `-r` +- `--remove-all` / `-r` - Remove all previously installed packages first. This is equivalent - to running `nix-env --uninstall '.*'` first, except that everything happens - in a single transaction. + Remove all previously installed packages first. This is equivalent + to running `nix-env --uninstall '.*'` first, except that everything happens + in a single transaction. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-env/opt-common.md b/doc/manual/src/command-ref/nix-env/opt-common.md index 636281b6d..1479ca0bd 100644 --- a/doc/manual/src/command-ref/nix-env/opt-common.md +++ b/doc/manual/src/command-ref/nix-env/opt-common.md @@ -2,34 +2,37 @@ The following options are allowed for all `nix-env` operations, but may not always have an effect. - - `--file` / `-f` *path*\ - Specifies the Nix expression (designated below as the *active Nix - expression*) used by the `--install`, `--upgrade`, and `--query - --available` operations to obtain derivations. The default is - `~/.nix-defexpr`. +- `--file` / `-f` *path* - If the argument starts with `http://` or `https://`, it is - interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must include a single - top-level directory containing at least a file named `default.nix`. + Specifies the Nix expression (designated below as the *active Nix + expression*) used by the `--install`, `--upgrade`, and `--query + --available` operations to obtain derivations. The default is + `~/.nix-defexpr`. - - `--profile` / `-p` *path*\ - Specifies the profile to be used by those operations that operate on - a profile (designated below as the *active profile*). A profile is a - sequence of user environments called *generations*, one of which is - the *current generation*. + If the argument starts with `http://` or `https://`, it is + interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must include a single + top-level directory containing at least a file named `default.nix`. - - `--dry-run`\ - For the `--install`, `--upgrade`, `--uninstall`, - `--switch-generation`, `--delete-generations` and `--rollback` - operations, this flag will cause `nix-env` to print what *would* be - done if this flag had not been specified, without actually doing it. +- `--profile` / `-p` *path* - `--dry-run` also prints out which paths will be - [substituted](@docroot@/glossary.md) (i.e., downloaded) and which paths - will be built from source (because no substitute is available). + Specifies the profile to be used by those operations that operate on + a profile (designated below as the *active profile*). A profile is a + sequence of user environments called *generations*, one of which is + the *current generation*. - - `--system-filter` *system*\ - By default, operations such as `--query - --available` show derivations matching any platform. This option - allows you to use derivations for the specified platform *system*. +- `--dry-run` + + For the `--install`, `--upgrade`, `--uninstall`, + `--switch-generation`, `--delete-generations` and `--rollback` + operations, this flag will cause `nix-env` to print what *would* be + done if this flag had not been specified, without actually doing it. + + `--dry-run` also prints out which paths will be + [substituted](@docroot@/glossary.md) (i.e., downloaded) and which paths + will be built from source (because no substitute is available). + +- `--system-filter` *system* + + By default, operations such as `--query --available` show derivations matching any platform. This option + allows you to use derivations for the specified platform *system*. diff --git a/doc/manual/src/command-ref/nix-env/query.md b/doc/manual/src/command-ref/nix-env/query.md index c9b4d8513..c67794ed5 100644 --- a/doc/manual/src/command-ref/nix-env/query.md +++ b/doc/manual/src/command-ref/nix-env/query.md @@ -35,11 +35,13 @@ The derivations are sorted by their `name` attributes. The following flags specify the set of things on which the query operates. - - `--installed`\ + - `--installed` + The query operates on the store paths that are installed in the current generation of the active profile. This is the default. - - `--available`; `-a`\ + - `--available` / `-a` + The query operates on the derivations that are available in the active Nix expression. @@ -50,24 +52,28 @@ selected derivations. Multiple flags may be specified, in which case the information is shown in the order given here. Note that the name of the derivation is shown unless `--no-name` is specified. - - `--xml`\ + - `--xml` + Print the result in an XML representation suitable for automatic processing by other tools. The root element is called `items`, which contains a `item` element for each available or installed derivation. The fields discussed below are all stored in attributes of the `item` elements. - - `--json`\ + - `--json` + Print the result in a JSON representation suitable for automatic processing by other tools. - - `--prebuilt-only` / `-b`\ + - `--prebuilt-only` / `-b` + Show only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can be downloaded in lieu of building the derivation. Thus, this shows all packages that probably can be installed quickly. - - `--status`; `-s`\ + - `--status` / `-s` + Print the *status* of the derivation. The status consists of three characters. The first is `I` or `-`, indicating whether the derivation is currently installed in the current generation of the @@ -78,49 +84,61 @@ derivation is shown unless `--no-name` is specified. derivation to be built. The third is `S` or `-`, indicating whether a substitute is available for the derivation. - - `--attr-path`; `-P`\ + - `--attr-path` / `-P` + Print the *attribute path* of the derivation, which can be used to unambiguously select it using the `--attr` option available in commands that install derivations like `nix-env --install`. This option only works together with `--available` - - `--no-name`\ + - `--no-name` + Suppress printing of the `name` attribute of each derivation. - - `--compare-versions` / `-c`\ + - `--compare-versions` / `-c` + Compare installed versions to available versions, or vice versa (if `--available` is given). This is useful for quickly seeing whether upgrades for installed packages are available in a Nix expression. A column is added with the following meaning: - - `<` *version*\ + - `<` *version* + A newer version of the package is available or installed. - - `=` *version*\ + - `=` *version* + At most the same version of the package is available or installed. - - `>` *version*\ + - `>` *version* + Only older versions of the package are available or installed. - - `- ?`\ + - `- ?` + No version of the package is available or installed. - - `--system`\ + - `--system` + Print the `system` attribute of the derivation. - - `--drv-path`\ + - `--drv-path` + Print the path of the [store derivation](@docroot@/glossary.md#gloss-store-derivation). - - `--out-path`\ + - `--out-path` + Print the output path of the derivation. - - `--description`\ + - `--description` + Print a short (one-line) description of the derivation, if available. The description is taken from the `meta.description` attribute of the derivation. - - `--meta`\ + - `--meta` + Print all of the meta-attributes of the derivation. This option is only available with `--xml` or `--json`. diff --git a/doc/manual/src/command-ref/nix-env/set-flag.md b/doc/manual/src/command-ref/nix-env/set-flag.md index e04b22a91..58a0248bb 100644 --- a/doc/manual/src/command-ref/nix-env/set-flag.md +++ b/doc/manual/src/command-ref/nix-env/set-flag.md @@ -13,24 +13,24 @@ to be modified. There are several attributes that can be usefully modified, because they affect the behaviour of `nix-env` or the user environment build script: - - `priority` can be changed to resolve filename clashes. The user - environment build script uses the `meta.priority` attribute of - derivations to resolve filename collisions between packages. Lower - priority values denote a higher priority. For instance, the GCC - wrapper package and the Binutils package in Nixpkgs both have a file - `bin/ld`, so previously if you tried to install both you would get a - collision. Now, on the other hand, the GCC wrapper declares a higher - priority than Binutils, so the former’s `bin/ld` is symlinked in the - user environment. +- `priority` can be changed to resolve filename clashes. The user + environment build script uses the `meta.priority` attribute of + derivations to resolve filename collisions between packages. Lower + priority values denote a higher priority. For instance, the GCC + wrapper package and the Binutils package in Nixpkgs both have a file + `bin/ld`, so previously if you tried to install both you would get a + collision. Now, on the other hand, the GCC wrapper declares a higher + priority than Binutils, so the former’s `bin/ld` is symlinked in the + user environment. - - `keep` can be set to `true` to prevent the package from being - upgraded or replaced. This is useful if you want to hang on to an - older version of a package. +- `keep` can be set to `true` to prevent the package from being + upgraded or replaced. This is useful if you want to hang on to an + older version of a package. - - `active` can be set to `false` to “disable” the package. That is, no - symlinks will be generated to the files of the package, but it - remains part of the profile (so it won’t be garbage-collected). It - can be set back to `true` to re-enable the package. +- `active` can be set to `false` to “disable” the package. That is, no + symlinks will be generated to the files of the package, but it + remains part of the profile (so it won’t be garbage-collected). It + can be set back to `true` to re-enable the package. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-env/upgrade.md b/doc/manual/src/command-ref/nix-env/upgrade.md index 322dfbda2..2779363c3 100644 --- a/doc/manual/src/command-ref/nix-env/upgrade.md +++ b/doc/manual/src/command-ref/nix-env/upgrade.md @@ -28,42 +28,48 @@ version is installed. # Flags - - `--lt`\ - Only upgrade a derivation to newer versions. This is the default. +- `--lt` - - `--leq`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same version. Version are not a unique - identification of a derivation, so there may be many derivations - that have the same version. This flag may be useful to force - “synchronisation” between the installed and available derivations. + Only upgrade a derivation to newer versions. This is the default. - - `--eq`\ - *Only* “upgrade” to derivations that have the same version. This may - not seem very useful, but it actually is, e.g., when there is a new - release of Nixpkgs and you want to replace installed applications - with the same versions built against newer dependencies (to reduce - the number of dependencies floating around on your system). +- `--leq` - - `--always`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same or a lower version. I.e., derivations - may actually be downgraded depending on what is available in the - active Nix expression. + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same version. Version are not a unique + identification of a derivation, so there may be many derivations + that have the same version. This flag may be useful to force + “synchronisation” between the installed and available derivations. - - `--prebuilt-only` / `-b`\ - Use only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, no packages will be built from - source. +- `--eq` - - `--preserve-installed` / `-P`\ - Do not remove derivations with a name matching one of the - derivations being installed. Usually, trying to have two versions of - the same package installed in the same generation of a profile will - lead to an error in building the generation, due to file name - clashes between the two versions. However, this is not the case for - all packages. + *Only* “upgrade” to derivations that have the same version. This may + not seem very useful, but it actually is, e.g., when there is a new + release of Nixpkgs and you want to replace installed applications + with the same versions built against newer dependencies (to reduce + the number of dependencies floating around on your system). + +- `--always` + + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same or a lower version. I.e., derivations + may actually be downgraded depending on what is available in the + active Nix expression. + +- `--prebuilt-only` / `-b` + + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. + +- `--preserve-installed` / `-P` + + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md index 24e91df12..f249c2b84 100644 --- a/doc/manual/src/command-ref/nix-hash.md +++ b/doc/manual/src/command-ref/nix-hash.md @@ -29,54 +29,65 @@ md5sum`. # Options - - `--flat`\ - Print the cryptographic hash of the contents of each regular file *path*. - That is, instead of computing - the hash of the [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) of *path*, - just [directly hash]((@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is. - This requires *path* to resolve to a regular file rather than directory. - The result is identical to that produced by the GNU commands - `md5sum` and `sha1sum`. +- `--flat` - - `--base16`\ - Print the hash in a hexadecimal representation (default). + Print the cryptographic hash of the contents of each regular file *path*. + That is, instead of computing + the hash of the [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) of *path*, + just [directly hash]((@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is. + This requires *path* to resolve to a regular file rather than directory. + The result is identical to that produced by the GNU commands + `md5sum` and `sha1sum`. - - `--base32`\ - Print the hash in a base-32 representation rather than hexadecimal. - This base-32 representation is more compact and can be used in Nix - expressions (such as in calls to `fetchurl`). +- `--base16` - - `--base64`\ - Similar to --base32, but print the hash in a base-64 representation, - which is more compact than the base-32 one. + Print the hash in a hexadecimal representation (default). - - `--sri`\ - Print the hash in SRI format with base-64 encoding. - The type of hash algorithm will be prepended to the hash string, - followed by a hyphen (-) and the base-64 hash body. +- `--base32` - - `--truncate`\ - Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits. + Print the hash in a base-32 representation rather than hexadecimal. + This base-32 representation is more compact and can be used in Nix + expressions (such as in calls to `fetchurl`). - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, `sha256`, and `sha512`. +- `--base64` - - `--to-base16`\ - Don’t hash anything, but convert the base-32 hash representation - *hash* to hexadecimal. + Similar to --base32, but print the hash in a base-64 representation, + which is more compact than the base-32 one. - - `--to-base32`\ - Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-32. +- `--sri` - - `--to-base64`\ - Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-64. + Print the hash in SRI format with base-64 encoding. + The type of hash algorithm will be prepended to the hash string, + followed by a hyphen (-) and the base-64 hash body. - - `--to-sri`\ - Don’t hash anything, but convert the hexadecimal hash representation - *hash* to SRI. +- `--truncate` + + Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits. + +- `--type` *hashAlgo* + + Use the specified cryptographic hash algorithm, which can be one of + `md5`, `sha1`, `sha256`, and `sha512`. + +- `--to-base16` + + Don’t hash anything, but convert the base-32 hash representation + *hash* to hexadecimal. + +- `--to-base32` + + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to base-32. + +- `--to-base64` + + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to base-64. + +- `--to-sri` + + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to SRI. # Examples diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index dffbb2d70..6f6fcdc1f 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -30,89 +30,97 @@ standard input. # Options - - `--add-root` *path*\ - See the [corresponding option](nix-store.md) in `nix-store`. +- `--add-root` *path* - - `--parse`\ - Just parse the input files, and print their abstract syntax trees on - standard output as a Nix expression. + See the [corresponding option](nix-store.md) in `nix-store`. - - `--eval`\ - Just parse and evaluate the input files, and print the resulting - values on standard output. No instantiation of store derivations - takes place. +- `--parse` - > **Warning** - > - > This option produces output which can be parsed as a Nix expression which - > will produce a different result than the input expression when evaluated. - > For example, these two Nix expressions print the same result despite - > having different meaning: - > - > ```console - > $ nix-instantiate --eval --expr '{ a = {}; }' - > { a = ; } - > $ nix-instantiate --eval --expr '{ a = ; }' - > { a = ; } - > ``` - > - > For human-readable output, `nix eval` (experimental) is more informative: - > - > ```console - > $ nix-instantiate --eval --expr 'a: a' - > - > $ nix eval --expr 'a: a' - > «lambda @ «string»:1:1» - > ``` - > - > For machine-readable output, the `--xml` option produces unambiguous - > output: - > - > ```console - > $ nix-instantiate --eval --xml --expr '{ foo = ; }' - > - > - > - > - > - > - > - > - > ``` + Just parse the input files, and print their abstract syntax trees on + standard output as a Nix expression. - - `--find-file`\ - Look up the given files in Nix’s search path (as specified by the - `NIX_PATH` environment variable). If found, print the corresponding - absolute paths on standard output. For instance, if `NIX_PATH` is - `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file - nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. +- `--eval` - - `--strict`\ - When used with `--eval`, recursively evaluate list elements and - attributes. Normally, such sub-expressions are left unevaluated - (since the Nix language is lazy). + Just parse and evaluate the input files, and print the resulting + values on standard output. No instantiation of store derivations + takes place. - > **Warning** - > - > This option can cause non-termination, because lazy data - > structures can be infinitely large. + > **Warning** + > + > This option produces output which can be parsed as a Nix expression which + > will produce a different result than the input expression when evaluated. + > For example, these two Nix expressions print the same result despite + > having different meaning: + > + > ```console + > $ nix-instantiate --eval --expr '{ a = {}; }' + > { a = ; } + > $ nix-instantiate --eval --expr '{ a = ; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index a4dd051fc..ddec30f5b 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real” Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real” Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “roots”, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live” store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead” store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live” store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead” store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index 0bcacfe0c..b4efa734e 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: @docroot@/glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: @docroot@/glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: @docroot@/glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: @docroot@/glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 6e56387eb..e30b351a4 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..a42909e2d 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -143,7 +143,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index 6e7c0a967..a14ecedd6 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -196,15 +196,16 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html ``` or inside `nix-shell` or `nix develop`: -``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html ``` ## C API documentation @@ -216,13 +217,14 @@ You can also build and view it yourself: [C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs ```console -# nix build .#hydraJobs.external-api-docs -# xdg-open ./result/share/doc/nix/external-api/html/index.html +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make external-api-html -# xdg-open ./outputs/doc/share/doc/nix/external-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 08ba84faa..c128515e9 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -122,7 +122,6 @@ Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/ The docs can take a while to build, so you may want to disable this for local development. - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. - `OPTIMIZE=1` to enable optimizations. - `libraries=libutil programs=` to only build a specific library. diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 88b3c5cd9..a96ba997b 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BREIF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,6 +131,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -252,13 +271,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 4c9b2c52e..f65ada63a 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -312,7 +312,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -325,9 +325,9 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set - [base directory]{#gloss-base-directory} diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 70ec06f57..e916c7360 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -302,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - -
diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ -
diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index 491d221b3..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,486 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; ① -assert httpServer -> httpd != null && httpd.expat == expat; ② -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; ④ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -Variables coming from outer `with` expressions *are* shadowed: - -```nix -with { a = "outer"; }; -with { a = "inner"; }; -a -``` - -Does evaluate to `"inner"`. - -## Comments - -- Inline comments start with `#` and run until the end of the line. - - > **Example** - > - > ```nix - > # A number - > 2 # Equals 1 + 1 - > ``` - > - > ```console - > 2 - > ``` - -- Block comments start with `/*` and run until the next occurrence of `*/`. - - > **Example** - > - > ```nix - > /* - > Block comments - > can span multiple lines. - > */ "hello" - > ``` - > - > ```console - > "hello" - > ``` - - This means that block comments cannot be nested. - - > **Example** - > - > ```nix - > /* /* nope */ */ 1 - > ``` - > - > ```console - > error: syntax error, unexpected '*' - > - > at «string»:1:15: - > - > 1| /* /* nope */ * - > | ^ - > ``` - - Consider escaping nested comments and unescaping them in post-processing. - - > **Example** - > - > ```nix - > /* /* nested *\/ */ 1 - > ``` - > - > ```console - > 1 - > ``` - -## Scoping rules - -Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope), but with multiple scopes and shadowing rules. - -* primary scope --- explicitly-bound variables - * [`let`](#let-expressions) - * [`inherit`](#inheriting-attributes) - * function arguments - -* secondary scope --- implicitly-bound variables - * [`with`](#with-expressions) -Primary scope takes precedence over secondary scope. -See [`with`](#with-expressions) for a detailed example. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..11b9fe88c 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,9 +4,9 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index b95900cdd..8e3f0f791 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,7 +12,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 3694480d7..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -53,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values ([primitives](@docroot@/language/values.md#primitives))* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -71,7 +71,7 @@ This is an incomplete overview of language features, by example. - A [string](@docroot@/language/values.md#type-string) + A [string](@docroot@/language/types.md#type-string) @@ -102,7 +102,7 @@ This is an incomplete overview of language features, by example. - A [comment](@docroot@/language/constructs.md#comments). + A [comment](@docroot@/language/syntax.md#comments). @@ -130,7 +130,7 @@ This is an incomplete overview of language features, by example. - [Booleans](@docroot@/language/values.md#type-boolean) + [Booleans](@docroot@/language/types.md#type-boolean) @@ -142,7 +142,7 @@ This is an incomplete overview of language features, by example. - [Null](@docroot@/language/values.md#type-null) value + [Null](@docroot@/language/types.md#type-null) value @@ -154,7 +154,7 @@ This is an incomplete overview of language features, by example. - An [integer](@docroot@/language/values.md#type-number) + An [integer](@docroot@/language/types.md#type-int) @@ -166,7 +166,7 @@ This is an incomplete overview of language features, by example. - A [floating point number](@docroot@/language/values.md#type-number) + A [floating point number](@docroot@/language/types.md#type-float) @@ -178,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute [path](@docroot@/language/values.md#type-path) + An absolute [path](@docroot@/language/types.md#type-path) @@ -190,7 +190,7 @@ This is an incomplete overview of language features, by example. - A [path](@docroot@/language/values.md#type-path) relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -202,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home [path](@docroot@/language/values.md#type-path). Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -238,7 +238,7 @@ This is an incomplete overview of language features, by example. - An [attribute set](@docroot@/language/values.md#attribute-set) with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -262,7 +262,7 @@ This is an incomplete overview of language features, by example. - A [recursive set](@docroot@/language/constructs.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -278,7 +278,7 @@ This is an incomplete overview of language features, by example. - [Lists](@docroot@/language/values.md#list) with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -362,7 +362,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/values.md#attribute-set) (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -374,7 +374,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/values.md#attribute-set) with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -410,7 +410,7 @@ This is an incomplete overview of language features, by example. - [Conditional expression](@docroot@/language/constructs.md#conditionals). + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -422,7 +422,7 @@ This is an incomplete overview of language features, by example. - [Assertion](@docroot@/language/constructs.md#assertions) check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -434,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition. See [`let`-expressions](@docroot@/language/constructs.md#let-expressions). + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -448,7 +448,7 @@ This is an incomplete overview of language features, by example. Add all attributes from the given set to the scope (evaluates to `1`). - See [`with`-expressions](@docroot@/language/constructs.md#with-expressions) for details and shadowing caveats. + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. @@ -462,7 +462,7 @@ This is an incomplete overview of language features, by example. Adds the variables to the current scope (attribute set or `let` binding). Desugars to `pkgs = pkgs; src = src;`. - See [Inheriting attributes](@docroot@/language/constructs.md#inheriting-attributes). + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). @@ -476,14 +476,14 @@ This is an incomplete overview of language features, by example. Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. - See [Inheriting attributes](@docroot@/language/constructs.md#inheriting-attributes). + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). - *[Functions](@docroot@/language/constructs.md#functions) (lambdas)* + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -500,7 +500,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects an integer and returns it increased by 1. + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -512,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried [function](@docroot@/language/constructs.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -524,7 +524,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -536,7 +536,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -548,7 +548,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -560,7 +560,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -572,7 +572,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -586,7 +586,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 9e5ab52a2..9660a764d 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -27,11 +27,11 @@ | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#list +[attribute set]: ./types.md#attribute-set ## Attribute selection @@ -42,7 +42,7 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). +An attribute path is a dot-separated list of [attribute names](./types.md#attribute-set). > **Syntax** > @@ -61,7 +61,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-boolean [Has attribute]: #has-attribute @@ -172,7 +172,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..5c6aed38d --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,14 @@ +# Scoping rules + +Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope), but with multiple scopes and shadowing rules. + +* primary scope: explicitly-bound variables + * [`let`](./syntax.md#let-expressions) + * [`inherit`](./syntax.md#inheriting-attributes) + * [function](./syntax.md#functions) arguments + +* secondary scope: implicitly-bound variables + * [`with`](./syntax.md#with-expressions) + +Primary scope takes precedence over secondary scope. +See [`with`](./syntax.md#with-expressions) for a detailed example. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md index 88ae0d8b0..6a3482cfd 100644 --- a/doc/manual/src/language/string-context.md +++ b/doc/manual/src/language/string-context.md @@ -111,7 +111,7 @@ It creates an [attribute set] representing the string context, which can be insp [`builtins.hasContext`]: ./builtins.md#builtins-hasContext [`builtins.getContext`]: ./builtins.md#builtins-getContext -[attribute set]: ./values.md#attribute-set +[attribute set]: ./types.md#attribute-set ## Clearing string contexts diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 1e2c4ad95..1778bdfa0 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,9 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set ## Examples @@ -43,6 +43,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..238c502f9 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,841 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + + *Strings* can be written in three ways. + + The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. + Strings can span multiple lines. + The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + + [string interpolation]: ./string-interpolation.md + + The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + + - Double quote (`"`) + + > **Example** + > + > ```nix + > "\"" + > ``` + > + > "\"" + + - Backslash (`\`) + + > **Example** + > + > ```nix + > "\\" + > ``` + > + > "\\" + + - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + + > **Example** + > + > ```nix + > "\${" + > ``` + > + > "\${" + + The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > "$${" + > ``` + > + > "$\${" + + String values are output on the terminal with Nix-specific escaping. + Strings written to files will contain the characters encoded by the escaping. + + The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is + + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` + + > **Note** + > + > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + + > **Warning** + > + > Prefixed tab characters are not stripped. + > + > > **Example** + > > + > > The following indented string is prefixed with tabs: + > > + > > '' + > > all: + > > @echo hello + > > '' + > > + > > "\tall:\n\t\t@echo hello\n" + + Indented strings support [string interpolation]. + + The following must be escaped to represent them in an indented string: + + - `$` is escaped by prefixing it with two single quotes (`''`) + + > **Example** + > + > ```nix + > '' + > ''$ + > '' + > ``` + > + > "$\n" + + - `''` is escaped by prefixing it with one single quote (`'`) + + > **Example** + > + > ```nix + > '' + > ''' + > '' + > ``` + > + > "''\n" + + These special characters are escaped as follows: + - Linefeed (`\n`): `''\n` + - Carriage return (`\r`): `''\r` + - Tab (`\t`): `''\t` + + `''\` escapes any other character. + + A "double-dollar-curly" (`$${`) can be written literally. + + > **Example** + > + > ```nix + > '' + > $${ + > '' + > ``` + > + > "$\${\n" + + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: + + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` + + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* are distinct from strings and can be expressed by path literals such as `./builder.sh`. + + Paths are suitable for referring to local files, and are often preferable over strings. + - Path values do not contain trailing slashes, `.` and `..`, as they are resolved when evaluating a path literal. + - Path literals are automatically resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + - The files referred to by path values are automatically copied into the Nix store when used in a string interpolation or concatenation. + - Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Paths can be used in [string interpolation] and string concatenation. + For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. + + Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. + For example, assume you used a file path in an interpolated string during a `nix repl` session. + Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. Use `:r` to reset the repl as needed. + + [store path]: @docroot@/store/store-path.md + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). + +An attribute name can be an identifier or a [string](#string). +An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). + +> **Syntax** +> +> *name* = *identifier* | *string* \ +> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` + +Names and values are separated by an equal sign (`=`). +Each value is an arbitrary expression terminated by a semicolon (`;`). + +> **Syntax** +> +> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` + +Attributes can appear in any order. +An attribute name may only occur once. + +Example: + +```nix +{ + x = 123; + text = "Hello"; + y = f { bla = 456; }; +} +``` + +This defines a set with attributes named `x`, `text`, `y`. + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a `__functor` attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; ① +assert httpServer -> httpd != null && httpd.expat == expat; ② +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; ④ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..c6cfb3c69 --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,91 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](syntax.md#string-literal). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + + + +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 4eb1887fa..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,280 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths* are distinct from strings and can be expressed by path literals such as `./builder.sh`. - - Paths are suitable for referring to local files, and are often preferable over strings. - - Path values do not contain trailing slashes, `.` and `..`, as they are resolved when evaluating a path literal. - - Path literals are automatically resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). - - The files referred to by path values are automatically copied into the Nix store when used in a string interpolation or concatenation. - - Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. - - A path literal must contain at least one slash to be recognised as such. - For instance, `builder.sh` is not a path: - it's parsed as an expression that selects the attribute `sh` from the variable `builder`. - - Path literals may also refer to absolute paths by starting with a slash. - - > **Note** - > - > Absolute paths make expressions less portable. - > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. - > This avoids some confusion about whether files at that location will be used during evaluation. - > It also avoids unintentional situations where some function might try to copy everything at the location into the store. - - If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. - For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. - Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. - - Paths can be used in [string interpolation] and string concatenation. - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. Use `:r` to reset the repl as needed. - - [store path]: @docroot@/store/store-path.md - - Path literals can also include [string interpolation], besides being [interpolated into other expressions]. - - [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid number division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md index bfc523b3d..640b527f1 100644 --- a/doc/manual/src/protocols/nix-archive.md +++ b/doc/manual/src/protocols/nix-archive.md @@ -29,7 +29,7 @@ regular = [ str("executable"), str("") ], str("contents"), str(contents); symlink = str("target"), str(target); (* side condition: directory entries must be ordered by their names *) -directory = str("type"), str("directory") { directory-entry }; +directory = { directory-entry }; directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); ``` diff --git a/flake.nix b/flake.nix index 5dbc554fc..d83c2ecad 100644 --- a/flake.nix +++ b/flake.nix @@ -25,7 +25,6 @@ let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; @@ -58,6 +57,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -117,76 +128,23 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = final.boehmgc.override { - enableLargeConfig = true; - }; + nix = final.nixComponents.nix; - libseccomp-nix = final.libseccomp.overrideAttrs (_: rec { - version = "2.5.5"; - src = final.fetchurl { - url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; - hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; - }; - }); - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - libseccomp = final.libseccomp-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; + nix_noTests = final.nix.override { + doInstallCheck = false; }; # See https://github.com/NixOS/nixpkgs/pull/214409 @@ -200,10 +158,10 @@ in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = import ./build/hydra.nix { + hydraJobs = import ./packaging/hydra.nix { inherit inputs binaryTarball @@ -232,15 +190,67 @@ # Some perl dependencies are broken on i686-linux. # Since the support is only best-effort there, disable the perl # bindings - perlBindings = self.hydraJobs.perlBindings.${system}; - } // devFlake.checks.${system} or {} + + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} ); - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -255,25 +265,26 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: let modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; in { pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -288,13 +299,31 @@ src = null; env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; # For `make format`, to work without installing pre-commit _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; }; + mesonFlags = + map (transformFlag "libutil") pkgs.nixComponents.nix-util.mesonFlags + ++ map (transformFlag "libstore") pkgs.nixComponents.nix-store.mesonFlags + ++ map (transformFlag "libfetchers") pkgs.nixComponents.nix-fetchers.mesonFlags + ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nixComponents.nix-perl-bindings.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs ++ [ + pkgs.buildPackages.cmake modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) @@ -303,6 +332,14 @@ # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: diff --git a/maintainers/README.md b/maintainers/README.md index bfa0cb5a1..2a718e283 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,7 +30,6 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 3006d5e30..46b3e1363 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -18,7 +18,6 @@ ''^tests/unit/[^/]*/data/.*$'' # Don't format vendored code - ''^src/toml11/.*'' ''^doc/manual/redirects\.js$'' ''^doc/manual/theme/highlight\.js$'' @@ -65,14 +64,6 @@ ''^src/libexpr/eval-settings\.hh$'' ''^src/libexpr/eval\.cc$'' ''^src/libexpr/eval\.hh$'' - ''^src/libexpr/flake/config\.cc$'' - ''^src/libexpr/flake/flake\.cc$'' - ''^src/libexpr/flake/flake\.hh$'' - ''^src/libexpr/flake/flakeref\.cc$'' - ''^src/libexpr/flake/flakeref\.hh$'' - ''^src/libexpr/flake/lockfile\.cc$'' - ''^src/libexpr/flake/lockfile\.hh$'' - ''^src/libexpr/flake/url-name\.cc$'' ''^src/libexpr/function-trace\.cc$'' ''^src/libexpr/gc-small-vector\.hh$'' ''^src/libexpr/get-drvs\.cc$'' @@ -127,6 +118,14 @@ ''^src/libfetchers/tarball\.hh$'' ''^src/libfetchers/git\.cc$'' ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' ''^src/libmain/common-args\.cc$'' ''^src/libmain/common-args\.hh$'' ''^src/libmain/loggers\.cc$'' @@ -429,14 +428,13 @@ ''^tests/functional/test-libstoreconsumer/main\.cc'' ''^tests/nixos/ca-fd-leak/sender\.c'' ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' ''^tests/unit/libexpr-support/tests/libexpr\.hh'' ''^tests/unit/libexpr-support/tests/value/context\.cc'' ''^tests/unit/libexpr-support/tests/value/context\.hh'' ''^tests/unit/libexpr/derived-path\.cc'' ''^tests/unit/libexpr/error_traces\.cc'' ''^tests/unit/libexpr/eval\.cc'' - ''^tests/unit/libexpr/flake/flakeref\.cc'' - ''^tests/unit/libexpr/flake/url-name\.cc'' ''^tests/unit/libexpr/json\.cc'' ''^tests/unit/libexpr/main\.cc'' ''^tests/unit/libexpr/primops\.cc'' @@ -445,9 +443,10 @@ ''^tests/unit/libexpr/value/context\.cc'' ''^tests/unit/libexpr/value/print\.cc'' ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' ''^tests/unit/libstore-support/tests/derived-path\.cc'' ''^tests/unit/libstore-support/tests/derived-path\.hh'' - ''^tests/unit/libstore-support/tests/libstore\.hh'' ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' @@ -500,57 +499,37 @@ ''^misc/bash/completion\.sh$'' ''^misc/fish/completion\.fish$'' ''^misc/zsh/completion\.zsh$'' - ''^scripts/check-hydra-status\.sh$'' ''^scripts/create-darwin-volume\.sh$'' ''^scripts/install-darwin-multi-user\.sh$'' ''^scripts/install-multi-user\.sh$'' ''^scripts/install-nix-from-closure\.sh$'' ''^scripts/install-systemd-multi-user\.sh$'' ''^src/nix/get-env\.sh$'' - ''^tests/functional/bash-profile\.sh$'' - ''^tests/functional/binary-cache-build-remote\.sh$'' - ''^tests/functional/binary-cache\.sh$'' - ''^tests/functional/brotli\.sh$'' - ''^tests/functional/build-delete\.sh$'' - ''^tests/functional/build-dry\.sh$'' ''^tests/functional/build\.sh$'' - ''^tests/functional/ca/build-cache\.sh$'' ''^tests/functional/ca/build-dry\.sh$'' ''^tests/functional/ca/build-with-garbage-path\.sh$'' - ''^tests/functional/ca/build\.sh$'' ''^tests/functional/ca/common\.sh$'' ''^tests/functional/ca/concurrent-builds\.sh$'' - ''^tests/functional/ca/derivation-json\.sh$'' - ''^tests/functional/ca/duplicate-realisation-in-closure\.sh$'' ''^tests/functional/ca/eval-store\.sh$'' ''^tests/functional/ca/gc\.sh$'' ''^tests/functional/ca/import-derivation\.sh$'' ''^tests/functional/ca/new-build-cmd\.sh$'' - ''^tests/functional/ca/nix-copy\.sh$'' - ''^tests/functional/ca/nix-run\.sh$'' ''^tests/functional/ca/nix-shell\.sh$'' ''^tests/functional/ca/post-hook\.sh$'' ''^tests/functional/ca/recursive\.sh$'' ''^tests/functional/ca/repl\.sh$'' ''^tests/functional/ca/selfref-gc\.sh$'' - ''^tests/functional/ca/signatures\.sh$'' - ''^tests/functional/ca/substitute\.sh$'' ''^tests/functional/ca/why-depends\.sh$'' - ''^tests/functional/case-hack\.sh$'' - ''^tests/functional/check-refs\.sh$'' - ''^tests/functional/check-reqs\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' ''^tests/functional/check\.sh$'' - ''^tests/functional/chroot-store\.sh$'' ''^tests/functional/common/vars-and-functions\.sh$'' ''^tests/functional/completions\.sh$'' - ''^tests/functional/compression-levels\.sh$'' ''^tests/functional/compute-levels\.sh$'' ''^tests/functional/config\.sh$'' ''^tests/functional/db-migration\.sh$'' ''^tests/functional/debugger\.sh$'' ''^tests/functional/dependencies\.builder0\.sh$'' ''^tests/functional/dependencies\.sh$'' - ''^tests/functional/derivation-json\.sh$'' ''^tests/functional/dump-db\.sh$'' ''^tests/functional/dyn-drv/build-built-drv\.sh$'' ''^tests/functional/dyn-drv/common\.sh$'' @@ -558,10 +537,8 @@ ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' - ''^tests/functional/dyn-drv/text-hashed-output\.sh$'' ''^tests/functional/eval-store\.sh$'' ''^tests/functional/eval\.sh$'' - ''^tests/functional/experimental-features\.sh$'' ''^tests/functional/export-graph\.sh$'' ''^tests/functional/export\.sh$'' ''^tests/functional/extra-sandbox-profile\.sh$'' @@ -571,56 +548,37 @@ ''^tests/functional/fetchGitSubmodules\.sh$'' ''^tests/functional/fetchGitVerification\.sh$'' ''^tests/functional/fetchMercurial\.sh$'' - ''^tests/functional/fetchPath\.sh$'' - ''^tests/functional/fetchTree-file\.sh$'' ''^tests/functional/fetchurl\.sh$'' - ''^tests/functional/filter-source\.sh$'' ''^tests/functional/fixed\.builder1\.sh$'' ''^tests/functional/fixed\.builder2\.sh$'' ''^tests/functional/fixed\.sh$'' - ''^tests/functional/flakes/absolute-attr-paths\.sh$'' ''^tests/functional/flakes/absolute-paths\.sh$'' - ''^tests/functional/flakes/build-paths\.sh$'' - ''^tests/functional/flakes/bundle\.sh$'' ''^tests/functional/flakes/check\.sh$'' - ''^tests/functional/flakes/circular\.sh$'' ''^tests/functional/flakes/common\.sh$'' ''^tests/functional/flakes/config\.sh$'' ''^tests/functional/flakes/develop\.sh$'' - ''^tests/functional/flakes/flake-in-submodule\.sh$'' ''^tests/functional/flakes/flakes\.sh$'' ''^tests/functional/flakes/follow-paths\.sh$'' - ''^tests/functional/flakes/init\.sh$'' - ''^tests/functional/flakes/inputs\.sh$'' - ''^tests/functional/flakes/mercurial\.sh$'' ''^tests/functional/flakes/prefetch\.sh$'' ''^tests/functional/flakes/run\.sh$'' - ''^tests/functional/flakes/search-root\.sh$'' ''^tests/functional/flakes/show\.sh$'' - ''^tests/functional/flakes/unlocked-override\.sh$'' ''^tests/functional/fmt\.sh$'' ''^tests/functional/fmt\.simple\.sh$'' - ''^tests/functional/function-trace\.sh$'' ''^tests/functional/gc-auto\.sh$'' ''^tests/functional/gc-concurrent\.builder\.sh$'' ''^tests/functional/gc-concurrent\.sh$'' ''^tests/functional/gc-concurrent2\.builder\.sh$'' ''^tests/functional/gc-non-blocking\.sh$'' - ''^tests/functional/gc-runtime\.sh$'' ''^tests/functional/gc\.sh$'' ''^tests/functional/git-hashing/common\.sh$'' ''^tests/functional/git-hashing/simple\.sh$'' ''^tests/functional/hash-convert\.sh$'' - ''^tests/functional/hash-path\.sh$'' ''^tests/functional/help\.sh$'' - ''^tests/functional/import-derivation\.sh$'' ''^tests/functional/impure-derivations\.sh$'' ''^tests/functional/impure-env\.sh$'' ''^tests/functional/impure-eval\.sh$'' ''^tests/functional/install-darwin\.sh$'' - ''^tests/functional/lang-test-infra\.sh$'' ''^tests/functional/lang\.sh$'' - ''^tests/functional/lang/framework\.sh$'' ''^tests/functional/legacy-ssh-store\.sh$'' ''^tests/functional/linux-sandbox\.sh$'' ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..e6bdc2eac --- /dev/null +++ b/meson.build @@ -0,0 +1,41 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') + +# Language Bindings +subproject('perl') + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') diff --git a/mk/common-test.sh b/mk/common-test.sh index c80abd381..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -4,7 +4,7 @@ # remove file extension. test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/package.nix b/package.nix index cf1654c6a..c3e565399 100644 --- a/package.nix +++ b/package.nix @@ -1,12 +1,10 @@ { lib -, fetchurl , stdenv , releaseTools , autoconf-archive , autoreconfHook , aws-sdk-cpp , boehmgc -, buildPackages , nlohmann_json , bison , boost @@ -15,12 +13,10 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 @@ -36,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -51,10 +48,8 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -93,11 +88,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal/external API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? forDevShell -, enableExternalAPIDocs ? forDevShell - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -120,6 +110,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some @@ -180,20 +172,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - ] ++ lib.optionals enableExternalAPIDocs [ - ./doc/external-api - ] ++ lib.optionals (enableInternalAPIDocs || enableExternalAPIDocs) [ - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -207,8 +190,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs || enableExternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -228,12 +213,10 @@ in { man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional (enableInternalAPIDocs || enableExternalAPIDocs) doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; buildInputs = lib.optionals doBuild [ - boost brotli bzip2 curl @@ -242,6 +225,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -260,40 +244,17 @@ in { ; propagatedBuildInputs = [ + boost nlohmann_json ] ++ lib.optional enableGC boehmgc; dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") - (lib.enableFeature enableExternalAPIDocs "external-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -317,9 +278,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html" - ++ lib.optional enableExternalAPIDocs "external-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -334,22 +297,10 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products - '' - + lib.optionalString enableExternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; # So the check output gets links for DLLs in the out output. @@ -385,8 +336,7 @@ in { separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..0e369a055 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,42 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../src/nix/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..34b344971 --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,117 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: + +let + inherit (pkgs) lib; + + root = ../.; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + +in +scope: { + inherit stdenv versionSuffix; + version = lib.fileContents ../.version + versionSuffix; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: stdenv.mkDerivation (lib.extends localSourceLayer f); +} diff --git a/build/hydra.nix b/packaging/hydra.nix similarity index 78% rename from build/hydra.nix rename to packaging/hydra.nix index 857b7f1f0..4dfaf9bbf 100644 --- a/build/hydra.nix +++ b/packaging/hydra.nix @@ -9,7 +9,6 @@ }: let inherit (inputs) nixpkgs nixpkgs-regression; - inherit (lib) fileset; installScriptFor = tarballs: nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { @@ -25,36 +24,57 @@ let lib.versionAtLeast client.version "2.4pre20211005") "-${client.version}-against-${daemon.version}"; - inherit fileset; - test-client = client; test-daemon = daemon; doBuild = false; }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-cmd" + "nix-ng" + ]; in { # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs [ "x86_64-linux" ] (system: self.packages.${system}."nix-${crossSystem}")); + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); buildNoGc = forAllSystems (system: self.packages.${system}.nix.override { enableGC = false; } ); - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); # Toggles some settings for better coverage. Windows needs these # library combinations, and Debian build Nix with GNU readline too. @@ -66,7 +86,7 @@ in ); # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of @@ -76,7 +96,7 @@ in binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: forAllCrossSystems (crossSystem: binaryTarball - self.packages.${system}."nix-${crossSystem}" + nixpkgsFor.${system}.cross.${crossSystem}.nix nixpkgsFor.${system}.cross.${crossSystem})); # The first half of the installation script. This is uploaded @@ -98,7 +118,7 @@ in installerScriptForGHA = installScriptFor [ # Native self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" # Cross self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" @@ -115,18 +135,10 @@ in }; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ../package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; # API docs for Nix's C bindings. - external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ../package.nix { - inherit fileset; - doBuild = false; - enableExternalAPIDocs = true; - }; + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; # System tests. tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/doc/external-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/external-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/external-api/README.md b/src/external-api-docs/README.md similarity index 100% rename from doc/external-api/README.md rename to src/external-api-docs/README.md diff --git a/doc/external-api/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in similarity index 91% rename from doc/external-api/doxygen.cfg.in rename to src/external-api-docs/doxygen.cfg.in index cd8b4989b..1be71d895 100644 --- a/doc/external-api/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,10 +38,10 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libutil-c \ - src/libexpr-c \ - src/libstore-c \ - doc/external-api/README.md + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md FILE_PATTERNS = nix_api_*.h *.md @@ -49,7 +51,6 @@ FILE_PATTERNS = nix_api_*.h *.md # RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. -INCLUDE_PATH = @RAPIDCHECK_HEADERS@ EXCLUDE_PATTERNS = *_internal.h GENERATE_TREEVIEW = YES OPTIMIZE_OUTPUT_FOR_C = YES diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..da136bbe1 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,63 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + enableParallelBuilding = true; + + strictDeps = true; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/doc/internal-api/.gitignore b/src/internal-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/internal-api-docs/.gitignore diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..f2077dcaf --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,58 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + enableParallelBuilding = true; + + strictDeps = true; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 543250da3..74d146c66 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -127,12 +127,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - lookupPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - lookupPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), evalSettings, getStore()) ; evalState->repair = repair; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index cd0f19257..62745b681 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,6 +1,7 @@ #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" @@ -10,9 +11,35 @@ #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -54,7 +81,7 @@ MixEvalArgs::MixEvalArgs() .description = R"( Add *path* to the Nix search path. The Nix search path is initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle + variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/types.md#type-path) enclosed in angle brackets (i.e., ``). For instance, passing diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 75cb19334..8d303ee7c 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -12,9 +12,21 @@ namespace nix { class Store; class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..5dc0eaf2b --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,19 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to revert to the Nix 2.3 behavior. + )"}; +}; + +}; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 9aa33a9d3..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain) +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..687b37aac --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,128 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc index 4b7d2441f..738bf6147 100644 --- a/src/libcmd/network-proxy.cc +++ b/src/libcmd/network-proxy.cc @@ -1,7 +1,6 @@ #include "network-proxy.hh" #include -#include #include "environment-variables.hh" @@ -13,7 +12,10 @@ static StringSet getAllVariables() { StringSet variables = lowercaseVariables; for (const auto & variable : lowercaseVariables) { - variables.insert(boost::to_upper_copy(variable)); + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); } return variables; } diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..ec3aa4660 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,110 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated + # to work with `strictDeps`. + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index eb4361e25..420cce1db 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -18,7 +18,7 @@ extern "C" { #include "finally.hh" #include "repl-interacter.hh" #include "file-system.hh" -#include "libcmd/repl.hh" +#include "repl.hh" namespace nix { diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 1e80d14e1..ce1c5af69 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -3,11 +3,12 @@ #include #include -#include "libcmd/repl-interacter.hh" +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" #include "eval-cache.hh" #include "eval-inline.hh" diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..2a2669b3e --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,91 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index b86d745db..13e0f5f3a 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -7,6 +7,7 @@ #include "eval.hh" #include "globals.hh" #include "util.hh" +#include "eval-settings.hh" #include "nix_api_expr.h" #include "nix_api_expr_internal.h" @@ -42,56 +43,56 @@ nix_err nix_libexpr_init(nix_c_context * context) } nix_err nix_expr_eval_from_string( - nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value) + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); - state->state.eval(parsedExpr, *(nix::Value *) value); - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value) +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.callFunction(*(nix::Value *) fn, *(nix::Value *) arg, *(nix::Value *) value, nix::noPos); - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value) +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.callFunction(*(nix::Value *) fn, nargs, (nix::Value * *)args, *(nix::Value *) value, nix::noPos); - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value) +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value) +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.forceValueDeep(*(nix::Value *) value); + state->state.forceValueDeep(value->value); } NIXC_CATCH_ERRS } @@ -106,7 +107,21 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c for (size_t i = 0; lookupPath_c[i] != nullptr; i++) lookupPath.push_back(lookupPath_c[i]); - return new EvalState{nix::EvalState(nix::LookupPath::parse(lookupPath), store->ptr)}; + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; } NIXC_CATCH_ERRS_NULL } @@ -181,6 +196,15 @@ nix_err nix_gc_decref(nix_c_context * context, const void *) void nix_gc_now() {} #endif +nix_err nix_value_incref(nix_c_context * context, nix_value *x) +{ + return nix_gc_incref(context, (const void *) x); +} +nix_err nix_value_decref(nix_c_context * context, nix_value *x) +{ + return nix_gc_decref(context, (const void *) x); +} + void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)) { #ifdef HAVE_BOEHMGC diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index cb6c00385..adf8b65b1 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -29,14 +29,23 @@ extern "C" { * @see nix_state_create */ typedef struct EvalState EvalState; // nix::EvalState -/** - * @brief Represents a value in the Nix language. + +/** @brief A Nix language value, or thunk that may evaluate to a value. + * + * Values are the primary objects manipulated in the Nix language. + * They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its + * ValueType if it was a thunk. After a value has been evaluated, its ValueType does not change. + * + * Evaluation in this context refers to the process of evaluating a single value object, also called "forcing" the + * value; see `nix_value_force`. + * + * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules. * - * Owned by the garbage collector. - * @struct Value * @see value_manip + * @see nix_value_incref, nix_value_decref */ -typedef void Value; // nix::Value +typedef struct nix_value nix_value; +[[deprecated("use nix_value instead")]] typedef nix_value Value; // Function prototypes /** @@ -65,7 +74,7 @@ nix_err nix_libexpr_init(nix_c_context * context); * @return NIX_OK if the evaluation was successful, an error code otherwise. */ nix_err nix_expr_eval_from_string( - nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value); + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value); /** * @brief Calls a Nix function with an argument. @@ -79,7 +88,7 @@ nix_err nix_expr_eval_from_string( * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk. * Note the different argument order. */ -nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value); +nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * fn, nix_value * arg, nix_value * value); /** * @brief Calls a Nix function with multiple arguments. @@ -98,7 +107,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V * @see NIX_VALUE_CALL For a macro that wraps this function for convenience. */ nix_err nix_value_call_multi( - nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value); + nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value); /** * @brief Calls a Nix function with multiple arguments. @@ -116,7 +125,7 @@ nix_err nix_value_call_multi( */ #define NIX_VALUE_CALL(context, state, value, fn, ...) \ do { \ - Value * args_array[] = {__VA_ARGS__}; \ + nix_value * args_array[] = {__VA_ARGS__}; \ size_t nargs = sizeof(args_array) / sizeof(args_array[0]); \ nix_value_call_multi(context, state, fn, nargs, args_array, value); \ } while (0) @@ -124,12 +133,10 @@ nix_err nix_value_call_multi( /** * @brief Forces the evaluation of a Nix value. * - * The Nix interpreter is lazy, and not-yet-evaluated Values can be + * The Nix interpreter is lazy, and not-yet-evaluated values can be * of type NIX_TYPE_THUNK instead of their actual value. * - * This function converts these Values into their final type. - * - * @note This function is mainly needed before calling @ref getters, but not for API calls that return a `Value`. + * This function mutates such a `nix_value`, so that, if successful, it has its final type. * * @param[out] context Optional, stores error information * @param[in] state The state of the evaluation. @@ -138,7 +145,7 @@ nix_err nix_value_call_multi( * @return NIX_OK if the force operation was successful, an error code * otherwise. */ -nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value); +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value); /** * @brief Forces the deep evaluation of a Nix value. @@ -154,7 +161,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * valu * @return NIX_OK if the deep force operation was successful, an error code * otherwise. */ -nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value); +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value); /** * @brief Create a new Nix language evaluator state. @@ -188,6 +195,11 @@ void nix_state_free(EvalState * state); * you're done with a value returned by the evaluator. * @{ */ + +// TODO: Deprecate nix_gc_incref in favor of the type-specific reference counting functions? +// e.g. nix_value_incref. +// It gives implementors more flexibility, and adds safety, so that generated +// bindings can be used without fighting the host type system (where applicable). /** * @brief Increment the garbage collector reference counter for the given object. * diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 7743849fd..d4ccffd29 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -2,11 +2,13 @@ #define NIX_API_EXPR_INTERNAL_H #include "eval.hh" +#include "eval-settings.hh" #include "attr-set.hh" #include "nix_api_value.h" struct EvalState { + nix::EvalSettings settings; nix::EvalState state; }; @@ -20,6 +22,11 @@ struct ListBuilder nix::ListBuilder builder; }; +struct nix_value +{ + nix::Value value; +}; + struct nix_string_return { std::string str; diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 978cf7f43..2f2f99617 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -21,49 +21,54 @@ #endif // Internal helper functions to check [in] and [out] `Value *` parameters -static const nix::Value & check_value_not_null(const Value * value) +static const nix::Value & check_value_not_null(const nix_value * value) { if (!value) { - throw std::runtime_error("Value is null"); + throw std::runtime_error("nix_value is null"); } return *((const nix::Value *) value); } -static nix::Value & check_value_not_null(Value * value) +static nix::Value & check_value_not_null(nix_value * value) { if (!value) { - throw std::runtime_error("Value is null"); + throw std::runtime_error("nix_value is null"); } - return *((nix::Value *) value); + return value->value; } -static const nix::Value & check_value_in(const Value * value) +static const nix::Value & check_value_in(const nix_value * value) { auto & v = check_value_not_null(value); if (!v.isValid()) { - throw std::runtime_error("Uninitialized Value"); + throw std::runtime_error("Uninitialized nix_value"); } return v; } -static nix::Value & check_value_in(Value * value) +static nix::Value & check_value_in(nix_value * value) { auto & v = check_value_not_null(value); if (!v.isValid()) { - throw std::runtime_error("Uninitialized Value"); + throw std::runtime_error("Uninitialized nix_value"); } return v; } -static nix::Value & check_value_out(Value * value) +static nix::Value & check_value_out(nix_value * value) { auto & v = check_value_not_null(value); if (v.isValid()) { - throw std::runtime_error("Value already initialized. Variables are immutable"); + throw std::runtime_error("nix_value already initialized. Variables are immutable"); } return v; } +static inline nix_value * as_nix_value_ptr(nix::Value * v) +{ + return reinterpret_cast(v); +} + /** * Helper function to convert calls from nix into C API. * @@ -87,7 +92,7 @@ static void nix_c_primop_wrapper( // or maybe something to make blackholes work better; we don't know). nix::Value vTmp; - f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &vTmp); + f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp); if (ctx.last_err_code != NIX_OK) { /* TODO: Throw different errors depending on the error code */ @@ -154,19 +159,19 @@ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp) NIXC_CATCH_ERRS } -Value * nix_alloc_value(nix_c_context * context, EvalState * state) +nix_value * nix_alloc_value(nix_c_context * context, EvalState * state) { if (context) context->last_err_code = NIX_OK; try { - Value * res = state->state.allocValue(); + nix_value * res = as_nix_value_ptr(state->state.allocValue()); nix_gc_incref(nullptr, res); return res; } NIXC_CATCH_ERRS_NULL } -ValueType nix_get_type(nix_c_context * context, const Value * value) +ValueType nix_get_type(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -202,7 +207,7 @@ ValueType nix_get_type(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL); } -const char * nix_get_typename(nix_c_context * context, const Value * value) +const char * nix_get_typename(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -214,7 +219,7 @@ const char * nix_get_typename(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_NULL } -bool nix_get_bool(nix_c_context * context, const Value * value) +bool nix_get_bool(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -226,7 +231,8 @@ bool nix_get_bool(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(false); } -nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data) +nix_err +nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -238,7 +244,7 @@ nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_str NIXC_CATCH_ERRS } -const char * nix_get_path_string(nix_c_context * context, const Value * value) +const char * nix_get_path_string(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -257,7 +263,7 @@ const char * nix_get_path_string(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_NULL } -unsigned int nix_get_list_size(nix_c_context * context, const Value * value) +unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -269,7 +275,7 @@ unsigned int nix_get_list_size(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0); } -unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value) +unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -281,7 +287,7 @@ unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0); } -double nix_get_float(nix_c_context * context, const Value * value) +double nix_get_float(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -293,7 +299,7 @@ double nix_get_float(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0.0); } -int64_t nix_get_int(nix_c_context * context, const Value * value) +int64_t nix_get_int(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -305,7 +311,7 @@ int64_t nix_get_int(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0); } -ExternalValue * nix_get_external(nix_c_context * context, Value * value) +ExternalValue * nix_get_external(nix_c_context * context, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -317,7 +323,7 @@ ExternalValue * nix_get_external(nix_c_context * context, Value * value) NIXC_CATCH_ERRS_NULL; } -Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix) +nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix) { if (context) context->last_err_code = NIX_OK; @@ -328,12 +334,12 @@ Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalSta nix_gc_incref(nullptr, p); if (p != nullptr) state->state.forceValue(*p, nix::noPos); - return (Value *) p; + return as_nix_value_ptr(p); } NIXC_CATCH_ERRS_NULL } -Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name) +nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) { if (context) context->last_err_code = NIX_OK; @@ -345,7 +351,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt if (attr) { nix_gc_incref(nullptr, attr->value); state->state.forceValue(*attr->value, nix::noPos); - return attr->value; + return as_nix_value_ptr(attr->value); } nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); return nullptr; @@ -353,7 +359,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt NIXC_CATCH_ERRS_NULL } -bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name) +bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) { if (context) context->last_err_code = NIX_OK; @@ -369,8 +375,8 @@ bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState NIXC_CATCH_ERRS_RES(false); } -Value * -nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name) +nix_value * nix_get_attr_byidx( + nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name) { if (context) context->last_err_code = NIX_OK; @@ -380,12 +386,13 @@ nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * sta *name = ((const std::string &) (state->state.symbols[a.name])).c_str(); nix_gc_incref(nullptr, a.value); state->state.forceValue(*a.value, nix::noPos); - return a.value; + return as_nix_value_ptr(a.value); } NIXC_CATCH_ERRS_NULL } -const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i) +const char * +nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i) { if (context) context->last_err_code = NIX_OK; @@ -397,7 +404,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * valu NIXC_CATCH_ERRS_NULL } -nix_err nix_init_bool(nix_c_context * context, Value * value, bool b) +nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b) { if (context) context->last_err_code = NIX_OK; @@ -409,7 +416,7 @@ nix_err nix_init_bool(nix_c_context * context, Value * value, bool b) } // todo string context -nix_err nix_init_string(nix_c_context * context, Value * value, const char * str) +nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str) { if (context) context->last_err_code = NIX_OK; @@ -420,7 +427,7 @@ nix_err nix_init_string(nix_c_context * context, Value * value, const char * str NIXC_CATCH_ERRS } -nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str) +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str) { if (context) context->last_err_code = NIX_OK; @@ -431,7 +438,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * val NIXC_CATCH_ERRS } -nix_err nix_init_float(nix_c_context * context, Value * value, double d) +nix_err nix_init_float(nix_c_context * context, nix_value * value, double d) { if (context) context->last_err_code = NIX_OK; @@ -442,7 +449,7 @@ nix_err nix_init_float(nix_c_context * context, Value * value, double d) NIXC_CATCH_ERRS } -nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i) +nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i) { if (context) context->last_err_code = NIX_OK; @@ -453,7 +460,7 @@ nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i) NIXC_CATCH_ERRS } -nix_err nix_init_null(nix_c_context * context, Value * value) +nix_err nix_init_null(nix_c_context * context, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -464,7 +471,7 @@ nix_err nix_init_null(nix_c_context * context, Value * value) NIXC_CATCH_ERRS } -nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg) +nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg) { if (context) context->last_err_code = NIX_OK; @@ -477,7 +484,7 @@ nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value NIXC_CATCH_ERRS } -nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val) +nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val) { if (context) context->last_err_code = NIX_OK; @@ -504,7 +511,8 @@ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, NIXC_CATCH_ERRS_NULL } -nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value) +nix_err +nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -524,7 +532,7 @@ void nix_list_builder_free(ListBuilder * list_builder) #endif } -nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value) +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -535,7 +543,7 @@ nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value NIXC_CATCH_ERRS } -nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p) +nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * p) { if (context) context->last_err_code = NIX_OK; @@ -546,7 +554,7 @@ nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p) NIXC_CATCH_ERRS } -nix_err nix_copy_value(nix_c_context * context, Value * value, const Value * source) +nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source) { if (context) context->last_err_code = NIX_OK; @@ -558,7 +566,7 @@ nix_err nix_copy_value(nix_c_context * context, Value * value, const Value * sou NIXC_CATCH_ERRS } -nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b) +nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b) { if (context) context->last_err_code = NIX_OK; @@ -584,7 +592,7 @@ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * NIXC_CATCH_ERRS_NULL } -nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, Value * value) +nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -605,7 +613,7 @@ void nix_bindings_builder_free(BindingsBuilder * bb) #endif } -nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD) +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD) { if (context) context->last_err_code = NIX_OK; diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 244860707..044f68c9e 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -35,8 +35,11 @@ typedef enum { } ValueType; // forward declarations -typedef void Value; +typedef struct nix_value nix_value; typedef struct EvalState EvalState; + +[[deprecated("use nix_value instead")]] typedef nix_value Value; + // type defs /** @brief Stores an under-construction set of bindings * @ingroup value_manip @@ -90,7 +93,8 @@ typedef struct nix_realised_string nix_realised_string; * @param[out] ret return value * @see nix_alloc_primop, nix_init_primop */ -typedef void (*PrimOpFun)(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret); +typedef void (*PrimOpFun)( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret); /** @brief Allocate a PrimOp * @@ -142,10 +146,29 @@ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp); * @return value, or null in case of errors * */ -Value * nix_alloc_value(nix_c_context * context, EvalState * state); +nix_value * nix_alloc_value(nix_c_context * context, EvalState * state); + +/** + * @brief Increment the garbage collector reference counter for the given `nix_value`. + * + * The Nix language evaluator C API keeps track of alive objects by reference counting. + * When you're done with a refcounted pointer, call nix_value_decref(). + * + * @param[out] context Optional, stores error information + * @param[in] value The object to keep alive + */ +nix_err nix_value_incref(nix_c_context * context, nix_value * value); + +/** + * @brief Decrement the garbage collector reference counter for the given object + * + * @param[out] context Optional, stores error information + * @param[in] value The object to stop referencing + */ +nix_err nix_value_decref(nix_c_context * context, nix_value * value); /** @addtogroup value_manip Manipulating values - * @brief Functions to inspect and change Nix language values, represented by Value. + * @brief Functions to inspect and change Nix language values, represented by nix_value. * @{ */ /** @anchor getters @@ -157,7 +180,7 @@ Value * nix_alloc_value(nix_c_context * context, EvalState * state); * @param[in] value Nix value to inspect * @return type of nix value */ -ValueType nix_get_type(nix_c_context * context, const Value * value); +ValueType nix_get_type(nix_c_context * context, const nix_value * value); /** @brief Get type name of value as defined in the evaluator * @param[out] context Optional, stores error information @@ -165,14 +188,14 @@ ValueType nix_get_type(nix_c_context * context, const Value * value); * @return type name, owned string * @todo way to free the result */ -const char * nix_get_typename(nix_c_context * context, const Value * value); +const char * nix_get_typename(nix_c_context * context, const nix_value * value); /** @brief Get boolean value * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return true or false, error info via context */ -bool nix_get_bool(nix_c_context * context, const Value * value); +bool nix_get_bool(nix_c_context * context, const nix_value * value); /** @brief Get the raw string * @@ -186,7 +209,7 @@ bool nix_get_bool(nix_c_context * context, const Value * value); * @return error code, NIX_OK on success. */ nix_err -nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data); +nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data); /** @brief Get path as string * @param[out] context Optional, stores error information @@ -194,42 +217,42 @@ nix_get_string(nix_c_context * context, const Value * value, nix_get_string_call * @return string * @return NULL in case of error. */ -const char * nix_get_path_string(nix_c_context * context, const Value * value); +const char * nix_get_path_string(nix_c_context * context, const nix_value * value); /** @brief Get the length of a list * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return length of list, error info via context */ -unsigned int nix_get_list_size(nix_c_context * context, const Value * value); +unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value); /** @brief Get the element count of an attrset * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return attrset element count, error info via context */ -unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value); +unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value); /** @brief Get float value in 64 bits * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return float contents, error info via context */ -double nix_get_float(nix_c_context * context, const Value * value); +double nix_get_float(nix_c_context * context, const nix_value * value); /** @brief Get int value * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return int contents, error info via context */ -int64_t nix_get_int(nix_c_context * context, const Value * value); +int64_t nix_get_int(nix_c_context * context, const nix_value * value); /** @brief Get external reference * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return reference to external, NULL in case of error */ -ExternalValue * nix_get_external(nix_c_context * context, Value *); +ExternalValue * nix_get_external(nix_c_context * context, nix_value *); /** @brief Get the ix'th element of a list * @@ -240,7 +263,7 @@ ExternalValue * nix_get_external(nix_c_context * context, Value *); * @param[in] ix list element to get * @return value, NULL in case of errors */ -Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix); +nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); /** @brief Get an attr by name * @@ -251,7 +274,7 @@ Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalSta * @param[in] name attribute name * @return value, NULL in case of errors */ -Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name); +nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Check if an attribute name exists on a value * @param[out] context Optional, stores error information @@ -260,7 +283,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt * @param[in] name attribute name * @return value, error info via context */ -bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name); +bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Get an attribute by index in the sorted bindings * @@ -274,8 +297,8 @@ bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * @param[out] name will store a pointer to the attribute name * @return value, NULL in case of errors */ -Value * -nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name); +nix_value * nix_get_attr_byidx( + nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name); /** @brief Get an attribute name by index in the sorted bindings * @@ -288,7 +311,8 @@ nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * sta * @param[in] i attribute index * @return name, NULL in case of errors */ -const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i); +const char * +nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i); /**@}*/ /** @name Initializers @@ -305,7 +329,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * valu * @param[in] b the boolean value * @return error code, NIX_OK on success. */ -nix_err nix_init_bool(nix_c_context * context, Value * value, bool b); +nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b); /** @brief Set a string * @param[out] context Optional, stores error information @@ -313,7 +337,7 @@ nix_err nix_init_bool(nix_c_context * context, Value * value, bool b); * @param[in] str the string, copied * @return error code, NIX_OK on success. */ -nix_err nix_init_string(nix_c_context * context, Value * value, const char * str); +nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str); /** @brief Set a path * @param[out] context Optional, stores error information @@ -321,7 +345,7 @@ nix_err nix_init_string(nix_c_context * context, Value * value, const char * str * @param[in] str the path string, copied * @return error code, NIX_OK on success. */ -nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str); +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str); /** @brief Set a float * @param[out] context Optional, stores error information @@ -329,7 +353,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * val * @param[in] d the float, 64-bits * @return error code, NIX_OK on success. */ -nix_err nix_init_float(nix_c_context * context, Value * value, double d); +nix_err nix_init_float(nix_c_context * context, nix_value * value, double d); /** @brief Set an int * @param[out] context Optional, stores error information @@ -338,13 +362,13 @@ nix_err nix_init_float(nix_c_context * context, Value * value, double d); * @return error code, NIX_OK on success. */ -nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i); +nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i); /** @brief Set null * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @return error code, NIX_OK on success. */ -nix_err nix_init_null(nix_c_context * context, Value * value); +nix_err nix_init_null(nix_c_context * context, nix_value * value); /** @brief Set the value to a thunk that will perform a function application when needed. * @@ -360,7 +384,7 @@ nix_err nix_init_null(nix_c_context * context, Value * value); * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value. * Note the different argument order. */ -nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg); +nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg); /** @brief Set an external value * @param[out] context Optional, stores error information @@ -368,7 +392,7 @@ nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * @param[in] val the external value to set. Will be GC-referenced by the value. * @return error code, NIX_OK on success. */ -nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val); +nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val); /** @brief Create a list from a list builder * @param[out] context Optional, stores error information @@ -376,7 +400,7 @@ nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * @param[out] value Nix value to modify * @return error code, NIX_OK on success. */ -nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value); +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value); /** @brief Create a list builder * @param[out] context Optional, stores error information @@ -393,7 +417,8 @@ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, * @param[in] value value to insert * @return error code, NIX_OK on success. */ -nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value); +nix_err +nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value); /** @brief Free a list builder * @@ -408,7 +433,7 @@ void nix_list_builder_free(ListBuilder * list_builder); * @param[in] b bindings builder to use. Make sure to unref this afterwards. * @return error code, NIX_OK on success. */ -nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b); +nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b); /** @brief Set primop * @param[out] context Optional, stores error information @@ -417,14 +442,14 @@ nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * * @see nix_alloc_primop * @return error code, NIX_OK on success. */ -nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * op); +nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op); /** @brief Copy from another value * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] source value to copy from * @return error code, NIX_OK on success. */ -nix_err nix_copy_value(nix_c_context * context, Value * value, const Value * source); +nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source); /**@}*/ /** @brief Create a bindings builder @@ -444,7 +469,7 @@ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * * @return error code, NIX_OK on success. */ nix_err -nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, Value * value); +nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, nix_value * value); /** @brief Free a bindings builder * @@ -471,7 +496,7 @@ void nix_bindings_builder_free(BindingsBuilder * builder); You should set this to false when building for your application's purpose. * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free */ -nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD); +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD); /** @brief Start of the string * @param[in] realised_string diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix new file mode 100644 index 000000000..0b895437b --- /dev/null +++ b/src/libexpr-c/package.nix @@ -0,0 +1,78 @@ +{ lib +, stdenv +, mkMesonDerivation + +, meson +, ninja +, pkg-config + +, nix-store-c +, nix-expr + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-expr-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-store-c + nix-expr + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libexpr/.version b/src/libexpr/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr/build-utils-meson b/src/libexpr/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/call-flake.nix similarity index 100% rename from src/libexpr/flake/call-flake.nix rename to src/libexpr/call-flake.nix diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc new file mode 100644 index 000000000..baf9df332 --- /dev/null +++ b/src/libexpr/eval-gc.cc @@ -0,0 +1,226 @@ +#include "error.hh" +#include "environment-variables.hh" +#include "serialise.hh" +#include "eval-gc.hh" + +#if HAVE_BOEHMGC + +# include +# if __FreeBSD__ +# include +# endif + +# include +# include +# include + +# include +# include +# include + +#endif + +namespace nix { + +#if HAVE_BOEHMGC +/* Called when the Boehm GC runs out of memory. */ +static void * oomHandler(size_t requested) +{ + /* Convert this to a proper C++ exception. */ + throw std::bad_alloc(); +} + +class BoehmGCStackAllocator : public StackAllocator +{ + boost::coroutines2::protected_fixedsize_stack stack{ + // We allocate 8 MB, the default max stack size on NixOS. + // A smaller stack might be quicker to allocate but reduces the stack + // depth available for source filter expressions etc. + std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024))}; + + // This is specific to boost::coroutines2::protected_fixedsize_stack. + // The stack protection page is included in sctx.size, so we have to + // subtract one page size from the stack size. + std::size_t pfss_usable_stack_size(boost::context::stack_context & sctx) + { + return sctx.size - boost::context::stack_traits::page_size(); + } + +public: + boost::context::stack_context allocate() override + { + auto sctx = stack.allocate(); + + // Stacks generally start at a high address and grow to lower addresses. + // Architectures that do the opposite are rare; in fact so rare that + // boost_routine does not implement it. + // So we subtract the stack size. + GC_add_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); + return sctx; + } + + void deallocate(boost::context::stack_context sctx) override + { + GC_remove_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); + stack.deallocate(sctx); + } +}; + +static BoehmGCStackAllocator boehmGCStackAllocator; + +/** + * When a thread goes into a coroutine, we lose its original sp until + * control flow returns to the thread. + * While in the coroutine, the sp points outside the thread stack, + * so we can detect this and push the entire thread stack instead, + * as an approximation. + * The coroutine's stack is covered by `BoehmGCStackAllocator`. + * This is not an optimal solution, because the garbage is scanned when a + * coroutine is active, for both the coroutine and the original thread stack. + * However, the implementation is quite lean, and usually we don't have active + * coroutines during evaluation, so this is acceptable. + */ +void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) +{ + void *& sp = *sp_ptr; + auto pthread_id = reinterpret_cast(_pthread_id); + pthread_attr_t pattr; + size_t osStackSize; + void * osStackLow; + void * osStackBase; + +# ifdef __APPLE__ + osStackSize = pthread_get_stacksize_np(pthread_id); + osStackLow = pthread_get_stackaddr_np(pthread_id); +# else + if (pthread_attr_init(&pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); + } +# ifdef HAVE_PTHREAD_GETATTR_NP + if (pthread_getattr_np(pthread_id, &pattr)) { + throw Error("fixupBoehmStackPointer: pthread_getattr_np failed"); + } +# elif HAVE_PTHREAD_ATTR_GET_NP + if (!pthread_attr_init(&pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); + } + if (!pthread_attr_get_np(pthread_id, &pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_get_np failed"); + } +# else +# error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`" +# endif + if (pthread_attr_getstack(&pattr, &osStackLow, &osStackSize)) { + throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); + } + if (pthread_attr_destroy(&pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); + } +# endif + osStackBase = (char *) osStackLow + osStackSize; + // NOTE: We assume the stack grows down, as it does on all architectures we support. + // Architectures that grow the stack up are rare. + if (sp >= osStackBase || sp < osStackLow) { // lo is outside the os stack + sp = osStackBase; + } +} + +/* Disable GC while this object lives. Used by CoroutineContext. + * + * Boehm keeps a count of GC_disable() and GC_enable() calls, + * and only enables GC when the count matches. + */ +class BoehmDisableGC +{ +public: + BoehmDisableGC() + { + GC_disable(); + }; + ~BoehmDisableGC() + { + GC_enable(); + }; +}; + +static inline void initGCReal() +{ + /* Initialise the Boehm garbage collector. */ + + /* Don't look for interior pointers. This reduces the odds of + misdetection a bit. */ + GC_set_all_interior_pointers(0); + + /* We don't have any roots in data segments, so don't scan from + there. */ + GC_set_no_dls(1); + + GC_INIT(); + + GC_set_oom_fn(oomHandler); + + StackAllocator::defaultAllocator = &boehmGCStackAllocator; + +// TODO: Remove __APPLE__ condition. +// Comment suggests an implementation that works on darwin and windows +// https://github.com/ivmai/bdwgc/issues/362#issuecomment-1936672196 +# if GC_VERSION_MAJOR >= 8 && GC_VERSION_MINOR >= 2 && GC_VERSION_MICRO >= 4 && !defined(__APPLE__) + GC_set_sp_corrector(&fixupBoehmStackPointer); + + if (!GC_get_sp_corrector()) { + printTalkative("BoehmGC on this platform does not support sp_corrector; will disable GC inside coroutines"); + /* Used to disable GC when entering coroutines on macOS */ + create_coro_gc_hook = []() -> std::shared_ptr { return std::make_shared(); }; + } +# else +# warning \ + "BoehmGC version does not support GC while coroutine exists. GC will be disabled inside coroutines. Consider updating bdw-gc to 8.2.4 or later." +# endif + + /* Set the initial heap size to something fairly big (25% of + physical RAM, up to a maximum of 384 MiB) so that in most cases + we don't need to garbage collect at all. (Collection has a + fairly significant overhead.) The heap size can be overridden + through libgc's GC_INITIAL_HEAP_SIZE environment variable. We + should probably also provide a nix.conf setting for this. Note + that GC_expand_hp() causes a lot of virtual, but not physical + (resident) memory to be allocated. This might be a problem on + systems that don't overcommit. */ + if (!getEnv("GC_INITIAL_HEAP_SIZE")) { + size_t size = 32 * 1024 * 1024; +# if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) + size_t maxSize = 384 * 1024 * 1024; + long pageSize = sysconf(_SC_PAGESIZE); + long pages = sysconf(_SC_PHYS_PAGES); + if (pageSize != -1) + size = (pageSize * pages) / 4; // 25% of RAM + if (size > maxSize) + size = maxSize; +# endif + debug("setting initial heap size to %1% bytes", size); + GC_expand_hp(size); + } +} + +#endif + +static bool gcInitialised = false; + +void initGC() +{ + if (gcInitialised) + return; + +#if HAVE_BOEHMGC + initGCReal(); +#endif + + gcInitialised = true; +} + +void assertGCInitialized() +{ + assert(gcInitialised); +} + +} diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh new file mode 100644 index 000000000..cd4ea914d --- /dev/null +++ b/src/libexpr/eval-gc.hh @@ -0,0 +1,16 @@ +#pragma once +///@file + +namespace nix { + +/** + * Initialise the Boehm GC, if applicable. + */ +void initGC(); + +/** + * Make sure `initGC` has already been called. + */ +void assertGCInitialized(); + +} diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 85b1677ea..6b7b52cef 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,4 +1,5 @@ #include "users.hh" +#include "config-global.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" @@ -44,7 +45,9 @@ static Strings parseNixPath(const std::string & s) return res; } -EvalSettings::EvalSettings() +EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks) + : readOnlyMode{readOnlyMode} + , lookupPathHooks{lookupPathHooks} { auto var = getEnv("NIX_PATH"); if (var) nixPath = parseNixPath(*var); @@ -54,7 +57,7 @@ EvalSettings::EvalSettings() builtinsAbortOnWarn = true; } -Strings EvalSettings::getDefaultNixPath() +Strings EvalSettings::getDefaultNixPath() const { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { @@ -67,7 +70,7 @@ Strings EvalSettings::getDefaultNixPath() } }; - if (!evalSettings.restrictEval && !evalSettings.pureEval) { + if (!restrictEval && !pureEval) { add(getNixDefExpr() + "/channels"); add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); add(rootChannelsDir()); @@ -93,16 +96,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url) return std::string(url); } -const std::string & EvalSettings::getCurrentSystem() +const std::string & EvalSettings::getCurrentSystem() const { const auto & evalSystem = currentSystem.get(); return evalSystem != "" ? evalSystem : settings.thisSystem.get(); } -EvalSettings evalSettings; - -static GlobalConfig::Register rEvalSettings(&evalSettings); - Path getNixDefExpr() { return settings.useXDGBaseDirectories diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index d01de980f..191dde21a 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -5,16 +5,51 @@ namespace nix { +class Store; + struct EvalSettings : Config { - EvalSettings(); + /** + * Function used to interpet look path entries of a given scheme. + * + * The argument is the non-scheme part of the lookup path entry (see + * `LookupPathHooks` below). + * + * The return value is (a) whether the entry was valid, and, if so, + * what does it map to. + * + * @todo Return (`std::optional` of) `SourceAccssor` or something + * more structured instead of mere `std::string`? + */ + using LookupPathHook = std::optional(ref store, std::string_view); - static Strings getDefaultNixPath(); + /** + * Map from "scheme" to a `LookupPathHook`. + * + * Given a lookup path value (i.e. either the whole thing, or after + * the `=`) in the form of: + * + * ``` + * : + * ``` + * + * if `` is a key in this map, then `` is + * passed to the hook that is the value in this map. + */ + using LookupPathHooks = std::map>; + + EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); + + bool & readOnlyMode; + + Strings getDefaultNixPath() const; static bool isPseudoUrl(std::string_view s); static std::string resolvePseudoUrl(std::string_view url); + LookupPathHooks lookupPathHooks; + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"( Enable built-in functions that allow executing native code. @@ -39,7 +74,7 @@ struct EvalSettings : Config R"( List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution. This setting determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtin-constants.md#builtins-findFile). + [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). The default value is @@ -60,7 +95,7 @@ struct EvalSettings : Config this, "", "eval-system", R"( This option defines - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) in the Nix language if it is set as a non-empty string. Otherwise, if it is defined as the empty string (the default), the value of the [`system` ](#conf-system) @@ -74,14 +109,14 @@ struct EvalSettings : Config * Implements the `eval-system` vs `system` defaulting logic * described for `eval-system`. */ - const std::string & getCurrentSystem(); + const std::string & getCurrentSystem() const; Setting restrictEval{ this, false, "restrict-eval", R"( If set to `true`, the Nix evaluator will not allow access to any files outside of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath), + [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath), or to URIs outside of [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). )"}; @@ -92,10 +127,10 @@ struct EvalSettings : Config - Restrict file system and network access to files specified by cryptographic hash - Disable impure constants: - - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) - - [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) - - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) - - [`builtins.storePath`](@docroot@/language/builtin-constants.md#builtins-storePath) + - [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) + - [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime) + - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) + - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath) )" }; @@ -193,8 +228,6 @@ struct EvalSettings : Config )"}; }; -extern EvalSettings evalSettings; - /** * Conventionally part of the default nix path in impure mode. */ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f441977a5..2a0862123 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3,13 +3,12 @@ #include "hash.hh" #include "primops.hh" #include "print-options.hh" -#include "shared.hh" +#include "exit.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" #include "derivations.hh" #include "downstream-placeholder.hh" -#include "globals.hh" #include "eval-inline.hh" #include "filetransfer.hh" #include "function-trace.hh" @@ -22,7 +21,6 @@ #include "url.hh" #include "fetch-to-store.hh" #include "tarball.hh" -#include "flake/flakeref.hh" #include "parser-tab.hh" #include @@ -40,22 +38,16 @@ #include #ifndef _WIN32 // TODO use portable implementation -# include +# include #endif #if HAVE_BOEHMGC -#define GC_INCLUDE_NEW +# define GC_INCLUDE_NEW -#include - -#include -#include -#include - -#include -#include -#include +# include +# include +# include #endif @@ -208,97 +200,6 @@ bool Value::isTrivial() const } -#if HAVE_BOEHMGC -/* Called when the Boehm GC runs out of memory. */ -static void * oomHandler(size_t requested) -{ - /* Convert this to a proper C++ exception. */ - throw std::bad_alloc(); -} - -class BoehmGCStackAllocator : public StackAllocator { - boost::coroutines2::protected_fixedsize_stack stack { - // We allocate 8 MB, the default max stack size on NixOS. - // A smaller stack might be quicker to allocate but reduces the stack - // depth available for source filter expressions etc. - std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) - }; - - // This is specific to boost::coroutines2::protected_fixedsize_stack. - // The stack protection page is included in sctx.size, so we have to - // subtract one page size from the stack size. - std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) { - return sctx.size - boost::context::stack_traits::page_size(); - } - - public: - boost::context::stack_context allocate() override { - auto sctx = stack.allocate(); - - // Stacks generally start at a high address and grow to lower addresses. - // Architectures that do the opposite are rare; in fact so rare that - // boost_routine does not implement it. - // So we subtract the stack size. - GC_add_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - return sctx; - } - - void deallocate(boost::context::stack_context sctx) override { - GC_remove_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - stack.deallocate(sctx); - } - -}; - -static BoehmGCStackAllocator boehmGCStackAllocator; - -/** - * When a thread goes into a coroutine, we lose its original sp until - * control flow returns to the thread. - * While in the coroutine, the sp points outside the thread stack, - * so we can detect this and push the entire thread stack instead, - * as an approximation. - * The coroutine's stack is covered by `BoehmGCStackAllocator`. - * This is not an optimal solution, because the garbage is scanned when a - * coroutine is active, for both the coroutine and the original thread stack. - * However, the implementation is quite lean, and usually we don't have active - * coroutines during evaluation, so this is acceptable. - */ -void fixupBoehmStackPointer(void ** sp_ptr, void * pthread_id) { - void *& sp = *sp_ptr; - pthread_attr_t pattr; - size_t osStackSize; - void * osStackLow; - void * osStackBase; - - #ifdef __APPLE__ - osStackSize = pthread_get_stacksize_np((pthread_t)pthread_id); - osStackLow = pthread_get_stackaddr_np((pthread_t)pthread_id); - #else - if (pthread_attr_init(&pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); - } - if (pthread_getattr_np((pthread_t)pthread_id, &pattr)) { - throw Error("fixupBoehmStackPointer: pthread_getattr_np failed"); - } - if (pthread_attr_getstack(&pattr, &osStackLow, &osStackSize)) { - throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); - } - if (pthread_attr_destroy(&pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); - } - #endif - osStackBase = (char *)osStackLow + osStackSize; - // NOTE: We assume the stack grows down, as it does on all architectures we support. - // Architectures that grow the stack up are rare. - if (sp >= osStackBase || sp < osStackLow) { // lo is outside the os stack - sp = osStackBase; - } -} - -#endif - - static Symbol getName(const AttrName & name, EvalState & state, Env & env) { if (name.symbol) { @@ -311,99 +212,15 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } } -#if HAVE_BOEHMGC -/* Disable GC while this object lives. Used by CoroutineContext. - * - * Boehm keeps a count of GC_disable() and GC_enable() calls, - * and only enables GC when the count matches. - */ -class BoehmDisableGC { -public: - BoehmDisableGC() { - GC_disable(); - }; - ~BoehmDisableGC() { - GC_enable(); - }; -}; -#endif - -static bool gcInitialised = false; - -void initGC() -{ - if (gcInitialised) return; - -#if HAVE_BOEHMGC - /* Initialise the Boehm garbage collector. */ - - /* Don't look for interior pointers. This reduces the odds of - misdetection a bit. */ - GC_set_all_interior_pointers(0); - - /* We don't have any roots in data segments, so don't scan from - there. */ - GC_set_no_dls(1); - - GC_INIT(); - - GC_set_oom_fn(oomHandler); - - StackAllocator::defaultAllocator = &boehmGCStackAllocator; - - // TODO: Remove __APPLE__ condition. - // Comment suggests an implementation that works on darwin and windows - // https://github.com/ivmai/bdwgc/issues/362#issuecomment-1936672196 - #if GC_VERSION_MAJOR >= 8 && GC_VERSION_MINOR >= 2 && GC_VERSION_MICRO >= 4 && !defined(__APPLE__) - GC_set_sp_corrector(&fixupBoehmStackPointer); - - if (!GC_get_sp_corrector()) { - printTalkative("BoehmGC on this platform does not support sp_corrector; will disable GC inside coroutines"); - /* Used to disable GC when entering coroutines on macOS */ - create_coro_gc_hook = []() -> std::shared_ptr { - return std::make_shared(); - }; - } - #else - #warning "BoehmGC version does not support GC while coroutine exists. GC will be disabled inside coroutines. Consider updating bdw-gc to 8.2.4 or later." - #endif - - - /* Set the initial heap size to something fairly big (25% of - physical RAM, up to a maximum of 384 MiB) so that in most cases - we don't need to garbage collect at all. (Collection has a - fairly significant overhead.) The heap size can be overridden - through libgc's GC_INITIAL_HEAP_SIZE environment variable. We - should probably also provide a nix.conf setting for this. Note - that GC_expand_hp() causes a lot of virtual, but not physical - (resident) memory to be allocated. This might be a problem on - systems that don't overcommit. */ - if (!getEnv("GC_INITIAL_HEAP_SIZE")) { - size_t size = 32 * 1024 * 1024; -#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) - size_t maxSize = 384 * 1024 * 1024; - long pageSize = sysconf(_SC_PAGESIZE); - long pages = sysconf(_SC_PHYS_PAGES); - if (pageSize != -1) - size = (pageSize * pages) / 4; // 25% of RAM - if (size > maxSize) size = maxSize; -#endif - debug("setting initial heap size to %1% bytes", size); - GC_expand_hp(size); - } - -#endif - - gcInitialised = true; -} - static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( const LookupPath & _lookupPath, ref store, + const EvalSettings & settings, std::shared_ptr buildStore) - : sWith(symbols.create("")) + : settings{settings} + , sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) , sType(symbols.create("type")) @@ -423,6 +240,12 @@ EvalState::EvalState( , sRight(symbols.create("right")) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sAllowedReferences(symbols.create("allowedReferences")) + , sAllowedRequisites(symbols.create("allowedRequisites")) + , sDisallowedReferences(symbols.create("disallowedReferences")) + , sDisallowedRequisites(symbols.create("disallowedRequisites")) + , sMaxSize(symbols.create("maxSize")) + , sMaxClosureSize(symbols.create("maxClosureSize")) , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) , sContentAddressed(symbols.create("__contentAddressed")) @@ -453,10 +276,10 @@ EvalState::EvalState( , repair(NoRepair) , emptyBindings(0) , rootFS( - evalSettings.restrictEval || evalSettings.pureEval + settings.restrictEval || settings.pureEval ? ref(AllowListSourceAccessor::create(getFSSourceAccessor(), {}, - [](const CanonPath & path) -> RestrictedPathError { - auto modeInformation = evalSettings.pureEval + [&settings](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = settings.pureEval ? "in pure evaluation mode (use '--impure' to override)" : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); @@ -470,7 +293,7 @@ EvalState::EvalState( )} , callFlakeInternal{internalFS->addFile( CanonPath("call-flake.nix"), - #include "flake/call-flake.nix.gen.hh" + #include "call-flake.nix.gen.hh" )} , store(store) , buildStore(buildStore ? buildStore : store) @@ -493,7 +316,7 @@ EvalState::EvalState( countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; - assert(gcInitialised); + assertGCInitialized(); static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); @@ -507,10 +330,10 @@ EvalState::EvalState( vStringUnknown.mkString("unknown"); /* Initialise the Nix expression search path. */ - if (!evalSettings.pureEval) { + if (!settings.pureEval) { for (auto & i : _lookupPath.elements) lookupPath.elements.emplace_back(LookupPath::Elem {i}); - for (auto & i : evalSettings.nixPath.get()) + for (auto & i : settings.nixPath.get()) lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); } @@ -588,9 +411,9 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) void EvalState::checkURI(const std::string & uri) { - if (!evalSettings.restrictEval) return; + if (!settings.restrictEval) return; - if (isAllowedURI(uri, evalSettings.allowedUris.get())) return; + if (isAllowedURI(uri, settings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ @@ -635,7 +458,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) constantInfos.push_back({name2, info}); - if (!(evalSettings.pureEval && info.impureOnly)) { + if (!(settings.pureEval && info.impureOnly)) { /* Check the type, if possible. We might know the type of a thunk in advance, so be allowed @@ -1590,11 +1413,11 @@ public: void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { - if (callDepth > evalSettings.maxCallDepth) + if (callDepth > settings.maxCallDepth) error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); CallDepth _level(callDepth); - auto trace = evalSettings.traceFunctionCalls + auto trace = settings.traceFunctionCalls ? std::make_unique(positions[pos]) : nullptr; @@ -2480,7 +2303,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat path.resolveSymlinks(), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.baseName(), - FileIngestionMethod::Recursive, + ContentAddressMethod::Raw::NixArchive, nullptr, repair); allowPath(dstPath); @@ -2827,7 +2650,7 @@ void EvalState::printStatistics() } -SourcePath resolveExprPath(SourcePath path) +SourcePath resolveExprPath(SourcePath path, bool addDefaultNix) { unsigned int followCount = 0, maxFollow = 1024; @@ -2843,7 +2666,7 @@ SourcePath resolveExprPath(SourcePath path) } /* If `path' refers to a directory, append `/default.nix'. */ - if (path.resolveSymlinks().lstat().type == SourceAccessor::tDirectory) + if (addDefaultNix && path.resolveSymlinks().lstat().type == SourceAccessor::tDirectory) return path / "default.nix"; return path; @@ -2922,7 +2745,7 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ return {corepkgsFS, CanonPath(path.substr(3))}; error( - evalSettings.pureEval + settings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", path @@ -2936,14 +2759,18 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa auto i = lookupPathResolved.find(value); if (i != lookupPathResolved.end()) return i->second; - std::optional res; + auto finish = [&](std::string res) { + debug("resolved search path element '%s' to '%s'", value, res); + lookupPathResolved.emplace(value, res); + return res; + }; if (EvalSettings::isPseudoUrl(value)) { try { auto accessor = fetchers::downloadTarball( EvalSettings::resolvePseudoUrl(value)).accessor; auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); - res = { store->toRealPath(storePath) }; + return finish(store->toRealPath(storePath)); } catch (Error & e) { logWarning({ .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) @@ -2951,15 +2778,17 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa } } - else if (hasPrefix(value, "flake:")) { - experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first; - res = { store->toRealPath(storePath) }; + if (auto colPos = value.find(':'); colPos != value.npos) { + auto scheme = value.substr(0, colPos); + auto rest = value.substr(colPos + 1); + if (auto * hook = get(settings.lookupPathHooks, scheme)) { + auto res = (*hook)(store, rest); + if (res) + return finish(std::move(*res)); + } } - else { + { auto path = absPath(value); /* Allow access to paths in the search path. */ @@ -2976,22 +2805,17 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa } if (pathExists(path)) - res = { path }; + return finish(std::move(path)); else { logWarning({ .msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value) }); - res = std::nullopt; } } - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else - debug("failed to resolve search path element '%s'", value); + debug("failed to resolve search path element '%s'", value); + return std::nullopt; - lookupPathResolved.emplace(value, res); - return res; } @@ -3002,7 +2826,7 @@ Expr * EvalState::parse( const SourcePath & basePath, std::shared_ptr & staticEnv) { - auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS, exprSymbols); + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, rootFS, exprSymbols); result->bindVars(*this, staticEnv); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index dac763268..e45358055 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -3,6 +3,7 @@ #include "attr-set.hh" #include "eval-error.hh" +#include "eval-gc.hh" #include "types.hh" #include "value.hh" #include "nixexpr.hh" @@ -29,6 +30,7 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; +struct EvalSettings; class EvalState; class StorePath; struct SingleDerivedPath; @@ -38,7 +40,6 @@ namespace eval_cache { class EvalCache; } - /** * Function that implements a primop. */ @@ -146,12 +147,6 @@ std::string printValue(EvalState & state, Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -/** - * Initialise the Boehm GC, if applicable. - */ -void initGC(); - - struct RegexCache; std::shared_ptr makeRegexCache(); @@ -167,13 +162,17 @@ struct DebugTrace { class EvalState : public std::enable_shared_from_this { public: + const EvalSettings & settings; SymbolTable symbols; PosTable positions; const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sRight, sWrong, sStructuredAttrs, + sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites, + sMaxSize, sMaxClosureSize, + sBuilder, sArgs, sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, @@ -354,6 +353,7 @@ public: EvalState( const LookupPath & _lookupPath, ref store, + const EvalSettings & settings, std::shared_ptr buildStore = nullptr); ~EvalState(); @@ -850,8 +850,10 @@ std::string showType(const Value & v); /** * If `path` refers to a directory, then append "/default.nix". + * + * @param addDefaultNix Whether to append "/default.nix" after resolving symlinks. */ -SourcePath resolveExprPath(SourcePath path); +SourcePath resolveExprPath(SourcePath path, bool addDefaultNix = true); /** * Whether a URI is allowed, assuming restrictEval is enabled diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 0d2aecc58..896733423 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -374,21 +374,26 @@ static void getDerivations(EvalState & state, Value & vIn, bound to the attribute with the "lower" name should take precedence). */ for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) { - debug("evaluating attribute '%1%'", state.symbols[i->name]); - if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex)) - continue; - std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]); - if (combineChannels) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { - /* If the value of this attribute is itself a set, - should we recurse into it? => Only if it has a - `recurseForDerivations = true' attribute. */ - if (i->value->type() == nAttrs) { - auto j = i->value->attrs()->get(state.sRecurseForDerivations); - if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + try { + debug("evaluating attribute '%1%'", state.symbols[i->name]); + if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex)) + continue; + std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]); + if (combineChannels) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { + /* If the value of this attribute is itself a set, + should we recurse into it? => Only if it has a + `recurseForDerivations = true' attribute. */ + if (i->value->type() == nAttrs) { + auto j = i->value->attrs()->get(state.sRecurseForDerivations); + if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + } } + } catch (Error & e) { + e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", state.symbols[i->name]); + throw; } } } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index d128064a5..68518e184 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,7 +8,6 @@ libexpr_SOURCES := \ $(wildcard $(d)/*.cc) \ $(wildcard $(d)/value/*.cc) \ $(wildcard $(d)/primops/*.cc) \ - $(wildcard $(d)/flake/*.cc) \ $(d)/lexer-tab.cc \ $(d)/parser-tab.cc # Not just for this library itself, but also for downstream libraries using this library @@ -16,7 +15,7 @@ libexpr_SOURCES := \ INCLUDE_libexpr := -I $(d) libexpr_CXXFLAGS += \ - $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libmain) $(INCLUDE_libexpr) \ + $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) \ -DGC_THREADS libexpr_LIBS = libutil libstore libfetchers @@ -45,11 +44,7 @@ $(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconf $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) -$(foreach i, $(wildcard src/libexpr/flake/*.hh), \ - $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh - -$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/call-flake.nix.gen.hh diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build new file mode 100644 index 000000000..e65cf545f --- /dev/null +++ b/src/libexpr/meson.build @@ -0,0 +1,202 @@ +project('nix-expr', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +boost = dependency( + 'boost', + modules : ['container', 'context'], +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +bdw_gc = dependency('bdw-gc', required : get_option('gc')) +if bdw_gc.found() + deps_public += bdw_gc + foreach funcspec : [ + 'pthread_attr_get_np', + 'pthread_getattr_np', + ] + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) + endforeach + configdata.set('GC_THREADS', 1) +endif +configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int()) + +toml11 = dependency('toml11', version : '>=3.7.0', method : 'cmake') +deps_other += toml11 + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +parser_tab = custom_target( + input : 'parser.y', + output : [ + 'parser-tab.cc', + 'parser-tab.hh', + ], + command : [ + 'bison', + '-v', + '-o', + '@OUTPUT0@', + '@INPUT@', + '-d', + ], + # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add + # an install script below which removes parser-tab.cc. + install : true, + install_dir : get_option('includedir') / 'nix', +) + +lexer_tab = custom_target( + input : [ + 'lexer.l', + parser_tab, + ], + output : [ + 'lexer-tab.cc', + 'lexer-tab.hh', + ], + command : [ + 'flex', + '--outfile', + '@OUTPUT0@', + '--header-file=' + '@OUTPUT1@', + '@INPUT0@', + ], + # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add + # an install script below which removes lexer-tab.cc. + install : true, + install_dir : get_option('includedir') / 'nix', +) + +subdir('build-utils-meson/generate-header') + +generated_headers = [] +foreach header : [ + 'imported-drv-to-derivation.nix', + 'fetchurl.nix', + 'call-flake.nix', +] + generated_headers += gen_header.process(header) +endforeach + +sources = files( + 'attr-path.cc', + 'attr-set.cc', + 'eval-cache.cc', + 'eval-error.cc', + 'eval-gc.cc', + 'eval-settings.cc', + 'eval.cc', + 'function-trace.cc', + 'get-drvs.cc', + 'json-to-value.cc', + 'nixexpr.cc', + 'paths.cc', + 'primops.cc', + 'print-ambiguous.cc', + 'print.cc', + 'search-path.cc', + 'value-to-json.cc', + 'value-to-xml.cc', + 'value/context.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'attr-path.hh', + 'attr-set.hh', + 'eval-cache.hh', + 'eval-error.hh', + 'eval-gc.hh', + 'eval-inline.hh', + 'eval-settings.hh', + 'eval.hh', + 'function-trace.hh', + 'gc-small-vector.hh', + 'get-drvs.hh', + 'json-to-value.hh', + 'nixexpr.hh', + 'parser-state.hh', + 'pos-idx.hh', + 'pos-table.hh', + 'primops.hh', + 'print-ambiguous.hh', + 'print-options.hh', + 'print.hh', + 'repl-exit-status.hh', + 'search-path.hh', + 'symbol-table.hh', + 'value-to-json.hh', + 'value-to-xml.hh', + 'value.hh', + 'value/context.hh', +) + +subdir('primops') + +this_library = library( + 'nixexpr', + sources, + parser_tab, + lexer_tab, + generated_headers, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr/meson.options b/src/libexpr/meson.options new file mode 100644 index 000000000..242d30ea7 --- /dev/null +++ b/src/libexpr/meson.options @@ -0,0 +1,3 @@ +option('gc', type : 'feature', + description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)', +) diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix new file mode 100644 index 000000000..704456c96 --- /dev/null +++ b/src/libexpr/package.nix @@ -0,0 +1,117 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, bison +, flex +, cmake # for resolving toml11 dep + +, nix-util +, nix-store +, nix-fetchers +, boost +, boehmgc +, nlohmann_json +, toml11 + +# Configuration Options + +, version + +# Whether to use garbage collection for the Nix language evaluator. +# +# If it is disabled, we just leak memory, but this is not as bad as it +# sounds so long as evaluation just takes places within short-lived +# processes. (When the process exits, the memory is reclaimed; it is +# only leaked *within* the process.) +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-expr"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + ./primops/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ./lexer.l + ./parser.y + (fileset.fileFilter (file: file.hasExt "nix") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + bison + flex + cmake + ]; + + buildInputs = [ + toml11 + ]; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + boost + nlohmann_json + ] ++ lib.optional enableGC boehmgc; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "gc" enableGC) + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 5a928e9aa..cff6282fa 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -46,6 +46,7 @@ struct ParserState PosTable::Origin origin; const ref rootFS; const Expr::AstSymbols & s; + const EvalSettings & settings; void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 00300449f..709a4532a 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -25,7 +25,6 @@ #include "nixexpr.hh" #include "eval.hh" #include "eval-settings.hh" -#include "globals.hh" #include "parser-state.hh" #define YYLTYPE ::nix::ParserLocation @@ -40,6 +39,7 @@ Expr * parseExprFromBuf( Pos::Origin origin, const SourcePath & basePath, SymbolTable & symbols, + const EvalSettings & settings, PosTable & positions, const ref rootFS, const Expr::AstSymbols & astSymbols); @@ -294,7 +294,7 @@ path_start $$ = new ExprPath(ref(state->rootFS), std::move(path)); } | HPATH { - if (evalSettings.pureEval) { + if (state->settings.pureEval) { throw Error( "the path '%s' can not be resolved in pure mode", std::string_view($1.p, $1.l) @@ -429,6 +429,7 @@ Expr * parseExprFromBuf( Pos::Origin origin, const SourcePath & basePath, SymbolTable & symbols, + const EvalSettings & settings, PosTable & positions, const ref rootFS, const Expr::AstSymbols & astSymbols) @@ -441,6 +442,7 @@ Expr * parseExprFromBuf( .origin = positions.addOrigin(origin, length), .rootFS = rootFS, .s = astSymbols, + .settings = settings, }; yylex_init(&scanner); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9177b0a2f..134363e1a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -5,7 +5,6 @@ #include "eval.hh" #include "eval-settings.hh" #include "gc-small-vector.hh" -#include "globals.hh" #include "json-to-value.hh" #include "names.hh" #include "path-references.hh" @@ -78,8 +77,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (drvs.empty()) return {}; - if (isIFD && !evalSettings.enableImportFromDerivation) - error( + if (isIFD && !settings.enableImportFromDerivation) + error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", drvs.begin()->to_string(*store) ).debugThrow(); @@ -733,11 +732,12 @@ static RegisterPrimOp primop_genericClosure(PrimOp { Each attribute set in the list `startSet` and the list returned by `operator` must have an attribute `key`, which must support equality comparison. The value of `key` can be one of the following types: - - [Number](@docroot@/language/values.md#type-number) - - [Boolean](@docroot@/language/values.md#type-boolean) - - [String](@docroot@/language/values.md#type-string) - - [Path](@docroot@/language/values.md#type-path) - - [List](@docroot@/language/values.md#list) + - [Int](@docroot@/language/types.md#type-int) + - [Float](@docroot@/language/types.md#type-float) + - [Boolean](@docroot@/language/types.md#type-boolean) + - [String](@docroot@/language/types.md#type-string) + - [Path](@docroot@/language/types.md#type-path) + - [List](@docroot@/language/types.md#list) The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added. Items are compared by their `key` attribute. @@ -901,7 +901,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va MaintainCount trylevel(state.trylevel); ReplExitStatus (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; - if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry) + if (state.debugRepl && state.settings.ignoreExceptionsDuringTry) { /* to prevent starting the repl from exceptions withing a tryEval, null it. */ savedDebugRepl = state.debugRepl; @@ -950,7 +950,7 @@ static RegisterPrimOp primop_tryEval({ static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv")); - v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); + v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or("")); } static RegisterPrimOp primop_getEnv({ @@ -1017,7 +1017,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", ValuePrinter(state, *args[0])); - if (evalSettings.builtinsTraceDebugger) { + if (state.settings.builtinsTraceDebugger) { state.runDebugRepl(nullptr); } state.forceValue(*args[1], pos); @@ -1056,11 +1056,11 @@ static void prim_warn(EvalState & state, const PosIdx pos, Value * * args, Value logWarning(info); } - if (evalSettings.builtinsAbortOnWarn) { + if (state.settings.builtinsAbortOnWarn) { // Not an EvalError or subclass, which would cause the error to be stored in the eval cache. state.error("aborting to reveal stack trace of warning, as abort-on-warn is set").setIsFromExpr().debugThrow(); } - if (evalSettings.builtinsTraceDebugger || evalSettings.builtinsDebuggerOnWarn) { + if (state.settings.builtinsTraceDebugger || state.settings.builtinsDebuggerOnWarn) { state.runDebugRepl(nullptr); } state.forceValue(*args[1], pos); @@ -1163,12 +1163,34 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * } } +/** + * Early validation for the derivation name, for better error message. + * It is checked again when constructing store paths. + * + * @todo Check that the `.drv` suffix also fits. + */ +static void checkDerivationName(EvalState & state, std::string_view drvName) +{ + try { + checkName(drvName); + } catch (BadStorePathName & e) { + // "Please pass a different name": Users may not be aware that they can + // pass a different one, in functions like `fetchurl` where the name + // is optional. + // Note that Nixpkgs generally won't trigger this, because `mkDerivation` + // sanitizes the name. + state.error("invalid derivation name: %s. Please pass a different '%s'.", Uncolored(e.message()), "name").debugThrow(); + } +} + static void derivationStrictInternal( EvalState & state, const std::string & drvName, const Bindings * attrs, Value & v) { + checkDerivationName(state, drvName); + /* Check whether attributes should be passed as a JSON file. */ using nlohmann::json; std::optional jsonObject; @@ -1209,7 +1231,7 @@ static void derivationStrictInternal( auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") { // back compat, new name is "nar" - ingestionMethod = FileIngestionMethod::Recursive; + ingestionMethod = ContentAddressMethod::Raw::NixArchive; } else try { ingestionMethod = ContentAddressMethod::parse(s); } catch (UsageError &) { @@ -1217,9 +1239,9 @@ static void derivationStrictInternal( "invalid value '%s' for 'outputHashMode' attribute", s ).atPos(v).debugThrow(); } - if (ingestionMethod == TextIngestionMethod {}) + if (ingestionMethod == ContentAddressMethod::Raw::Text) experimentalFeatureSettings.require(Xp::DynamicDerivations); - if (ingestionMethod == FileIngestionMethod::Git) + if (ingestionMethod == ContentAddressMethod::Raw::Git) experimentalFeatureSettings.require(Xp::GitHashing); }; @@ -1308,6 +1330,20 @@ static void derivationStrictInternal( handleOutputs(ss); } + if (i->name == state.sAllowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead", drvName); + if (i->name == state.sAllowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead", drvName); + if (i->name == state.sDisallowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead", drvName); + if (i->name == state.sDisallowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead", drvName); + if (i->name == state.sMaxSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead", drvName); + if (i->name == state.sMaxClosureSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead", drvName); + + } else { auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); @@ -1377,7 +1413,7 @@ static void derivationStrictInternal( /* Check whether the derivation name is valid. */ if (isDerivation(drvName) && - !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } && + !(ingestionMethod == ContentAddressMethod::Raw::Text && outputs.size() == 1 && *(outputs.begin()) == "out")) { @@ -1399,7 +1435,7 @@ static void derivationStrictInternal( auto h = newHashAllowEmpty(*outputHash, outputHashAlgo); - auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::Flat); DerivationOutput::CAFixed dof { .ca = ContentAddress { @@ -1418,7 +1454,7 @@ static void derivationStrictInternal( .atPos(v).debugThrow(); auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256); - auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive); for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); @@ -1564,7 +1600,7 @@ static RegisterPrimOp primop_toPath({ corner cases. */ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - if (evalSettings.pureEval) + if (state.settings.pureEval) state.error( "'%s' is not allowed in pure evaluation mode", "builtins.storePath" @@ -1674,7 +1710,7 @@ static RegisterPrimOp primop_baseNameOf({ .name = "baseNameOf", .args = {"x"}, .doc = R"( - Return the *base name* of either a [path value](@docroot@/language/values.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules. + Return the *base name* of either a [path value](@docroot@/language/types.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules. For a path value, the *base name* is considered to be the part of the path after the last directory separator, including any file extensions. This is the simple case, as path values don't have trailing slashes. @@ -1808,7 +1844,7 @@ static RegisterPrimOp primop_findFile(PrimOp { .doc = R"( Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes: + A search path is represented list of [attribute sets](./types.md#attribute-set) with two attributes: - `prefix` is a relative path. - `path` denotes a file system location The exact syntax depends on the command line interface. @@ -1829,14 +1865,14 @@ static RegisterPrimOp primop_findFile(PrimOp { } ``` - The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match: + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. Note that the `path` may need to be downloaded at this point to look inside. - If the suffix is found inside that directory, then the entry is a match. The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): ```nix @@ -2194,7 +2230,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val }) : ({ StringSource s { contents }; - state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair); + state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair); }); /* Note: we don't need to add `context' to the context of the @@ -2257,7 +2293,7 @@ static RegisterPrimOp primop_toFile({ ``` Note that `${configFile}` is a - [string interpolation](@docroot@/language/values.md#type-string), so the result of the + [string interpolation](@docroot@/language/types.md#type-string), so the result of the expression `configFile` (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be spliced into the resulting string. @@ -2377,7 +2413,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2440,7 +2476,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value std::optional path; std::string name; Value * filterFun = nullptr; - ContentAddressMethod method = FileIngestionMethod::Recursive; + auto method = ContentAddressMethod::Raw::NixArchive; std::optional expectedHash; NixStringContext context; @@ -2456,8 +2492,8 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path"); else if (n == "recursive") method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") - ? FileIngestionMethod::Recursive - : FileIngestionMethod::Flat; + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else @@ -4483,7 +4519,7 @@ void EvalState::createBaseEnv() addConstant("builtins", v, { .type = nAttrs, .doc = R"( - Contains all the [built-in functions](@docroot@/language/builtins.md) and values. + Contains all the built-in functions and values. Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: @@ -4503,7 +4539,7 @@ void EvalState::createBaseEnv() It can be returned by [comparison operators](@docroot@/language/operators.md#Comparison) and used in - [conditional expressions](@docroot@/language/constructs.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#Conditionals). The name `true` is not special, and can be shadowed: @@ -4523,7 +4559,7 @@ void EvalState::createBaseEnv() It can be returned by [comparison operators](@docroot@/language/operators.md#Comparison) and used in - [conditional expressions](@docroot@/language/constructs.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#Conditionals). The name `false` is not special, and can be shadowed: @@ -4548,7 +4584,7 @@ void EvalState::createBaseEnv() )", }); - if (!evalSettings.pureEval) { + if (!settings.pureEval) { v.mkInt(time(0)); } addConstant("__currentTime", v, { @@ -4575,8 +4611,8 @@ void EvalState::createBaseEnv() .impureOnly = true, }); - if (!evalSettings.pureEval) - v.mkString(evalSettings.getCurrentSystem()); + if (!settings.pureEval) + v.mkString(settings.getCurrentSystem()); addConstant("__currentSystem", v, { .type = nString, .doc = R"( @@ -4656,7 +4692,7 @@ void EvalState::createBaseEnv() #ifndef _WIN32 // TODO implement on Windows // Miscellaneous - if (evalSettings.enableNativeCode) { + if (settings.enableNativeCode) { addPrimOp({ .name = "__importNative", .arity = 2, @@ -4679,7 +4715,7 @@ void EvalState::createBaseEnv() error if `--trace-verbose` is enabled. Then return *e2*. This function is useful for debugging. )", - .fun = evalSettings.traceVerbose ? prim_trace : prim_second, + .fun = settings.traceVerbose ? prim_trace : prim_second, }); /* Add a value containing the current Nix expression search path. */ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index d9ba6aa97..7b5f4193a 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -53,7 +53,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a // whitelist. Ah well. state.checkURI(url); - if (evalSettings.pureEval && !rev) + if (state.settings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); fetchers::Attrs attrs; diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a99a71577..567b73f9a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,4 +1,4 @@ -#include "libfetchers/attrs.hh" +#include "attrs.hh" #include "primops.hh" #include "eval-inline.hh" #include "eval-settings.hh" @@ -171,10 +171,10 @@ static void fetchTree( } } - if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) + if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input.isLocked()) { + if (state.settings.pureEval && !input.isLocked()) { auto fetcher = "fetchTree"; if (params.isFetchGit) fetcher = "fetchGit"; @@ -431,7 +431,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.forceValue(*args[0], pos); - if (args[0]->type() == nAttrs) { + bool isArgAttrs = args[0]->type() == nAttrs; + bool nameAttrPassed = false; + + if (isArgAttrs) { for (auto & attr : *args[0]->attrs()) { std::string_view n(state.symbols[attr.name]); @@ -439,8 +442,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256); - else if (n == "name") + else if (n == "name") { + nameAttrPassed = true; name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); + } else state.error("unsupported argument '%s' to '%s'", n, who) .atPos(pos).debugThrow(); @@ -453,14 +458,27 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch"); if (who == "fetchTarball") - url = evalSettings.resolvePseudoUrl(*url); + url = state.settings.resolvePseudoUrl(*url); state.checkURI(*url); if (name == "") name = baseNameOf(*url); - if (evalSettings.pureEval && !expectedHash) + try { + checkName(name); + } catch (BadStorePathName & e) { + auto resolution = + nameAttrPassed ? HintFmt("Please change the value for the 'name' attribute passed to '%s', so that it can create a valid store path.", who) : + isArgAttrs ? HintFmt("Please add a valid 'name' attribute to the argument for '%s', so that it can create a valid store path.", who) : + HintFmt("Please pass an attribute set with 'url' and 'name' attributes to '%s', so that it can create a valid store path.", who); + + state.error( + std::string("invalid store path name when fetching URL '%s': %s. %s"), *url, Uncolored(e.message()), Uncolored(resolution.str())) + .atPos(pos).debugThrow(); + } + + if (state.settings.pureEval && !expectedHash) state.error("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow(); // early exit if pinned and already in the store @@ -468,7 +486,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .method = unpack ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat, .hash = *expectedHash, .references = {} }); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 9bee8ca38..6c7d303e8 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -1,9 +1,8 @@ #include "primops.hh" #include "eval-inline.hh" -#include "../../toml11/toml.hpp" - #include +#include namespace nix { diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build new file mode 100644 index 000000000..f910fe237 --- /dev/null +++ b/src/libexpr/primops/meson.build @@ -0,0 +1,12 @@ +generated_headers += gen_header.process( + 'derivation.nix', + preserve_path_from: meson.project_source_root(), +) + +sources += files( + 'context.cc', + 'fetchClosure.cc', + 'fetchMercurial.cc', + 'fetchTree.cc', + 'fromTOML.cc', +) diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 10fe7923f..2f377e588 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -163,8 +163,8 @@ private: EvalState & state; PrintOptions options; std::optional seen; - size_t attrsPrinted = 0; - size_t listItemsPrinted = 0; + size_t totalAttrsPrinted = 0; + size_t totalListItemsPrinted = 0; std::string indent; void increaseIndent() @@ -345,11 +345,13 @@ private: auto prettyPrint = shouldPrettyPrintAttrs(sorted); + size_t currentAttrsPrinted = 0; + for (auto & i : sorted) { printSpace(prettyPrint); - if (attrsPrinted >= options.maxAttrs) { - printElided(sorted.size() - attrsPrinted, "attribute", "attributes"); + if (totalAttrsPrinted >= options.maxAttrs) { + printElided(sorted.size() - currentAttrsPrinted, "attribute", "attributes"); break; } @@ -357,7 +359,8 @@ private: output << " = "; print(*i.second, depth + 1); output << ";"; - attrsPrinted++; + totalAttrsPrinted++; + currentAttrsPrinted++; } decreaseIndent(); @@ -402,11 +405,14 @@ private: output << "["; auto listItems = v.listItems(); auto prettyPrint = shouldPrettyPrintList(listItems); + + size_t currentListItemsPrinted = 0; + for (auto elem : listItems) { printSpace(prettyPrint); - if (listItemsPrinted >= options.maxListItems) { - printElided(listItems.size() - listItemsPrinted, "item", "items"); + if (totalListItemsPrinted >= options.maxListItems) { + printElided(listItems.size() - currentListItemsPrinted, "item", "items"); break; } @@ -415,7 +421,8 @@ private: } else { printNullptr(); } - listItemsPrinted++; + totalListItemsPrinted++; + currentListItemsPrinted++; } decreaseIndent(); @@ -588,8 +595,8 @@ public: void print(Value & v) { - attrsPrinted = 0; - listItemsPrinted = 0; + totalAttrsPrinted = 0; + totalListItemsPrinted = 0; indent.clear(); if (options.trackRepeated) { diff --git a/src/libfetchers/.version b/src/libfetchers/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libfetchers/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libfetchers/build-utils-meson b/src/libfetchers/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libfetchers/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index e7d5244dc..21c42567c 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,4 +1,5 @@ #include "fetch-settings.hh" +#include "config-global.hh" namespace nix { diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 50cd4d161..629967697 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -70,30 +70,6 @@ struct FetchSettings : public Config Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; - Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", - R"( - Path or URI of the global flake registry. - - When empty, disables the global flake registry. - )", - {}, true, Xp::Flakes}; - - Setting useRegistries{this, true, "use-registries", - "Whether to use flake registries to resolve flake references.", - {}, true, Xp::Flakes}; - - Setting acceptFlakeConfig{this, false, "accept-flake-config", - "Whether to accept nix configuration from a flake without prompting.", - {}, true, Xp::Flakes}; - - Setting commitLockFileSummary{ - this, "", "commit-lock-file-summary", - R"( - The commit summary to use when committing changed flake lock files. If - empty, the summary is generated based on the action performed. - )", - {"commit-lockfile-summary"}, true, Xp::Flakes}; - Setting trustTarballsFromGitForges{ this, true, "trust-tarballs-from-git-forges", R"( @@ -108,7 +84,6 @@ struct FetchSettings : public Config `narHash` attribute is specified, e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`. )"}; - }; // FIXME: don't use a global variable. diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index 81af1e240..c762629f3 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -18,7 +18,7 @@ StorePath fetchToStore( const SourcePath & path, FetchMode mode, std::string_view name = "source", - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 73923907c..294960678 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -260,6 +260,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, final] = scheme->getAccessor(store, *this); + assert(!accessor->fingerprint); accessor->fingerprint = scheme->getFingerprint(store, final); return {accessor, std::move(final)}; @@ -305,7 +306,7 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = *narHash, .references = {}, }); @@ -418,7 +419,7 @@ namespace nlohmann { using namespace nix; fetchers::PublicKey adl_serializer::from_json(const json & json) { - fetchers::PublicKey res = { }; + fetchers::PublicKey res = { }; if (auto type = optionalValueAt(json, "type")) res.type = getString(*type); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 64fd39aed..a88bdc8b6 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -851,10 +851,10 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink } void createRegularFile( - const Path & path, + const CanonPath & path, std::function func) override { - auto pathComponents = tokenizeString>(path, "/"); + auto pathComponents = tokenizeString>(path.rel(), "/"); if (!prepareDirs(pathComponents, false)) return; git_writestream * stream = nullptr; @@ -862,11 +862,11 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink throw Error("creating a blob stream object: %s", git_error_last()->message); struct CRF : CreateRegularFileSink { - const Path & path; + const CanonPath & path; GitFileSystemObjectSinkImpl & back; git_writestream * stream; bool executable = false; - CRF(const Path & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream) + CRF(const CanonPath & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream) : path(path), back(back), stream(stream) {} void operator () (std::string_view data) override @@ -891,15 +891,15 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink : GIT_FILEMODE_BLOB); } - void createDirectory(const Path & path) override + void createDirectory(const CanonPath & path) override { - auto pathComponents = tokenizeString>(path, "/"); + auto pathComponents = tokenizeString>(path.rel(), "/"); (void) prepareDirs(pathComponents, true); } - void createSymlink(const Path & path, const std::string & target) override + void createSymlink(const CanonPath & path, const std::string & target) override { - auto pathComponents = tokenizeString>(path, "/"); + auto pathComponents = tokenizeString>(path.rel(), "/"); if (!prepareDirs(pathComponents, false)) return; git_oid oid; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ce80932f6..184c1383e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -41,21 +41,6 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st) return st.st_mtime + settings.tarballTtl > now; } -bool touchCacheFile(const Path & path, time_t touch_time) -{ -#ifndef _WIN32 // TODO implement - struct timeval times[2]; - times[0].tv_sec = touch_time; - times[0].tv_usec = 0; - times[1].tv_sec = touch_time; - times[1].tv_usec = 0; - - return lutimes(path.c_str(), times) == 0; -#else - return false; -#endif -} - Path getCachePath(std::string_view key, bool shallow) { return getCacheDir() @@ -594,8 +579,11 @@ struct GitInputScheme : InputScheme warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); } - if (!touchCacheFile(localRefFile, now)) - warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); + try { + setWriteTime(localRefFile, now, now); + } catch (Error & e) { + warn("could not update mtime for file '%s': %s", localRefFile, e.msg()); + } if (!originalRef && !storeCachedHead(repoInfo.url, ref)) warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 267e8607f..ddb41e63f 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -433,7 +433,7 @@ struct GitLabInputScheme : GitArchiveInputScheme store->toRealPath( downloadFile(store, url, "source", headers).storePath))); - if (json.is_array() && json.size() == 1 && json[0]["id"] != nullptr) { + if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { return RefInfo { .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 7bdf1e937..198795caa 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -213,7 +213,7 @@ struct MercurialInputScheme : InputScheme auto storePath = store->addToStore( input.getName(), {getFSSourceAccessor(), CanonPath(actualPath)}, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, + ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, filter); return storePath; diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build new file mode 100644 index 000000000..c39fe99f3 --- /dev/null +++ b/src/libfetchers/meson.build @@ -0,0 +1,93 @@ +project('nix-fetchers', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), +] +subdir('build-utils-meson/subprojects') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +libgit2 = dependency('libgit2') +deps_private += libgit2 + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'attrs.cc', + 'cache.cc', + 'fetch-settings.cc', + 'fetch-to-store.cc', + 'fetchers.cc', + 'filtering-source-accessor.cc', + 'git.cc', + 'git-utils.cc', + 'github.cc', + 'indirect.cc', + 'mercurial.cc', + 'mounted-source-accessor.cc', + 'path.cc', + 'store-path-accessor.cc', + 'registry.cc', + 'tarball.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'attrs.hh', + 'cache.hh', + 'fetch-settings.hh', + 'fetch-to-store.hh', + 'filtering-source-accessor.hh', + 'git-utils.hh', + 'mounted-source-accessor.hh', + 'fetchers.hh', + 'registry.hh', + 'store-path-accessor.hh', + 'tarball.hh', +) + +this_library = library( + 'nixfetchers', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix new file mode 100644 index 000000000..b4abb144b --- /dev/null +++ b/src/libfetchers/package.nix @@ -0,0 +1,84 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nlohmann_json +, libgit2 +, man + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-fetchers"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + libgit2 + ]; + + propagatedBuildInputs = [ + nix-store + nix-util + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated + # to work with `strictDeps`. + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index e00b9de46..52cbac5e0 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,12 +1,11 @@ #include "registry.hh" #include "tarball.hh" #include "users.hh" +#include "config-global.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "fetch-settings.hh" - #include namespace nix::fetchers { @@ -149,10 +148,25 @@ void overrideRegistry( flagRegistry->add(from, to, extraAttrs); } +struct RegistrySettings : Config +{ + Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", + R"( + Path or URI of the global flake registry. + + When empty, disables the global flake registry. + )", + {}, true, Xp::Flakes}; +}; + +RegistrySettings registrySettings; + +static GlobalConfig::Register rRegistrySettings(®istrySettings); + static std::shared_ptr getGlobalRegistry(ref store) { static auto reg = [&]() { - auto path = fetchSettings.flakeRegistry.get(); + auto path = registrySettings.flakeRegistry.get(); if (path == "") { return std::make_shared(Registry::Global); // empty registry } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 5de367052..aa8ff652f 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -365,6 +365,16 @@ struct TarballInputScheme : CurlInputScheme return {result.accessor, input}; } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto narHash = input.getNarHash()) + return narHash->to_string(HashFormat::SRI, true); + else if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; + } }; static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libflake/.version b/src/libflake/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libflake/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libflake/build-utils-meson b/src/libflake/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libflake/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libflake/flake-settings.cc b/src/libflake/flake-settings.cc new file mode 100644 index 000000000..ba97e0ce7 --- /dev/null +++ b/src/libflake/flake-settings.cc @@ -0,0 +1,12 @@ +#include "flake-settings.hh" +#include "config-global.hh" + +namespace nix { + +FlakeSettings::FlakeSettings() {} + +FlakeSettings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + +} diff --git a/src/libflake/flake-settings.hh b/src/libflake/flake-settings.hh new file mode 100644 index 000000000..f97c175e8 --- /dev/null +++ b/src/libflake/flake-settings.hh @@ -0,0 +1,53 @@ +#pragma once +///@file + +#include "types.hh" +#include "config.hh" +#include "util.hh" + +#include +#include + +#include + +namespace nix { + +struct FlakeSettings : public Config +{ + FlakeSettings(); + + Setting useRegistries{ + this, + true, + "use-registries", + "Whether to use flake registries to resolve flake references.", + {}, + true, + Xp::Flakes}; + + Setting acceptFlakeConfig{ + this, + false, + "accept-flake-config", + "Whether to accept nix configuration from a flake without prompting.", + {}, + true, + Xp::Flakes}; + + Setting commitLockFileSummary{ + this, + "", + "commit-lock-file-summary", + R"( + The commit summary to use when committing changed flake lock files. If + empty, the summary is generated based on the action performed. + )", + {"commit-lockfile-summary"}, + true, + Xp::Flakes}; +}; + +// TODO: don't use a global variable. +extern FlakeSettings flakeSettings; + +} diff --git a/src/libexpr/flake/config.cc b/src/libflake/flake/config.cc similarity index 95% rename from src/libexpr/flake/config.cc rename to src/libflake/flake/config.cc index e0c5d4512..498595359 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libflake/flake/config.cc @@ -1,6 +1,6 @@ #include "users.hh" -#include "globals.hh" -#include "fetch-settings.hh" +#include "config-global.hh" +#include "flake-settings.hh" #include "flake.hh" #include @@ -51,7 +51,7 @@ void ConfigFile::apply() else assert(false); - if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) { + if (!whitelist.count(baseName) && !nix::flakeSettings.acceptFlakeConfig) { bool trusted = false; auto trustedList = readTrustedList(); auto tlname = get(trustedList, name); diff --git a/src/libexpr/flake/flake.cc b/src/libflake/flake/flake.cc similarity index 97% rename from src/libexpr/flake/flake.cc rename to src/libflake/flake/flake.cc index 3af9ef14e..6f47b5992 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -9,6 +9,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "flake-settings.hh" #include "value-to-json.hh" #include "local-fs-store.hh" @@ -346,7 +347,7 @@ LockedFlake lockFlake( FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(fetchSettings.useRegistries); + auto useRegistries = lockFlags.useRegistries.value_or(flakeSettings.useRegistries); auto flake = getFlake(state, topRef, useRegistries, flakeCache); @@ -691,7 +692,7 @@ LockedFlake lockFlake( if (lockFlags.commitLockFile) { std::string cm; - cm = fetchSettings.commitLockFileSummary.get(); + cm = flakeSettings.commitLockFileSummary.get(); if (cm == "") { cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); @@ -803,7 +804,7 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V { std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (evalSettings.pureEval && !flakeRef.input.isLocked()) + if (state.settings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); callFlake(state, @@ -811,8 +812,8 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V LockFlags { .updateLockFile = false, .writeLockFile = false, - .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, - .allowUnlocked = !evalSettings.pureEval, + .useRegistries = !state.settings.pureEval && flakeSettings.useRegistries, + .allowUnlocked = !state.settings.pureEval, }), v); } @@ -949,10 +950,20 @@ std::optional LockedFlake::getFingerprint(ref store) const auto fingerprint = flake.lockedRef.input.getFingerprint(store); if (!fingerprint) return std::nullopt; + *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); + + /* Include revCount and lastModified because they're not + necessarily implied by the content fingerprint (e.g. for + tarball flakes) but can influence the evaluation result. */ + if (auto revCount = flake.lockedRef.input.getRevCount()) + *fingerprint += fmt(";revCount=%d", *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + *fingerprint += fmt(";lastModified=%d", *lastModified); + // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. - return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%s", *fingerprint, flake.lockedRef.subdir, lockFile)); + return hashString(HashAlgorithm::SHA256, *fingerprint); } Flake::~Flake() { } diff --git a/src/libexpr/flake/flake.hh b/src/libflake/flake/flake.hh similarity index 100% rename from src/libexpr/flake/flake.hh rename to src/libflake/flake/flake.hh diff --git a/src/libexpr/flake/flakeref.cc b/src/libflake/flake/flakeref.cc similarity index 100% rename from src/libexpr/flake/flakeref.cc rename to src/libflake/flake/flakeref.cc diff --git a/src/libexpr/flake/flakeref.hh b/src/libflake/flake/flakeref.hh similarity index 100% rename from src/libexpr/flake/flakeref.hh rename to src/libflake/flake/flakeref.hh diff --git a/src/libexpr/flake/lockfile.cc b/src/libflake/flake/lockfile.cc similarity index 100% rename from src/libexpr/flake/lockfile.cc rename to src/libflake/flake/lockfile.cc diff --git a/src/libexpr/flake/lockfile.hh b/src/libflake/flake/lockfile.hh similarity index 100% rename from src/libexpr/flake/lockfile.hh rename to src/libflake/flake/lockfile.hh diff --git a/src/libexpr/flake/url-name.cc b/src/libflake/flake/url-name.cc similarity index 100% rename from src/libexpr/flake/url-name.cc rename to src/libflake/flake/url-name.cc diff --git a/src/libexpr/flake/url-name.hh b/src/libflake/flake/url-name.hh similarity index 100% rename from src/libexpr/flake/url-name.hh rename to src/libflake/flake/url-name.hh diff --git a/src/libflake/local.mk b/src/libflake/local.mk new file mode 100644 index 000000000..2cceda2bf --- /dev/null +++ b/src/libflake/local.mk @@ -0,0 +1,17 @@ +libraries += libflake + +libflake_NAME = libnixflake + +libflake_DIR := $(d) + +libflake_SOURCES := $(wildcard $(d)/*.cc $(d)/flake/*.cc) + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libflake := -I $(d) + +libflake_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) + +libflake_LDFLAGS += $(THREAD_LDFLAGS) + +libflake_LIBS = libutil libstore libfetchers libexpr diff --git a/src/libflake/meson.build b/src/libflake/meson.build new file mode 100644 index 000000000..d3c3d3079 --- /dev/null +++ b/src/libflake/meson.build @@ -0,0 +1,75 @@ +project('nix-flake', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), +] +subdir('build-utils-meson/subprojects') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'flake-settings.cc', + 'flake/config.cc', + 'flake/flake.cc', + 'flake/flakeref.cc', + 'flake/url-name.cc', + 'flake/lockfile.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'flake-settings.hh', + 'flake/flake.hh', + 'flake/flakeref.hh', + 'flake/lockfile.hh', + 'flake/url-name.hh', +) + +this_library = library( + 'nixflake', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libflake/package.nix b/src/libflake/package.nix new file mode 100644 index 000000000..af6f5da94 --- /dev/null +++ b/src/libflake/package.nix @@ -0,0 +1,84 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nlohmann_json +, libgit2 +, man + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-flake"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-store + nix-util + nix-fetchers + nix-expr + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated + # to work with `strictDeps`. + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain/.version b/src/libmain/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libmain/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libmain/build-utils-meson b/src/libmain/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libmain/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 5b49aaabc..a94845ab8 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,5 +1,6 @@ #include "common-args.hh" #include "args/root.hh" +#include "config-global.hh" #include "globals.hh" #include "logging.hh" #include "loggers.hh" diff --git a/src/libmain/meson.build b/src/libmain/meson.build new file mode 100644 index 000000000..859ce22f8 --- /dev/null +++ b/src/libmain/meson.build @@ -0,0 +1,98 @@ +project('nix-main', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), +] +subdir('build-utils-meson/subprojects') + + +pubsetbuf_test = ''' +#include + +using namespace std; + +char buf[1024]; + +int main() { + cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); +} +''' + +configdata.set( + 'HAVE_PUBSETBUF', + cxx.compiles(pubsetbuf_test).to_int(), + description: 'Optionally used for buffering on standard error' +) + +config_h = configure_file( + configuration : configdata, + output : 'config-main.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-main.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'common-args.cc', + 'loggers.cc', + 'progress-bar.cc', + 'shared.cc', +) + +if host_machine.system() != 'windows' + sources += files( + 'unix/stack.cc', + ) +endif + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'common-args.hh', + 'loggers.hh', + 'progress-bar.hh', + 'shared.hh', +) + +this_library = library( + 'nixmain', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libmain/package.nix b/src/libmain/package.nix new file mode 100644 index 000000000..bbd97ec3e --- /dev/null +++ b/src/libmain/package.nix @@ -0,0 +1,79 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, openssl + +, nix-util +, nix-store + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-main"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util + nix-store + openssl + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated + # to work with `strictDeps`. + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index c1c936248..fc55fe3f1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -320,6 +320,10 @@ void showManPage(const std::string & name) restoreProcessContext(); setEnv("MANPATH", settings.nixManDir.c_str()); execlp("man", "man", name.c_str(), nullptr); + if (errno == ENOENT) { + // Not SysError because we don't want to suffix the errno, aka No such file or directory. + throw Error("The '%1%' command was not found, but it is needed for '%2%' and some other '%3%' commands' help text. Perhaps you could install the '%1%' command?", "man", name.c_str(), "nix-*"); + } throw SysError("command 'man %1%' failed", name.c_str()); } diff --git a/src/libstore-c/.version b/src/libstore-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libstore-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libstore-c/build-utils-meson b/src/libstore-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libstore-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build new file mode 100644 index 000000000..917de4cda --- /dev/null +++ b/src/libstore-c/meson.build @@ -0,0 +1,83 @@ +project('nix-store-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), +] +subdir('build-utils-meson/subprojects') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-store.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_store.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_store.h', +) + +# TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` +headers += files('nix_api_store_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixstorec', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix new file mode 100644 index 000000000..fc34c1bda --- /dev/null +++ b/src/libstore-c/package.nix @@ -0,0 +1,79 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-c +, nix-store + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-c + nix-store + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libstore/.version b/src/libstore/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libstore/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 95a8d5a7a..e8c8892b3 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -322,7 +322,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( if (static_cast(dumpMethod) == hashMethod.getFileIngestionMethod()) caHash = hashString(HashAlgorithm::SHA256, dump2.s); switch (dumpMethod) { - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: // The dump is already NAR in this case, just use it. nar = dump2.s; break; @@ -339,7 +339,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( } else { // Otherwise, we have to do th same hashing as NAR so our single // hash will suffice for both purposes. - if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); } StringSource narDump { nar }; diff --git a/src/libstore/build-utils-meson b/src/libstore/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libstore/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 4226fb61a..64b8495e1 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3,6 +3,7 @@ # include "hook-instance.hh" #endif #include "processes.hh" +#include "config-global.hh" #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" @@ -25,6 +26,10 @@ #include #include +#ifndef _WIN32 // TODO abstract over proc exit status +# include +#endif + #include namespace nix { @@ -1033,7 +1038,7 @@ void DerivationGoal::buildDone() BuildResult::Status st = BuildResult::MiscFailure; -#ifndef _WIN32 +#ifndef _WIN32 // TODO abstract over proc exit status if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) st = BuildResult::TimedOut; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b53dc771a..8a5d6de72 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -19,7 +19,6 @@ Worker::Worker(Store & store, Store & evalStore) , store(store) , evalStore(evalStore) { - /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; nrSubstitutions = 0; lastWokenUp = steady_time_point::min(); @@ -530,7 +529,7 @@ bool Worker::pathContentsGood(const StorePath & path) else { auto current = hashPath( {store.getFSAccessor(), CanonPath(store.printStorePath(path))}, - FileIngestionMethod::Recursive, info->narHash.algo).first; + FileIngestionMethod::NixArchive, info->narHash.algo).first; Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current; } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 7d67030d7..33a7bf015 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -59,7 +59,7 @@ struct HookInstance; #endif /** - * The worker class. + * Coordinates one or more realisations and their interdependencies. */ class Worker { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 4ed4f2de5..e1cdfece6 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -8,98 +8,136 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: + // Not prefixed for back compat return ""; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return "r:"; case FileIngestionMethod::Git: experimentalFeatureSettings.require(Xp::GitHashing); return "git:"; default: - throw Error("impossible, caught both cases"); + assert(false); } } std::string_view ContentAddressMethod::render() const { - return std::visit(overloaded { - [](TextIngestionMethod) -> std::string_view { return "text"; }, - [](FileIngestionMethod m2) { - /* Not prefixed for back compat with things that couldn't produce text before. */ - return renderFileIngestionMethod(m2); - }, - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return renderFileIngestionMethod(getFileIngestionMethod()); + default: + assert(false); + } +} + +/** + * **Not surjective** + * + * This is not exposed because `FileIngestionMethod::Flat` maps to + * `ContentAddressMethod::Raw::Flat` and + * `ContentAddressMethod::Raw::Text` alike. We can thus only safely use + * this when the latter is ruled out (e.g. because it is already + * handled). + */ +static ContentAddressMethod fileIngestionMethodToContentAddressMethod(FileIngestionMethod m) +{ + switch (m) { + case FileIngestionMethod::Flat: + return ContentAddressMethod::Raw::Flat; + case FileIngestionMethod::NixArchive: + return ContentAddressMethod::Raw::NixArchive; + case FileIngestionMethod::Git: + return ContentAddressMethod::Raw::Git; + default: + assert(false); + } } ContentAddressMethod ContentAddressMethod::parse(std::string_view m) { if (m == "text") - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; else - return parseFileIngestionMethod(m); + return fileIngestionMethodToContentAddressMethod( + parseFileIngestionMethod(m)); } std::string_view ContentAddressMethod::renderPrefix() const { - return std::visit(overloaded { - [](TextIngestionMethod) -> std::string_view { return "text:"; }, - [](FileIngestionMethod m2) { - /* Not prefixed for back compat with things that couldn't produce text before. */ - return makeFileIngestionPrefix(m2); - }, - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return makeFileIngestionPrefix(getFileIngestionMethod()); + default: + assert(false); + } } ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { if (splitPrefix(m, "r:")) { - return FileIngestionMethod::Recursive; + return ContentAddressMethod::Raw::NixArchive; } else if (splitPrefix(m, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); - return FileIngestionMethod::Git; + return ContentAddressMethod::Raw::Git; } else if (splitPrefix(m, "text:")) { - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; + } + return ContentAddressMethod::Raw::Flat; +} + +/** + * This is slightly more mindful of forward compat in that it uses `fixed:` + * rather than just doing a raw empty prefix or `r:`, which doesn't "save room" + * for future changes very well. + */ +static std::string renderPrefixModern(const ContentAddressMethod & ca) +{ + switch (ca.raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return "fixed:" + makeFileIngestionPrefix(ca.getFileIngestionMethod()); + default: + assert(false); } - return FileIngestionMethod::Flat; } std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const { - return std::visit(overloaded { - [&](const TextIngestionMethod & th) { - return std::string{"text:"} + printHashAlgo(ha); - }, - [&](const FileIngestionMethod & fim) { - return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha); - } - }, raw); + return renderPrefixModern(*this) + printHashAlgo(ha); } FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const { - return std::visit(overloaded { - [&](const TextIngestionMethod & th) { - return FileIngestionMethod::Flat; - }, - [&](const FileIngestionMethod & fim) { - return fim; - } - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Flat: + return FileIngestionMethod::Flat; + case ContentAddressMethod::Raw::NixArchive: + return FileIngestionMethod::NixArchive; + case ContentAddressMethod::Raw::Git: + return FileIngestionMethod::Git; + case ContentAddressMethod::Raw::Text: + return FileIngestionMethod::Flat; + default: + assert(false); + } } std::string ContentAddress::render() const { - return std::visit(overloaded { - [](const TextIngestionMethod &) -> std::string { - return "text:"; - }, - [](const FileIngestionMethod & method) { - return "fixed:" - + makeFileIngestionPrefix(method); - }, - }, method.raw) - + this->hash.to_string(HashFormat::Nix32, true); + return renderPrefixModern(method) + this->hash.to_string(HashFormat::Nix32, true); } /** @@ -130,17 +168,17 @@ static std::pair parseContentAddressMethodP // No parsing of the ingestion method, "text" only support flat. HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { - TextIngestionMethod {}, + ContentAddressMethod::Raw::Text, std::move(hashAlgo), }; } else if (prefix == "fixed") { // Parse method - auto method = FileIngestionMethod::Flat; + auto method = ContentAddressMethod::Raw::Flat; if (splitPrefix(rest, "r:")) - method = FileIngestionMethod::Recursive; + method = ContentAddressMethod::Raw::NixArchive; else if (splitPrefix(rest, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); - method = FileIngestionMethod::Git; + method = ContentAddressMethod::Raw::Git; } HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { @@ -201,57 +239,58 @@ size_t StoreReferences::size() const ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept { - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { - return TextInfo { - .hash = ca.hash, - .references = {}, - }; - }, - [&](const FileIngestionMethod & method) -> ContentAddressWithReferences { - return FixedOutputInfo { - .method = method, - .hash = ca.hash, - .references = {}, - }; - }, - }, ca.method.raw); + switch (ca.method.raw) { + case ContentAddressMethod::Raw::Text: + return TextInfo { + .hash = ca.hash, + .references = {}, + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = ca.method.getFileIngestionMethod(), + .hash = ca.hash, + .references = {}, + }; + default: + assert(false); + } } ContentAddressWithReferences ContentAddressWithReferences::fromParts( ContentAddressMethod method, Hash hash, StoreReferences refs) { - return std::visit(overloaded { - [&](TextIngestionMethod _) -> ContentAddressWithReferences { - if (refs.self) - throw Error("self-reference not allowed with text hashing"); - return ContentAddressWithReferences { - TextInfo { - .hash = std::move(hash), - .references = std::move(refs.others), - } - }; - }, - [&](FileIngestionMethod m2) -> ContentAddressWithReferences { - return ContentAddressWithReferences { - FixedOutputInfo { - .method = m2, - .hash = std::move(hash), - .references = std::move(refs), - } - }; - }, - }, method.raw); + switch (method.raw) { + case ContentAddressMethod::Raw::Text: + if (refs.self) + throw Error("self-reference not allowed with text hashing"); + return TextInfo { + .hash = std::move(hash), + .references = std::move(refs.others), + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = method.getFileIngestionMethod(), + .hash = std::move(hash), + .references = std::move(refs), + }; + default: + assert(false); + } } ContentAddressMethod ContentAddressWithReferences::getMethod() const { return std::visit(overloaded { [](const TextInfo & th) -> ContentAddressMethod { - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.method; + return fileIngestionMethodToContentAddressMethod( + fsh.method); }, }, raw); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 5925f8e01..6cc3b7cd9 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -5,7 +5,6 @@ #include "hash.hh" #include "path.hh" #include "file-content-address.hh" -#include "comparator.hh" #include "variant-wrapper.hh" namespace nix { @@ -14,24 +13,6 @@ namespace nix { * Content addressing method */ -/* We only have one way to hash text with references, so this is a single-value - type, mainly useful with std::variant. -*/ - -/** - * The single way we can serialize "text" file system objects. - * - * Somewhat obscure, used by \ref Derivation derivations and - * `builtins.toFile` currently. - * - * TextIngestionMethod is identical to FileIngestionMethod::Fixed except that - * the former may not have self-references and is tagged `text:${algo}:${hash}` - * rather than `fixed:${algo}:${hash}`. The contents of the store path are - * ingested and hashed identically, aside from the slightly different tag and - * restriction on self-references. - */ -struct TextIngestionMethod : std::monostate { }; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. @@ -48,14 +29,51 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m); */ struct ContentAddressMethod { - typedef std::variant< - TextIngestionMethod, - FileIngestionMethod - > Raw; + enum struct Raw { + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + Flat, + + /** + * Calculate a store path using the + * `FileIngestionMethod::NixArchive` hash of the file system + * objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + NixArchive, + + /** + * Calculate a store path using the `FileIngestionMethod::Git` + * hash of the file system objects, and references. + * + * Part of `ExperimentalFeature::GitHashing`. + * + * See `store-object/content-address.md#method-git` in the + * manual. + */ + Git, + + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references, but in a + * different way than `ContentAddressMethod::Raw::Flat`. + * + * See `store-object/content-address.md#method-text` in the + * manual. + */ + Text, + }; Raw raw; - GENERATE_CMP(ContentAddressMethod, me->raw); + auto operator <=>(const ContentAddressMethod &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); @@ -141,7 +159,7 @@ struct ContentAddress */ Hash hash; - GENERATE_CMP(ContentAddress, me->method, me->hash); + auto operator <=>(const ContentAddress &) const = default; /** * Compute the content-addressability assertion @@ -200,7 +218,7 @@ struct StoreReferences */ size_t size() const; - GENERATE_CMP(StoreReferences, me->self, me->others); + auto operator <=>(const StoreReferences &) const = default; }; // This matches the additional info that we need for makeTextPath @@ -217,7 +235,7 @@ struct TextInfo */ StorePathSet references; - GENERATE_CMP(TextInfo, me->hash, me->references); + auto operator <=>(const TextInfo &) const = default; }; struct FixedOutputInfo @@ -237,7 +255,7 @@ struct FixedOutputInfo */ StoreReferences references; - GENERATE_CMP(FixedOutputInfo, me->hash, me->references); + auto operator <=>(const FixedOutputInfo &) const = default; }; /** @@ -254,7 +272,7 @@ struct ContentAddressWithReferences Raw raw; - GENERATE_CMP(ContentAddressWithReferences, me->raw); + auto operator <=>(const ContentAddressWithReferences &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index fe60cb918..40163a621 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -415,12 +415,12 @@ static void performOp(TunnelLogger * logger, ref store, case FileIngestionMethod::Flat: dumpMethod = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - dumpMethod = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + dumpMethod = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - dumpMethod = FileSerialisationMethod::Recursive; + dumpMethod = FileSerialisationMethod::NixArchive; break; default: assert(false); @@ -435,19 +435,21 @@ static void performOp(TunnelLogger * logger, ref store, } else { HashAlgorithm hashAlgo; std::string baseName; - FileIngestionMethod method; + ContentAddressMethod method; { bool fixed; uint8_t recursive; std::string hashAlgoRaw; from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; - if (recursive > (uint8_t) FileIngestionMethod::Recursive) + if (recursive > true) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); - method = FileIngestionMethod { recursive }; + method = recursive + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Compatibility hack. */ if (!fixed) { hashAlgoRaw = "sha256"; - method = FileIngestionMethod::Recursive; + method = ContentAddressMethod::Raw::NixArchive; } hashAlgo = parseHashAlgo(hashAlgoRaw); } @@ -468,7 +470,7 @@ static void performOp(TunnelLogger * logger, ref store, }); logger->startWork(); auto path = store->addToStoreFromDump( - *dumpSource, baseName, FileSerialisationMethod::Recursive, method, hashAlgo); + *dumpSource, baseName, FileSerialisationMethod::NixArchive, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -500,7 +502,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto path = ({ StringSource source { s }; - store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair); + store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, NoRepair); }); logger->stopWork(); to << store->printStorePath(path); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 869880112..6dfcc408c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -150,7 +150,7 @@ StorePath writeDerivation(Store & store, }) : ({ StringSource s { contents }; - store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair); + store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, repair); }); } @@ -274,7 +274,7 @@ static DerivationOutput parseDerivationOutput( { if (hashAlgoStr != "") { ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); - if (method == TextIngestionMethod {}) + if (method == ContentAddressMethod::Raw::Text) xpSettings.require(Xp::DynamicDerivations); const auto hashAlgo = parseHashAlgo(hashAlgoStr); if (hashS == "impure") { @@ -1249,7 +1249,7 @@ DerivationOutput DerivationOutput::fromJSON( auto methodAlgo = [&]() -> std::pair { auto & method_ = getString(valueAt(json, "method")); ContentAddressMethod method = ContentAddressMethod::parse(method_); - if (method == TextIngestionMethod {}) + if (method == ContentAddressMethod::Raw::Text) xpSettings.require(Xp::DynamicDerivations); auto & hashAlgo_ = getString(valueAt(json, "hashAlgo")); diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 0d5d03091..17ebaace6 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -64,8 +64,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index a54ebdcf3..cbbb0fe7a 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,5 +1,6 @@ #include "filetransfer.hh" #include "globals.hh" +#include "config-global.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 88f899dbf..7e1d7ea6d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,4 +1,5 @@ #include "globals.hh" +#include "config-global.hh" #include "current-process.hh" #include "archive.hh" #include "args.hh" @@ -123,12 +124,12 @@ Settings::Settings() }; } -void loadConfFile() +void loadConfFile(AbstractConfig & config) { auto applyConfigFile = [&](const Path & path) { try { std::string contents = readFile(path); - globalConfig.applyConfig(contents, path); + config.applyConfig(contents, path); } catch (SystemError &) { } }; @@ -136,7 +137,7 @@ void loadConfFile() /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - globalConfig.resetOverridden(); + config.resetOverridden(); auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { @@ -145,7 +146,7 @@ void loadConfFile() auto nixConfEnv = getEnv("NIX_CONFIG"); if (nixConfEnv.has_value()) { - globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); + config.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); } } @@ -437,7 +438,7 @@ void initLibStore(bool loadConfig) { initLibUtil(); if (loadConfig) - loadConfFile(); + loadConfFile(globalConfig); preloadNSS(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 843e77bcf..dfe25f317 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -228,7 +228,7 @@ public: While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. This value is available in the Nix language as - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) if the [`eval-system`](#conf-eval-system) configuration option is set as the empty string. @@ -1284,7 +1284,13 @@ extern Settings settings; */ void initPlugins(); -void loadConfFile(); +/** + * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the + * given configuration object. + * + * Usually called with `globalConfig`. + */ +void loadConfFile(AbstractConfig & config); // Used by the Settings constructor std::vector getUserConfigFiles(); diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index b683ed580..db49188ec 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -76,8 +76,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/linux/meson.build b/src/libstore/linux/meson.build new file mode 100644 index 000000000..0c494b5d6 --- /dev/null +++ b/src/libstore/linux/meson.build @@ -0,0 +1,10 @@ +sources += files( + 'personality.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'fchmodat2-compat.hh', + 'personality.hh', +) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b44879cc9..2b4e01eb3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -233,7 +233,7 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); - else { + else if (!readOnly) { struct stat st; if (stat(realStoreDir.get().c_str(), &st)) throw SysError("getting attributes of path '%1%'", realStoreDir); @@ -1155,7 +1155,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto fim = specified.method.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { HashModuloSink caSink { specified.hash.algo, @@ -1253,7 +1253,7 @@ StorePath LocalStore::addToStoreFromDump( std::filesystem::path tempDir; AutoCloseFD tempDirFd; - bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod; + bool methodsMatch = static_cast(dumpMethod) == hashMethod.getFileIngestionMethod(); /* If the methods don't match, our streaming hash of the dump is the wrong sort, and we need to rehash. */ @@ -1314,7 +1314,7 @@ StorePath LocalStore::addToStoreFromDump( auto fim = hashMethod.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: restorePath(realPath, dumpSource, (FileSerialisationMethod) fim); break; case FileIngestionMethod::Git: @@ -1330,7 +1330,7 @@ StorePath LocalStore::addToStoreFromDump( /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ auto narHash = std::pair { dumpHash, size }; - if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) { + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) { HashSink narSink { HashAlgorithm::SHA256 }; dumpPath(realPath, narSink); narHash = narSink.finish(); @@ -1423,7 +1423,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PosixSourceAccessor accessor; std::string hash = hashPath( PosixSourceAccessor::createAtRoot(link.path()), - FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); + FileIngestionMethod::NixArchive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); if (hash != name.string()) { printError("link '%s' was modified! expected hash '%s', got '%s'", link.path(), name, hash); diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 170fe67b9..a3130d7cc 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,7 +52,7 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = narModuloHash, .references = std::move(refs), }, diff --git a/src/libstore/meson.build b/src/libstore/meson.build new file mode 100644 index 000000000..5324b2a1f --- /dev/null +++ b/src/libstore/meson.build @@ -0,0 +1,404 @@ +project('nix-store', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +configdata.set_quoted('SYSTEM', host_machine.system()) + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), +] +subdir('build-utils-meson/subprojects') + +run_command('ln', '-s', + meson.project_build_root() / '__nothing_link_target', + meson.project_build_root() / '__nothing_symlink', + check : true, +) +can_link_symlink = run_command('ln', + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : false, +).returncode() == 0 +run_command('rm', '-f', + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : true, +) +summary('can hardlink to symlink', can_link_symlink, bool_yn : true) +configdata.set('CAN_LINK_SYMLINK', can_link_symlink.to_int()) + +# Check for each of these functions, and create a define like `#define HAVE_LCHOWN 1`. +# +# Only need to do functions that deps (like `libnixutil`) didn't already +# check for. +check_funcs = [ + # Optionally used for canonicalising files from the build + 'lchown', +] +foreach funcspec : check_funcs + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) +endforeach + +has_acl_support = cxx.has_header('sys/xattr.h') \ + and cxx.has_function('llistxattr') \ + and cxx.has_function('lremovexattr') +configdata.set('HAVE_ACL_SUPPORT', has_acl_support.to_int()) + +subdir('build-utils-meson/threads') + +boost = dependency( + 'boost', + modules : ['container'], +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +curl = dependency('libcurl', 'curl') +deps_private += curl + +# seccomp only makes sense on Linux +is_linux = host_machine.system() == 'linux' +seccomp_required = get_option('seccomp-sandboxing') +if not is_linux and seccomp_required.enabled() + warning('Force-enabling seccomp on non-Linux does not make sense') +endif +seccomp = dependency('libseccomp', 'seccomp', required : seccomp_required, version : '>=2.5.5') +if is_linux and not seccomp.found() + warning('Sandbox security is reduced because libseccomp has not been found! Please provide libseccomp if it supports your CPU architecture.') +endif +configdata.set('HAVE_SECCOMP', seccomp.found().to_int()) +deps_private += seccomp + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') +deps_private += sqlite + +subdir('build-utils-meson/generate-header') + +generated_headers = [] +foreach header : [ + 'schema.sql', + 'ca-specific-schema.sql', +] + generated_headers += gen_header.process(header) +endforeach + +busybox = find_program(get_option('sandbox-shell'), required : false) + +if get_option('embedded-sandbox-shell') + # This one goes in config.h + # The path to busybox is passed as a -D flag when compiling this_library. + # This solution is inherited from the old make buildsystem + # TODO: do this differently? + configdata.set('HAVE_EMBEDDED_SANDBOX_SHELL', 1) + hexdump = find_program('hexdump', native : true) + embedded_sandbox_shell_gen = custom_target( + 'embedded-sandbox-shell.gen.hh', + command : [ + hexdump, + '-v', + '-e', + '1/1 "0x%x," "\n"' + ], + input : busybox.full_path(), + output : 'embedded-sandbox-shell.gen.hh', + capture : true, + feed : true, + ) + generated_headers += embedded_sandbox_shell_gen +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-store.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'binary-cache-store.cc', + 'build-result.cc', + 'build/derivation-goal.cc', + 'build/drv-output-substitution-goal.cc', + 'build/entry-points.cc', + 'build/goal.cc', + 'build/substitution-goal.cc', + 'build/worker.cc', + 'builtins/buildenv.cc', + 'builtins/fetchurl.cc', + 'builtins/unpack-channel.cc', + 'common-protocol.cc', + 'content-address.cc', + 'daemon.cc', + 'derivations.cc', + 'derived-path-map.cc', + 'derived-path.cc', + 'downstream-placeholder.cc', + 'dummy-store.cc', + 'export-import.cc', + 'filetransfer.cc', + 'gc.cc', + 'globals.cc', + 'http-binary-cache-store.cc', + 'indirect-root-store.cc', + 'keys.cc', + 'legacy-ssh-store.cc', + 'local-binary-cache-store.cc', + 'local-fs-store.cc', + 'local-overlay-store.cc', + 'local-store.cc', + 'log-store.cc', + 'machines.cc', + 'make-content-addressed.cc', + 'misc.cc', + 'names.cc', + 'nar-accessor.cc', + 'nar-info-disk-cache.cc', + 'nar-info.cc', + 'optimise-store.cc', + 'outputs-spec.cc', + 'parsed-derivations.cc', + 'path-info.cc', + 'path-references.cc', + 'path-with-outputs.cc', + 'path.cc', + 'pathlocks.cc', + 'posix-fs-canonicalise.cc', + 'profiles.cc', + 'realisation.cc', + 'remote-fs-accessor.cc', + 'remote-store.cc', + 's3-binary-cache-store.cc', + 'serve-protocol-connection.cc', + 'serve-protocol.cc', + 'sqlite.cc', + 'ssh-store-config.cc', + 'ssh-store.cc', + 'ssh.cc', + 'store-api.cc', + 'store-reference.cc', + 'uds-remote-store.cc', + 'worker-protocol-connection.cc', + 'worker-protocol.cc', +) + +include_dirs = [ + include_directories('.'), + include_directories('build'), +] + +headers = [config_h] + files( + 'binary-cache-store.hh', + 'build-result.hh', + 'build/derivation-goal.hh', + 'build/drv-output-substitution-goal.hh', + 'build/goal.hh', + 'build/substitution-goal.hh', + 'build/worker.hh', + 'builtins.hh', + 'builtins/buildenv.hh', + 'common-protocol-impl.hh', + 'common-protocol.hh', + 'content-address.hh', + 'daemon.hh', + 'derivations.hh', + 'derived-path-map.hh', + 'derived-path.hh', + 'downstream-placeholder.hh', + 'filetransfer.hh', + 'gc-store.hh', + 'globals.hh', + 'indirect-root-store.hh', + 'keys.hh', + 'legacy-ssh-store.hh', + 'length-prefixed-protocol-helper.hh', + 'local-fs-store.hh', + 'local-overlay-store.hh', + 'local-store.hh', + 'log-store.hh', + 'machines.hh', + 'make-content-addressed.hh', + 'names.hh', + 'nar-accessor.hh', + 'nar-info-disk-cache.hh', + 'nar-info.hh', + 'outputs-spec.hh', + 'parsed-derivations.hh', + 'path-info.hh', + 'path-references.hh', + 'path-regex.hh', + 'path-with-outputs.hh', + 'path.hh', + 'pathlocks.hh', + 'posix-fs-canonicalise.hh', + 'profiles.hh', + 'realisation.hh', + 'remote-fs-accessor.hh', + 'remote-store-connection.hh', + 'remote-store.hh', + 's3-binary-cache-store.hh', + 's3.hh', + 'serve-protocol-connection.hh', + 'serve-protocol-impl.hh', + 'serve-protocol.hh', + 'sqlite.hh', + 'ssh-store-config.hh', + 'ssh.hh', + 'store-api.hh', + 'store-cast.hh', + 'store-dir-config.hh', + 'store-reference.hh', + 'uds-remote-store.hh', + 'worker-protocol-connection.hh', + 'worker-protocol-impl.hh', + 'worker-protocol.hh', +) + +if host_machine.system() == 'linux' + subdir('linux') +endif + +if host_machine.system() == 'windows' + subdir('windows') +else + subdir('unix') +endif + +fs = import('fs') + +prefix = get_option('prefix') +# For each of these paths, assume that it is relative to the prefix unless +# it is already an absolute path (which is the default for store-dir, state-dir, and log-dir). +path_opts = [ + # Meson built-ins. + 'datadir', + 'bindir', + 'mandir', + 'libdir', + 'includedir', + 'libexecdir', + # Homecooked Nix directories. + 'store-dir', + 'state-dir', + 'log-dir', +] +# For your grepping pleasure, this loop sets the following variables that aren't mentioned +# literally above: +# store_dir +# state_dir +# log_dir +# profile_dir +foreach optname : path_opts + varname = optname.replace('-', '_') + path = get_option(optname) + if fs.is_absolute(path) + set_variable(varname, path) + else + set_variable(varname, prefix / path) + endif +endforeach + +# sysconfdir doesn't get anything installed to directly, and is only used to +# tell Nix where to look for nix.conf, so it doesn't get appended to prefix. +sysconfdir = get_option('sysconfdir') +if not fs.is_absolute(sysconfdir) + sysconfdir = '/' / sysconfdir +endif + +lsof = find_program('lsof', required : false) + +# Aside from prefix itself, each of these was made into an absolute path +# by joining it with prefix, unless it was already an absolute path +# (which is the default for store-dir, state-dir, and log-dir). +cpp_str_defines = { + 'NIX_PREFIX': prefix, + 'NIX_STORE_DIR': store_dir, + 'NIX_DATA_DIR': datadir, + 'NIX_STATE_DIR': state_dir / 'nix', + 'NIX_LOG_DIR': log_dir, + 'NIX_CONF_DIR': sysconfdir / 'nix', + 'NIX_BIN_DIR': bindir, + 'NIX_MAN_DIR': mandir, +} + +if lsof.found() + lsof_path = lsof.full_path() +else + # Just look up on the PATH + lsof_path = 'lsof' +endif +cpp_str_defines += { + 'LSOF': lsof_path +} + +if get_option('embedded-sandbox-shell') + cpp_str_defines += { + 'SANDBOX_SHELL': '__embedded_sandbox_shell__' + } +elif busybox.found() + cpp_str_defines += { + 'SANDBOX_SHELL': busybox.full_path() + } +endif + +cpp_args = [] + +foreach name, value : cpp_str_defines + cpp_args += [ + '-D' + name + '=' + '"' + value + '"' + ] +endforeach + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixstore', + generated_headers, + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + cpp_args : cpp_args, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libstore/meson.options b/src/libstore/meson.options new file mode 100644 index 000000000..723a8e020 --- /dev/null +++ b/src/libstore/meson.options @@ -0,0 +1,25 @@ +# vim: filetype=meson + +option('embedded-sandbox-shell', type : 'boolean', value : false, + description : 'include the sandbox shell in the Nix binary', +) + +option('seccomp-sandboxing', type : 'feature', + description : 'build support for seccomp sandboxing (recommended unless your arch doesn\'t support libseccomp, only relevant on Linux)', +) + +option('sandbox-shell', type : 'string', value : 'busybox', + description : 'path to a statically-linked shell to use as /bin/sh in sandboxes (usually busybox)', +) + +option('store-dir', type : 'string', value : '/nix/store', + description : 'path of the Nix store', +) + +option('state-dir', type : 'string', value : '/nix/var', + description : 'path to store state in for Nix', +) + +option('log-dir', type : 'string', value : '/nix/var/log/nix', + description : 'path to store logs in for Nix', +) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index cecf8148f..b1079b027 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -71,9 +71,11 @@ struct NarAccessor : public SourceAccessor : acc(acc), source(source) { } - NarMember & createMember(const Path & path, NarMember member) + NarMember & createMember(const CanonPath & path, NarMember member) { - size_t level = std::count(path.begin(), path.end(), '/'); + size_t level = 0; + for (auto _ : path) ++level; + while (parents.size() > level) parents.pop(); if (parents.empty()) { @@ -83,14 +85,14 @@ struct NarAccessor : public SourceAccessor } else { if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); - auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); + auto result = parents.top()->children.emplace(*path.baseName(), std::move(member)); auto & ref = result.first->second; parents.push(&ref); return ref; } } - void createDirectory(const Path & path) override + void createDirectory(const CanonPath & path) override { createMember(path, NarMember{ .stat = { .type = Type::tDirectory, @@ -100,7 +102,7 @@ struct NarAccessor : public SourceAccessor } }); } - void createRegularFile(const Path & path, std::function func) override + void createRegularFile(const CanonPath & path, std::function func) override { auto & nm = createMember(path, NarMember{ .stat = { .type = Type::tRegular, @@ -112,7 +114,7 @@ struct NarAccessor : public SourceAccessor func(nmc); } - void createSymlink(const Path & path, const std::string & target) override + void createSymlink(const CanonPath & path, const std::string & target) override { createMember(path, NarMember{ diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 2477cf0c0..9d903f218 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -98,9 +98,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for - special files within .app dirs. *.app/Contents/PkgInfo and - *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See - https://github.com/NixOS/nix/issues/1443 for more discussion. */ + special files within .app dirs. Known affected paths include + *.app/Contents/{PkgInfo,Resources/\*.lproj,_CodeSignature} and .DS_Store. + See https://github.com/NixOS/nix/issues/1443 and + https://github.com/NixOS/nix/pull/2230 for more discussion. */ if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { @@ -150,7 +151,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Hash hash = ({ hashPath( {make_ref(), CanonPath(path)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; }); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); @@ -164,7 +165,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, || (repair && hash != ({ hashPath( PosixSourceAccessor::createAtRoot(linkPath), - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; }))) { // XXX: Consider overwriting linkPath with our valid version. diff --git a/src/libstore/package.nix b/src/libstore/package.nix new file mode 100644 index 000000000..0a2ace91e --- /dev/null +++ b/src/libstore/package.nix @@ -0,0 +1,116 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, unixtools + +, nix-util +, boost +, curl +, aws-sdk-cpp +, libseccomp +, nlohmann_json +, sqlite + +, busybox-sandbox-shell ? null + +# Configuration Options + +, version + +, embeddedSandboxShell ? stdenv.hostPlatform.isStatic +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + ./linux/meson.build + ./unix/meson.build + ./windows/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "sb") ./.) + (fileset.fileFilter (file: file.hasExt "md") ./.) + (fileset.fileFilter (file: file.hasExt "sql") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ] ++ lib.optional embeddedSandboxShell unixtools.hexdump; + + buildInputs = [ + boost + curl + sqlite + ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) + (aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }) + ; + + propagatedBuildInputs = [ + nix-util + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux) + (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell) + ] ++ lib.optionals stdenv.hostPlatform.isLinux [ + (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox") + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ddd7f50d9..5c27182b7 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -48,15 +48,21 @@ std::optional ValidPathInfo::contentAddressWithRef if (! ca) return std::nullopt; - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { + switch (ca->method.raw) { + case ContentAddressMethod::Raw::Text: + { assert(references.count(path) == 0); return TextInfo { .hash = ca->hash, .references = references, }; - }, - [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { + } + + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { @@ -64,15 +70,15 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - .method = m2, + .method = ca->method.getFileIngestionMethod(), .hash = ca->hash, .references = { .others = std::move(refs), .self = hasSelfReference, }, }; - }, - }, ca->method.raw); + } + } } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -127,22 +133,18 @@ ValidPathInfo::ValidPathInfo( : UnkeyedValidPathInfo(narHash) , path(store.makeFixedOutputPathFromCA(name, ca)) { + this->ca = ContentAddress { + .method = ca.getMethod(), + .hash = ca.getHash(), + }; std::visit(overloaded { [this](TextInfo && ti) { this->references = std::move(ti.references); - this->ca = ContentAddress { - .method = TextIngestionMethod {}, - .hash = std::move(ti.hash), - }; }, [this](FixedOutputInfo && foi) { this->references = std::move(foi.references.others); if (foi.references.self) this->references.insert(path); - this->ca = ContentAddress { - .method = std::move(foi.method), - .hash = std::move(foi.hash), - }; }, }, std::move(ca).raw); } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 8d9726722..3e9d05477 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,25 +2,24 @@ namespace nix { -static void checkName(std::string_view path, std::string_view name) +void checkName(std::string_view name) { if (name.empty()) - throw BadStorePath("store path '%s' has an empty name", path); + throw BadStorePathName("name must not be empty"); if (name.size() > StorePath::MaxPathLen) - throw BadStorePath("store path '%s' has a name longer than %d characters", - path, StorePath::MaxPathLen); + throw BadStorePathName("name '%s' must be no longer than %d characters", name, StorePath::MaxPathLen); // See nameRegexStr for the definition if (name[0] == '.') { // check against "." and "..", followed by end or dash if (name.size() == 1) - throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + throw BadStorePathName("name '%s' is not valid", name); if (name[1] == '-') - throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, "."); + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, "."); if (name[1] == '.') { if (name.size() == 2) - throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + throw BadStorePathName("name '%s' is not valid", name); if (name[2] == '-') - throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, ".."); + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, ".."); } } for (auto c : name) @@ -28,7 +27,16 @@ static void checkName(std::string_view path, std::string_view name) || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '=')) - throw BadStorePath("store path '%s' contains illegal character '%s'", path, c); + throw BadStorePathName("name '%s' contains illegal character '%s'", name, c); +} + +static void checkPathName(std::string_view path, std::string_view name) +{ + try { + checkName(name); + } catch (BadStorePathName & e) { + throw BadStorePath("path '%s' is not a valid store path: %s", path, Uncolored(e.message())); + } } StorePath::StorePath(std::string_view _baseName) @@ -40,13 +48,13 @@ StorePath::StorePath(std::string_view _baseName) if (c == 'e' || c == 'o' || c == 'u' || c == 't' || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); - checkName(baseName, name()); + checkPathName(baseName, name()); } StorePath::StorePath(const Hash & hash, std::string_view _name) : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name))) { - checkName(baseName, name()); + checkPathName(baseName, name()); } bool StorePath::isDerivation() const noexcept diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 4abbfcd7c..2380dc6a2 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -9,6 +9,13 @@ namespace nix { struct Hash; +/** + * Check whether a name is a valid store path name. + * + * @throws BadStorePathName if the name is invalid. The message is of the format "name %s is not valid, for this specific reason". + */ +void checkName(std::string_view name); + /** * \ref StorePath "Store path" is the fundamental reference type of Nix. * A store paths refers to a Store object. @@ -31,8 +38,10 @@ public: StorePath() = delete; + /** @throws BadStorePath */ StorePath(std::string_view baseName); + /** @throws BadStorePath */ StorePath(const Hash & hash, std::string_view name); std::string_view to_string() const noexcept diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index 8cb13d810..46a78cc86 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -33,19 +33,9 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #ifndef _WIN32 // TODO implement if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); + struct stat st2 = st; + st2.st_mtime = mtimeStore, + setWriteTime(path, st2); } #endif } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d6efc14f9..d749ccd0a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -392,8 +392,9 @@ ref RemoteStore::addCAToStore( else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); - std::visit(overloaded { - [&](const TextIngestionMethod & thm) -> void { + switch (caMethod.raw) { + case ContentAddressMethod::Raw::Text: + { if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", name, printHashAlgo(hashAlgo)); @@ -401,13 +402,19 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddTextToStore << name << s; WorkerProto::write(*this, *conn, references); conn.processStderr(); - }, - [&](const FileIngestionMethod & fim) -> void { + break; + } + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { + auto fim = caMethod.getFileIngestionMethod(); conn->to << WorkerProto::Op::AddToStore << name - << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fim == FileIngestionMethod::Recursive ? 1 : 0) + << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::NixArchive) ? 0 : 1) /* backwards compatibility hack */ + << (fim == FileIngestionMethod::NixArchive ? 1 : 0) << printHashAlgo(hashAlgo); try { @@ -415,7 +422,7 @@ ref RemoteStore::addCAToStore( connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - if (fim == FileIngestionMethod::Recursive) { + if (fim == FileIngestionMethod::NixArchive) { dump.drainInto(conn->to); } else { std::string contents = dump.drain(); @@ -432,9 +439,9 @@ ref RemoteStore::addCAToStore( } catch (EndOfFile & e) { } throw; } - + break; } - }, caMethod.raw); + } auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); @@ -457,12 +464,12 @@ StorePath RemoteStore::addToStoreFromDump( case FileIngestionMethod::Flat: fsm = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - fsm = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - fsm = FileSerialisationMethod::Recursive; + fsm = FileSerialisationMethod::NixArchive; break; default: assert(false); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index d630adc08..4e1896268 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -87,8 +87,8 @@ public: StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index e9850dce6..27013c0f1 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -132,6 +132,7 @@ ref S3Helper::makeConfig( { initAWS(); auto res = make_ref(); + res->allowSystemProxy = true; res->region = region; if (!scheme.empty()) { res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c67ccd7d4..05c4e1c5e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -19,6 +19,7 @@ #include "signals.hh" #include "users.hh" +#include #include using json = nlohmann::json; @@ -121,7 +122,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) throw Error("Git file ingestion must use SHA-1 hash"); - if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { + if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::NixArchive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { if (!info.references.empty()) { @@ -199,12 +200,12 @@ StorePath Store::addToStore( case FileIngestionMethod::Flat: fsm = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - fsm = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - fsm = FileSerialisationMethod::Recursive; + fsm = FileSerialisationMethod::NixArchive; break; } auto source = sinkToSource([&](Sink & sink) { @@ -355,7 +356,7 @@ ValidPathInfo Store::addToStoreSlow( RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256 + auto & narSink = method == ContentAddressMethod::Raw::NixArchive && hashAlgo != HashAlgorithm::SHA256 ? static_cast(unusualHashTee) : narHashSink; @@ -383,9 +384,9 @@ ValidPathInfo Store::addToStoreSlow( finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256 + auto hash = method == ContentAddressMethod::Raw::NixArchive && hashAlgo == HashAlgorithm::SHA256 ? narHash - : method == FileIngestionMethod::Git + : method == ContentAddressMethod::Raw::Git ? git::dumpHash(hashAlgo, srcPath).hash : caHashSink.finish().first; @@ -1303,7 +1304,7 @@ ref openStore(StoreReference && storeURI) if (!pathExists(chrootStore)) { try { createDirs(chrootStore); - } catch (Error & e) { + } catch (SystemError & e) { return std::make_shared(params); } warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 15712458c..a5effb4c1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -441,7 +441,7 @@ public: virtual StorePath addToStore( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), PathFilter & filter = defaultPathFilter, @@ -455,7 +455,7 @@ public: ValidPathInfo addToStoreSlow( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), std::optional expectedCAHash = {}); @@ -470,7 +470,7 @@ public: * * @param dumpMethod What serialisation format is `dump`, i.e. how * to deserialize it. Must either match hashMethod or be - * `FileSerialisationMethod::Recursive`. + * `FileSerialisationMethod::NixArchive`. * * @param hashMethod How content addressing? Need not match be the * same as `dumpMethod`. @@ -480,8 +480,8 @@ public: virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) = 0; diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 643f8854d..64c0dd8b7 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -16,6 +16,7 @@ namespace nix { struct SourcePath; MakeError(BadStorePath, Error); +MakeError(BadStorePathName, BadStorePath); struct StoreDirConfig : public Config { @@ -97,7 +98,7 @@ struct StoreDirConfig : public Config std::pair computeStorePath( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = {}, PathFilter & filter = defaultPathFilter) const; diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 5d045ec3d..dfc208798 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -1,4 +1,5 @@ #include "globals.hh" +#include "config-global.hh" #include "hook-instance.hh" #include "file-system.hh" #include "child.hh" diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 16095cf5d..d5a3e0034 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -503,8 +503,24 @@ void LocalDerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); + topTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); +#if __APPLE__ + if (false) { +#else + if (useChroot) { +#endif + /* If sandboxing is enabled, put the actual TMPDIR underneath + an inaccessible root-owned directory, to prevent outside + access. + On macOS, we don't use an actual chroot, so this isn't + possible. Any mitigation along these lines would have to be + done directly in the sandbox profile. */ + tmpDir = topTmpDir + "/build"; + createDir(tmpDir, 0700); + } else { + tmpDir = topTmpDir; + } chownToBuilder(tmpDir); for (auto & [outputName, status] : initialOutputs) { @@ -672,15 +688,19 @@ void LocalDerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store so that the build outputs can be moved efficiently from the chroot to their final location. */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); + chrootParentDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; + deletePath(chrootParentDir); /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); + autoDelChroot = std::make_shared(chrootParentDir); - printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootRootDir); + printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir); + + if (mkdir(chrootParentDir.c_str(), 0700) == -1) + throw SysError("cannot create '%s'", chrootRootDir); + + chrootRootDir = chrootParentDir + "/root"; - // FIXME: make this 0700 if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); @@ -1500,7 +1520,7 @@ void LocalDerivationGoal::startDaemon() throw SysError("accepting connection"); } - closeOnExec(remote.get()); + unix::closeOnExec(remote.get()); debug("received daemon connection"); @@ -1961,7 +1981,7 @@ void LocalDerivationGoal::runChild() throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + unix::closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); #if __linux__ linux::setPersonality(drv->platform); @@ -2489,7 +2509,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto fim = outputHash.method.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; auto fim = outputHash.method.getFileIngestionMethod(); @@ -2531,7 +2551,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() { HashResult narHashAndSize = hashPath( {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; } @@ -2554,7 +2574,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); HashResult narHashAndSize = hashPath( {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); @@ -2904,6 +2924,24 @@ void LocalDerivationGoal::checkOutputs(const std::mapgetStructuredAttrs()) { + if (get(*structuredAttrs, "allowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "allowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxClosureSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); + } if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { if (auto output = get(*outputChecks, outputName)) { Checks checks; @@ -2952,7 +2990,7 @@ void LocalDerivationGoal::checkOutputs(const std::mapisBuiltin()) { @@ -2960,7 +2998,8 @@ void LocalDerivationGoal::deleteTmpDir(bool force) chmod(tmpDir.c_str(), 0755); } else - deletePath(tmpDir); + deletePath(topTmpDir); + topTmpDir = ""; tmpDir = ""; } } diff --git a/src/libstore/unix/build/local-derivation-goal.hh b/src/libstore/unix/build/local-derivation-goal.hh index f25cb9424..4bcf5c9d4 100644 --- a/src/libstore/unix/build/local-derivation-goal.hh +++ b/src/libstore/unix/build/local-derivation-goal.hh @@ -27,10 +27,16 @@ struct LocalDerivationGoal : public DerivationGoal std::optional cgroup; /** - * The temporary directory. + * The temporary directory used for the build. */ Path tmpDir; + /** + * The top-level temporary directory. `tmpDir` is either equal to + * or a child of this directory. + */ + Path topTmpDir; + /** * The path of the temporary directory in the sandbox. */ @@ -65,6 +71,16 @@ struct LocalDerivationGoal : public DerivationGoal */ bool useChroot = false; + /** + * The parent directory of `chrootRootDir`. It has permission 700 + * and is owned by root to ensure other users cannot mess with + * `chrootRootDir`. + */ + Path chrootParentDir; + + /** + * The root of the chroot environment. + */ Path chrootRootDir; /** diff --git a/src/libstore/unix/build/sandbox-defaults.sb b/src/libstore/unix/build/sandbox-defaults.sb index 2ad5fb616..6da01b735 100644 --- a/src/libstore/unix/build/sandbox-defaults.sb +++ b/src/libstore/unix/build/sandbox-defaults.sb @@ -17,6 +17,9 @@ R""( ; Allow POSIX semaphores and shared memory. (allow ipc-posix*) +; Allow SYSV semaphores and shared memory. +(allow ipc-sysv*) + ; Allow socket creation. (allow system-socket) diff --git a/src/libstore/unix/meson.build b/src/libstore/unix/meson.build new file mode 100644 index 000000000..d9d190131 --- /dev/null +++ b/src/libstore/unix/meson.build @@ -0,0 +1,19 @@ +sources += files( + 'build/child.cc', + 'build/hook-instance.cc', + 'build/local-derivation-goal.cc', + 'pathlocks.cc', + 'user-lock.cc', +) + +include_dirs += include_directories( + '.', + 'build', +) + +headers += files( + 'build/child.hh', + 'build/hook-instance.hh', + 'build/local-derivation-goal.hh', + 'user-lock.hh', +) diff --git a/src/libstore/windows/meson.build b/src/libstore/windows/meson.build new file mode 100644 index 000000000..b81c5b2af --- /dev/null +++ b/src/libstore/windows/meson.build @@ -0,0 +1,11 @@ +sources += files( + 'pathlocks.cc', +) + +include_dirs += include_directories( + '.', + #'build', +) + +headers += files( +) diff --git a/src/libutil-c/.version b/src/libutil-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil-c/build-utils-meson b/src/libutil-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libutil-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build new file mode 100644 index 000000000..3f0d96282 --- /dev/null +++ b/src/libutil-c/meson.build @@ -0,0 +1,79 @@ +project('nix-util-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-util.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_util.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_util.h', +) + +# TODO don't install this once tests don't use it. +headers += files('nix_api_util_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixutilc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libutil-c/meson.options b/src/libutil-c/meson.options new file mode 100644 index 000000000..04422feaf --- /dev/null +++ b/src/libutil-c/meson.options @@ -0,0 +1 @@ +# vim: filetype=meson diff --git a/src/libutil-c/nix-util-c.pc.in b/src/libutil-c/nix-util-c.pc.in new file mode 100644 index 000000000..0ccae3f8a --- /dev/null +++ b/src/libutil-c/nix-util-c.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix libutil C API +Description: Common functions for the Nix C API, such as error handling +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixutil +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 0a9b49345..4f65a4c12 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -1,5 +1,5 @@ #include "nix_api_util.h" -#include "config.hh" +#include "config-global.hh" #include "error.hh" #include "nix_api_util_internal.h" #include "util.hh" diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix new file mode 100644 index 000000000..53451998d --- /dev/null +++ b/src/libutil-c/package.nix @@ -0,0 +1,77 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libutil/.version b/src/libutil/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index d20936de4..e2ebcda0c 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -6,7 +6,7 @@ #include // for strcasecmp #include "archive.hh" -#include "config.hh" +#include "config-global.hh" #include "posix-source-accessor.hh" #include "source-path.hh" #include "file-system.hh" @@ -165,7 +165,7 @@ struct CaseInsensitiveCompare }; -static void parse(FileSystemObjectSink & sink, Source & source, const Path & path) +static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath & path) { std::string s; @@ -246,7 +246,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const Path & pat } } else if (s == "node") { if (name.empty()) throw badArchive("entry name missing"); - parse(sink, source, path + "/" + name); + parse(sink, source, path / name); } else throw badArchive("unknown field " + s); } @@ -290,11 +290,11 @@ void parseDump(FileSystemObjectSink & sink, Source & source) } if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); - parse(sink, source, ""); + parse(sink, source, CanonPath::root); } -void restorePath(const Path & path, Source & source) +void restorePath(const std::filesystem::path & path, Source & source) { RestoreSink sink; sink.dstPath = path; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index bd70072ce..2da8f0cb1 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -75,7 +75,7 @@ void dumpString(std::string_view s, Sink & sink); void parseDump(FileSystemObjectSink & sink, Source & source); -void restorePath(const Path & path, Source & source); +void restorePath(const std::filesystem::path & path, Source & source); /** * Read a NAR from 'source' and write it to 'sink'. diff --git a/src/libutil/build-utils-meson b/src/libutil/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libutil/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libutil/config-global.cc b/src/libutil/config-global.cc new file mode 100644 index 000000000..4c08898a4 --- /dev/null +++ b/src/libutil/config-global.cc @@ -0,0 +1,67 @@ +#include "config-global.hh" + +namespace nix { + +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) + return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overriddenOnly); +} + +void GlobalConfig::resetOverridden() +{ + for (auto & config : *configRegistrations) + config->resetOverridden(); +} + +nlohmann::json GlobalConfig::toJSON() +{ + auto res = nlohmann::json::object(); + for (const auto & config : *configRegistrations) + res.update(config->toJSON()); + return res; +} + +std::string GlobalConfig::toKeyValue() +{ + std::string res; + std::map settings; + globalConfig.getSettings(settings); + for (const auto & s : settings) + res += fmt("%s = %s\n", s.first, s.second.value); + return res; +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + +ExperimentalFeatureSettings experimentalFeatureSettings; + +static GlobalConfig::Register rSettings(&experimentalFeatureSettings); + +} diff --git a/src/libutil/config-global.hh b/src/libutil/config-global.hh new file mode 100644 index 000000000..2caf51524 --- /dev/null +++ b/src/libutil/config-global.hh @@ -0,0 +1,33 @@ +#pragma once +///@file + +#include "config.hh" + +namespace nix { + +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map & res, bool overriddenOnly = false) override; + + void resetOverridden() override; + + nlohmann::json toJSON() override; + + std::string toKeyValue() override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index efde8591b..907ca7fc1 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -91,7 +91,14 @@ void Config::getSettings(std::map & res, bool overridd } -static void applyConfigInner(const std::string & contents, const std::string & path, std::vector> & parsedContents) { +/** + * Parse configuration in `contents`, and also the configuration files included from there, with their location specified relative to `path`. + * + * `contents` and `path` represent the file that is being parsed. + * The result is only an intermediate list of key-value pairs of strings. + * More parsing according to the settings-specific semantics is being done by `loadConfFile` in `libstore/globals.cc`. +*/ +static void parseConfigFiles(const std::string & contents, const std::string & path, std::vector> & parsedContents) { unsigned int pos = 0; while (pos < contents.size()) { @@ -125,7 +132,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p if (pathExists(p)) { try { std::string includedContents = readFile(p); - applyConfigInner(includedContents, p, parsedContents); + parseConfigFiles(includedContents, p, parsedContents); } catch (SystemError &) { // TODO: Do we actually want to ignore this? Or is it better to fail? } @@ -153,7 +160,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { std::vector> parsedContents; - applyConfigInner(contents, path, parsedContents); + parseConfigFiles(contents, path, parsedContents); // First apply experimental-feature related settings for (const auto & [name, value] : parsedContents) @@ -443,67 +450,6 @@ void OptionalPathSetting::operator =(const std::optional & v) this->assign(v); } -bool GlobalConfig::set(const std::string & name, const std::string & value) -{ - for (auto & config : *configRegistrations) - if (config->set(name, value)) return true; - - unknownSettings.emplace(name, value); - - return false; -} - -void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) -{ - for (auto & config : *configRegistrations) - config->getSettings(res, overriddenOnly); -} - -void GlobalConfig::resetOverridden() -{ - for (auto & config : *configRegistrations) - config->resetOverridden(); -} - -nlohmann::json GlobalConfig::toJSON() -{ - auto res = nlohmann::json::object(); - for (const auto & config : *configRegistrations) - res.update(config->toJSON()); - return res; -} - -std::string GlobalConfig::toKeyValue() -{ - std::string res; - std::map settings; - globalConfig.getSettings(settings); - for (const auto & s : settings) - res += fmt("%s = %s\n", s.first, s.second.value); - return res; -} - -void GlobalConfig::convertToArgs(Args & args, const std::string & category) -{ - for (auto & config : *configRegistrations) - config->convertToArgs(args, category); -} - -GlobalConfig globalConfig; - -GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; - -GlobalConfig::Register::Register(Config * config) -{ - if (!configRegistrations) - configRegistrations = new ConfigRegistrations; - configRegistrations->emplace_back(config); -} - -ExperimentalFeatureSettings experimentalFeatureSettings; - -static GlobalConfig::Register rSettings(&experimentalFeatureSettings); - bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const { auto & f = experimentalFeatures.get(); diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 07322b60d..1952ba1b8 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -375,31 +375,6 @@ public: void operator =(const std::optional & v); }; -struct GlobalConfig : public AbstractConfig -{ - typedef std::vector ConfigRegistrations; - static ConfigRegistrations * configRegistrations; - - bool set(const std::string & name, const std::string & value) override; - - void getSettings(std::map & res, bool overriddenOnly = false) override; - - void resetOverridden() override; - - nlohmann::json toJSON() override; - - std::string toKeyValue() override; - - void convertToArgs(Args & args, const std::string & category) override; - - struct Register - { - Register(Config * config); - }; -}; - -extern GlobalConfig globalConfig; - struct ExperimentalFeatureSettings : Config { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 269000016..4b08a045e 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -155,6 +155,7 @@ public: : err(e) { } + /** The error message without "error: " prefixed to it. */ std::string message() { return err.msg.str(); } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 9b7000f9f..1c080e372 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -66,7 +66,7 @@ constexpr std::array xpFeatureDetails an impure derivation cannot also be [content-addressed](#xp-feature-ca-derivations). - This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). + This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime). )", .trackingUrl = "https://github.com/NixOS/nix/milestone/42", }, diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 8b1e3117a..438dac7da 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -10,7 +10,7 @@ static std::optional parseFileSerialisationMethodOpt(st if (input == "flat") { return FileSerialisationMethod::Flat; } else if (input == "nar") { - return FileSerialisationMethod::Recursive; + return FileSerialisationMethod::NixArchive; } else { return std::nullopt; } @@ -45,7 +45,7 @@ std::string_view renderFileSerialisationMethod(FileSerialisationMethod method) switch (method) { case FileSerialisationMethod::Flat: return "flat"; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: return "nar"; default: assert(false); @@ -57,7 +57,7 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method) { switch (method) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return renderFileSerialisationMethod( static_cast(method)); case FileIngestionMethod::Git: @@ -78,7 +78,7 @@ void dumpPath( case FileSerialisationMethod::Flat: path.readFile(sink); break; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: path.dumpPath(sink, filter); break; } @@ -94,7 +94,7 @@ void restorePath( case FileSerialisationMethod::Flat: writeFile(path, source); break; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: restorePath(path, source); break; } @@ -119,7 +119,7 @@ std::pair> hashPath( { switch (method) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: { + case FileIngestionMethod::NixArchive: { auto res = hashPath(path, (FileSerialisationMethod) method, ht, filter); return {res.first, {res.second}}; } diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh index e216ee4a7..4c7218f19 100644 --- a/src/libutil/file-content-address.hh +++ b/src/libutil/file-content-address.hh @@ -35,14 +35,14 @@ enum struct FileSerialisationMethod : uint8_t { * See `file-system-object/content-address.md#serial-nix-archive` in * the manual. */ - Recursive, + NixArchive, }; /** * Parse a `FileSerialisationMethod` by name. Choice of: * * - `flat`: `FileSerialisationMethod::Flat` - * - `nar`: `FileSerialisationMethod::Recursive` + * - `nar`: `FileSerialisationMethod::NixArchive` * * Opposite of `renderFileSerialisationMethod`. */ @@ -107,16 +107,18 @@ enum struct FileIngestionMethod : uint8_t { Flat, /** - * Hash `FileSerialisationMethod::Recursive` serialisation. + * Hash `FileSerialisationMethod::NixArchive` serialisation. * * See `file-system-object/content-address.md#serial-flat` in the * manual. */ - Recursive, + NixArchive, /** * Git hashing. * + * Part of `ExperimentalFeature::GitHashing`. + * * See `file-system-object/content-address.md#serial-git` in the * manual. */ @@ -127,7 +129,7 @@ enum struct FileIngestionMethod : uint8_t { * Parse a `FileIngestionMethod` by name. Choice of: * * - `flat`: `FileIngestionMethod::Flat` - * - `nar`: `FileIngestionMethod::Recursive` + * - `nar`: `FileIngestionMethod::NixArchive` * - `git`: `FileIngestionMethod::Git` * * Opposite of `renderFileIngestionMethod`. diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index 84786e95a..be61375f6 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -140,6 +140,7 @@ public: }; #ifndef _WIN32 // Not needed on Windows, where we don't fork +namespace unix { /** * Close all file descriptors except those listed in the given set. @@ -152,13 +153,16 @@ void closeMostFDs(const std::set & exceptions); */ void closeOnExec(Descriptor fd); +} // namespace unix #endif -#ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 +#if defined(_WIN32) && _WIN32_WINNT >= 0x0600 +namespace windows { + Path handleToPath(Descriptor handle); std::wstring handleToFileName(Descriptor handle); -# endif + +} // namespace windows #endif MakeError(EndOfFile, Error); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 919bf5d50..f75851bbd 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -412,31 +412,19 @@ void deletePath(const fs::path & path) deletePath(path, dummy); } - -Paths createDirs(const Path & path) +void createDir(const Path & path, mode_t mode) { - Paths created; - if (path == "/") return created; + if (mkdir(path.c_str(), mode) == -1) + throw SysError("creating directory '%1%'", path); +} - struct stat st; - if (STAT(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str() -#ifndef _WIN32 // TODO abstract mkdir perms for Windows - , 0777 -#endif - ) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = STAT(path); - created.push_back(path); +void createDirs(const Path & path) +{ + try { + fs::create_directories(path); + } catch (fs::filesystem_error & e) { + throw SysError("creating directory '%1%'", path); } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; } @@ -546,7 +534,7 @@ std::pair createTempFile(const Path & prefix) if (!fd) throw SysError("creating temporary file '%s'", tmpl); #ifndef _WIN32 - closeOnExec(fd.get()); + unix::closeOnExec(fd.get()); #endif return {std::move(fd), tmpl}; } @@ -574,29 +562,69 @@ void replaceSymlink(const Path & target, const Path & link) } } -#ifndef _WIN32 -static void setWriteTime(const fs::path & p, const struct stat & st) +void setWriteTime( + const std::filesystem::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional optIsSymlink) { - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, +#ifndef _WIN32 + struct timeval times[2] = { + { + .tv_sec = accessedTime, + .tv_usec = 0, + }, + { + .tv_sec = modificationTime, + .tv_usec = 0, + }, }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} #endif + auto nonSymlink = [&]{ + bool isSymlink = optIsSymlink + ? *optIsSymlink + : fs::is_symlink(path); + + if (!isSymlink) { +#ifdef _WIN32 + // FIXME use `fs::last_write_time`. + // + // Would be nice to use std::filesystem unconditionally, but + // doesn't support access time just modification time. + // + // System clock vs File clock issues also make that annoying. + warn("Changing file times is not yet implemented on Windows, path is '%s'", path); +#else + if (utimes(path.c_str(), times) == -1) { + + throw SysError("changing modification time of '%s' (not a symlink)", path); + } +#endif + } else { + throw Error("Cannot modification time of symlink '%s'", path); + } + }; + +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) { + if (errno == ENOSYS) + nonSymlink(); + else + throw SysError("changing modification time of '%s'", path); + } +#else + nonSymlink(); +#endif +} + +void setWriteTime(const fs::path & path, const struct stat & st) +{ + setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); +} + void copyFile(const fs::path & from, const fs::path & to, bool andDelete) { -#ifndef _WIN32 - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.c_str()); -#endif auto fromStatus = fs::symlink_status(from); // Mark the directory as writable so that we can delete its children @@ -616,9 +644,7 @@ void copyFile(const fs::path & from, const fs::path & to, bool andDelete) throw Error("file '%s' has an unsupported type", from); } -#ifndef _WIN32 - setWriteTime(to, statOfFrom); -#endif + setWriteTime(to, lstat(from.string().c_str())); if (andDelete) { if (!fs::is_symlink(fromStatus)) fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index c6b6ecedb..4b215162d 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -148,15 +148,43 @@ void deletePath(const std::filesystem::path & path); void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed); /** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. + * Create a directory and all its parents, if necessary. */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) +void createDirs(const Path & path); +inline void createDirs(PathView path) { return createDirs(Path(path)); } +/** + * Create a single directory. + */ +void createDir(const Path & path, mode_t mode = 0755); + +/** + * Set the access and modification times of the given path, not + * following symlinks. + * + * @param accessTime Specified in seconds. + * + * @param modificationTime Specified in seconds. + * + * @param isSymlink Whether the file in question is a symlink. Used for + * fallback code where we don't have `lutimes` or similar. if + * `std::optional` is passed, the information will be recomputed if it + * is needed. Race conditions are possible so be careful! + */ +void setWriteTime( + const std::filesystem::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional isSymlink = std::nullopt); + +/** + * Convenience wrapper that takes all arguments from the `struct stat`. + */ +void setWriteTime(const std::filesystem::path & path, const struct stat & st); + /** * Create a symlink. */ diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index ef44a8409..850b7162d 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -111,6 +111,8 @@ std::ostream & operator<<(std::ostream & out, const Magenta & y) /** * Values wrapped in this class are printed without coloring. * + * Specifically, the color is reset to normal before printing the value. + * * By default, arguments to `HintFmt` are printed in magenta (see `Magenta`). */ template diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 91070ea89..194e86fdd 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -1,7 +1,7 @@ #include #include "error.hh" -#include "config.hh" +#include "config-global.hh" #include "fs-sink.hh" #if _WIN32 @@ -14,7 +14,7 @@ namespace nix { void copyRecursive( SourceAccessor & accessor, const CanonPath & from, - FileSystemObjectSink & sink, const Path & to) + FileSystemObjectSink & sink, const CanonPath & to) { auto stat = accessor.lstat(from); @@ -43,7 +43,7 @@ void copyRecursive( for (auto & [name, _] : accessor.readDirectory(from)) { copyRecursive( accessor, from / name, - sink, to + "/" + name); + sink, to / name); break; } break; @@ -69,17 +69,9 @@ static RestoreSinkSettings restoreSinkSettings; static GlobalConfig::Register r1(&restoreSinkSettings); -void RestoreSink::createDirectory(const Path & path) +void RestoreSink::createDirectory(const CanonPath & path) { - Path p = dstPath + path; - if ( -#ifndef _WIN32 // TODO abstract mkdir perms for Windows - mkdir(p.c_str(), 0777) == -1 -#else - !CreateDirectoryW(pathNG(p).c_str(), NULL) -#endif - ) - throw NativeSysError("creating directory '%1%'", p); + std::filesystem::create_directory(dstPath / path.rel()); }; struct RestoreRegularFile : CreateRegularFileSink { @@ -90,13 +82,22 @@ struct RestoreRegularFile : CreateRegularFileSink { void preallocateContents(uint64_t size) override; }; -void RestoreSink::createRegularFile(const Path & path, std::function func) +static std::filesystem::path append(const std::filesystem::path & src, const CanonPath & path) { - Path p = dstPath + path; + auto dst = src; + if (!path.rel().empty()) + dst /= path.rel(); + return dst; +} + +void RestoreSink::createRegularFile(const CanonPath & path, std::function func) +{ + auto p = append(dstPath, path); + RestoreRegularFile crf; crf.fd = #ifdef _WIN32 - CreateFileW(pathNG(path).c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL) + CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL) #else open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666) #endif @@ -141,14 +142,14 @@ void RestoreRegularFile::operator () (std::string_view data) writeFull(fd.get(), data); } -void RestoreSink::createSymlink(const Path & path, const std::string & target) +void RestoreSink::createSymlink(const CanonPath & path, const std::string & target) { - Path p = dstPath + path; + auto p = append(dstPath, path); nix::createSymlink(target, p); } -void RegularFileSink::createRegularFile(const Path & path, std::function func) +void RegularFileSink::createRegularFile(const CanonPath & path, std::function func) { struct CRF : CreateRegularFileSink { RegularFileSink & back; @@ -163,7 +164,7 @@ void RegularFileSink::createRegularFile(const Path & path, std::function func) +void NullFileSystemObjectSink::createRegularFile(const CanonPath & path, std::function func) { struct : CreateRegularFileSink { void operator () (std::string_view data) override {} diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 32dcb2f01..e5e240073 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -28,17 +28,17 @@ struct FileSystemObjectSink { virtual ~FileSystemObjectSink() = default; - virtual void createDirectory(const Path & path) = 0; + virtual void createDirectory(const CanonPath & path) = 0; /** * This function in general is no re-entrant. Only one file can be * written at a time. */ virtual void createRegularFile( - const Path & path, + const CanonPath & path, std::function) = 0; - virtual void createSymlink(const Path & path, const std::string & target) = 0; + virtual void createSymlink(const CanonPath & path, const std::string & target) = 0; }; /** @@ -59,17 +59,17 @@ struct ExtendedFileSystemObjectSink : virtual FileSystemObjectSink */ void copyRecursive( SourceAccessor & accessor, const CanonPath & sourcePath, - FileSystemObjectSink & sink, const Path & destPath); + FileSystemObjectSink & sink, const CanonPath & destPath); /** * Ignore everything and do nothing */ struct NullFileSystemObjectSink : FileSystemObjectSink { - void createDirectory(const Path & path) override { } - void createSymlink(const Path & path, const std::string & target) override { } + void createDirectory(const CanonPath & path) override { } + void createSymlink(const CanonPath & path, const std::string & target) override { } void createRegularFile( - const Path & path, + const CanonPath & path, std::function) override; }; @@ -78,15 +78,15 @@ struct NullFileSystemObjectSink : FileSystemObjectSink */ struct RestoreSink : FileSystemObjectSink { - Path dstPath; + std::filesystem::path dstPath; - void createDirectory(const Path & path) override; + void createDirectory(const CanonPath & path) override; void createRegularFile( - const Path & path, + const CanonPath & path, std::function) override; - void createSymlink(const Path & path, const std::string & target) override; + void createSymlink(const CanonPath & path, const std::string & target) override; }; /** @@ -101,18 +101,18 @@ struct RegularFileSink : FileSystemObjectSink RegularFileSink(Sink & sink) : sink(sink) { } - void createDirectory(const Path & path) override + void createDirectory(const CanonPath & path) override { regular = false; } - void createSymlink(const Path & path, const std::string & target) override + void createSymlink(const CanonPath & path, const std::string & target) override { regular = false; } void createRegularFile( - const Path & path, + const CanonPath & path, std::function) override; }; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 8c538c988..a6968a43e 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -53,7 +53,7 @@ static std::string getString(Source & source, int n) void parseBlob( FileSystemObjectSink & sink, - const Path & sinkPath, + const CanonPath & sinkPath, Source & source, BlobMode blobMode, const ExperimentalFeatureSettings & xpSettings) @@ -116,7 +116,7 @@ void parseBlob( void parseTree( FileSystemObjectSink & sink, - const Path & sinkPath, + const CanonPath & sinkPath, Source & source, std::function hook, const ExperimentalFeatureSettings & xpSettings) @@ -147,7 +147,7 @@ void parseTree( Hash hash(HashAlgorithm::SHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); - hook(name, TreeEntry { + hook(CanonPath{name}, TreeEntry { .mode = mode, .hash = hash, }); @@ -171,7 +171,7 @@ ObjectType parseObjectType( void parse( FileSystemObjectSink & sink, - const Path & sinkPath, + const CanonPath & sinkPath, Source & source, BlobMode rootModeIfBlob, std::function hook, @@ -208,7 +208,7 @@ std::optional convertMode(SourceAccessor::Type type) void restore(FileSystemObjectSink & sink, Source & source, std::function hook) { - parse(sink, "", source, BlobMode::Regular, [&](Path name, TreeEntry entry) { + parse(sink, CanonPath::root, source, BlobMode::Regular, [&](CanonPath name, TreeEntry entry) { auto [accessor, from] = hook(entry.hash); auto stat = accessor->lstat(from); auto gotOpt = convertMode(stat.type); diff --git a/src/libutil/git.hh b/src/libutil/git.hh index a65edb964..2190cc550 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -64,7 +64,7 @@ using Tree = std::map; * Implementations may seek to memoize resources (bandwidth, storage, * etc.) for the same Git hash. */ -using SinkHook = void(const Path & name, TreeEntry entry); +using SinkHook = void(const CanonPath & name, TreeEntry entry); /** * Parse the "blob " or "tree " prefix. @@ -89,13 +89,13 @@ enum struct BlobMode : RawMode }; void parseBlob( - FileSystemObjectSink & sink, const Path & sinkPath, + FileSystemObjectSink & sink, const CanonPath & sinkPath, Source & source, BlobMode blobMode, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); void parseTree( - FileSystemObjectSink & sink, const Path & sinkPath, + FileSystemObjectSink & sink, const CanonPath & sinkPath, Source & source, std::function hook, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -108,7 +108,7 @@ void parseTree( * a blob, this is ignored. */ void parse( - FileSystemObjectSink & sink, const Path & sinkPath, + FileSystemObjectSink & sink, const CanonPath & sinkPath, Source & source, BlobMode rootModeIfBlob, std::function hook, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2f2ed8138..7064e96e6 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -52,11 +52,11 @@ bool Hash::operator == (const Hash & h2) const std::strong_ordering Hash::operator <=> (const Hash & h) const { - if (auto cmp = algo <=> h.algo; cmp != 0) return cmp; if (auto cmp = hashSize <=> h.hashSize; cmp != 0) return cmp; for (unsigned int i = 0; i < hashSize; i++) { if (auto cmp = hash[i] <=> h.hash[i]; cmp != 0) return cmp; } + if (auto cmp = algo <=> h.algo; cmp != 0) return cmp; return std::strong_ordering::equivalent; } diff --git a/src/libutil/linux/meson.build b/src/libutil/linux/meson.build new file mode 100644 index 000000000..a1ded76ca --- /dev/null +++ b/src/libutil/linux/meson.build @@ -0,0 +1,11 @@ +sources += files( + 'cgroup.cc', + 'namespaces.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'cgroup.hh', + 'namespaces.hh', +) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 5fa01f0d9..55751b4cf 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,7 +3,7 @@ #include "environment-variables.hh" #include "terminal.hh" #include "util.hh" -#include "config.hh" +#include "config-global.hh" #include "source-path.hh" #include "position.hh" diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index b7207cffb..c4eee1031 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -124,9 +124,9 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents using File = MemorySourceAccessor::File; -void MemorySink::createDirectory(const Path & path) +void MemorySink::createDirectory(const CanonPath & path) { - auto * f = dst.open(CanonPath{path}, File { File::Directory { } }); + auto * f = dst.open(path, File { File::Directory { } }); if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); @@ -146,9 +146,9 @@ struct CreateMemoryRegularFile : CreateRegularFileSink { void preallocateContents(uint64_t size) override; }; -void MemorySink::createRegularFile(const Path & path, std::function func) +void MemorySink::createRegularFile(const CanonPath & path, std::function func) { - auto * f = dst.open(CanonPath{path}, File { File::Regular {} }); + auto * f = dst.open(path, File { File::Regular {} }); if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); if (auto * rp = std::get_if(&f->raw)) { @@ -173,9 +173,9 @@ void CreateMemoryRegularFile::operator () (std::string_view data) regularFile.contents += data; } -void MemorySink::createSymlink(const Path & path, const std::string & target) +void MemorySink::createSymlink(const CanonPath & path, const std::string & target) { - auto * f = dst.open(CanonPath{path}, File { File::Symlink { } }); + auto * f = dst.open(path, File { File::Symlink { } }); if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); if (auto * s = std::get_if(&f->raw)) diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh index c8f793922..cd5146c89 100644 --- a/src/libutil/memory-source-accessor.hh +++ b/src/libutil/memory-source-accessor.hh @@ -81,13 +81,13 @@ struct MemorySink : FileSystemObjectSink MemorySink(MemorySourceAccessor & dst) : dst(dst) { } - void createDirectory(const Path & path) override; + void createDirectory(const CanonPath & path) override; void createRegularFile( - const Path & path, + const CanonPath & path, std::function) override; - void createSymlink(const Path & path, const std::string & target) override; + void createSymlink(const CanonPath & path, const std::string & target) override; }; } diff --git a/src/libutil/meson.build b/src/libutil/meson.build new file mode 100644 index 000000000..ac2b83536 --- /dev/null +++ b/src/libutil/meson.build @@ -0,0 +1,265 @@ +project('nix-util', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +# Check for each of these functions, and create a define like `#define +# HAVE_LUTIMES 1`. The `#define` is unconditional, 0 for not found and 1 +# for found. One therefore uses it with `#if` not `#ifdef`. +check_funcs = [ + # Optionally used for changing the mtime of symlinks. + 'lutimes', + # Optionally used for creating pipes on Unix + 'pipe2', + # Optionally used to preallocate files to be large enough before + # writing to them. + 'posix_fallocate', + # Optionally used to get more information about processes failing due + # to a signal on Unix. + 'strsignal', + # Optionally used to try to close more file descriptors (e.g. before + # forking) on Unix. + 'sysconf', +] +foreach funcspec : check_funcs + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) +endforeach + +subdir('build-utils-meson/threads') + +if host_machine.system() == 'windows' + socket = cxx.find_library('ws2_32') + deps_other += socket +elif host_machine.system() == 'sunos' + socket = cxx.find_library('socket') + network_service_library = cxx.find_library('nsl') + deps_other += [socket, network_service_library] +endif + +boost = dependency( + 'boost', + modules : ['context', 'coroutine'], +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +openssl = dependency( + 'libcrypto', + 'openssl', + version : '>= 1.1.1', +) +deps_private += openssl + +libarchive = dependency('libarchive', version : '>= 3.1.2') +deps_public += libarchive +if get_option('default_library') == 'static' + # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed + add_project_arguments('-lz', language : 'cpp') +endif + +sodium = dependency('libsodium', 'sodium') +deps_private += sodium + +brotli = [ + dependency('libbrotlicommon'), + dependency('libbrotlidec'), + dependency('libbrotlienc'), +] +deps_private += brotli + +cpuid_required = get_option('cpuid') +if host_machine.cpu_family() != 'x86_64' and cpuid_required.enabled() + warning('Force-enabling seccomp on non-x86_64 does not make sense') +endif +cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required) +configdata.set('HAVE_LIBCPUID', cpuid.found().to_int()) +deps_private += cpuid + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +config_h = configure_file( + configuration : configdata, + output : 'config-util.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'archive.cc', + 'args.cc', + 'canon-path.cc', + 'compression.cc', + 'compute-levels.cc', + 'config.cc', + 'config-global.cc', + 'current-process.cc', + 'english.cc', + 'environment-variables.cc', + 'error.cc', + 'exit.cc', + 'experimental-features.cc', + 'file-content-address.cc', + 'file-descriptor.cc', + 'file-system.cc', + 'fs-sink.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'memory-source-accessor.cc', + 'position.cc', + 'posix-source-accessor.cc', + 'references.cc', + 'serialise.cc', + 'signature/local-keys.cc', + 'signature/signer.cc', + 'source-accessor.cc', + 'source-path.cc', + 'suggestions.cc', + 'tarfile.cc', + 'terminal.cc', + 'thread-pool.cc', + 'unix-domain-socket.cc', + 'url.cc', + 'users.cc', + 'util.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'abstract-setting-to-json.hh', + 'ansicolor.hh', + 'archive.hh', + 'args.hh', + 'args/root.hh', + 'callback.hh', + 'canon-path.hh', + 'chunked-vector.hh', + 'closure.hh', + 'comparator.hh', + 'compression.hh', + 'compute-levels.hh', + 'config-global.hh', + 'config-impl.hh', + 'config.hh', + 'current-process.hh', + 'english.hh', + 'environment-variables.hh', + 'error.hh', + 'exit.hh', + 'experimental-features.hh', + 'file-content-address.hh', + 'file-descriptor.hh', + 'file-path-impl.hh', + 'file-path.hh', + 'file-system.hh', + 'finally.hh', + 'fmt.hh', + 'fs-sink.hh', + 'git.hh', + 'hash.hh', + 'hilite.hh', + 'json-impls.hh', + 'json-utils.hh', + 'logging.hh', + 'lru-cache.hh', + 'memory-source-accessor.hh', + 'muxable-pipe.hh', + 'pool.hh', + 'position.hh', + 'posix-source-accessor.hh', + 'processes.hh', + 'ref.hh', + 'references.hh', + 'regex-combinators.hh', + 'repair-flag.hh', + 'serialise.hh', + 'signals.hh', + 'signature/local-keys.hh', + 'signature/signer.hh', + 'source-accessor.hh', + 'source-path.hh', + 'split.hh', + 'suggestions.hh', + 'sync.hh', + 'tarfile.hh', + 'terminal.hh', + 'thread-pool.hh', + 'topo-sort.hh', + 'types.hh', + 'unix-domain-socket.hh', + 'url-parts.hh', + 'url.hh', + 'users.hh', + 'util.hh', + 'variant-wrapper.hh', + 'xml-writer.hh', +) + +if host_machine.system() == 'linux' + subdir('linux') +endif + +if host_machine.system() == 'windows' + subdir('windows') +else + subdir('unix') +endif + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixutil', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] +if host_machine.system() == 'windows' + # `libraries_private` cannot contain ad-hoc dependencies (from + # `find_library), so we need to do this manually + libraries_private += ['-lws2_32'] +endif + +subdir('build-utils-meson/export') diff --git a/src/libutil/meson.options b/src/libutil/meson.options new file mode 100644 index 000000000..21883af01 --- /dev/null +++ b/src/libutil/meson.options @@ -0,0 +1,5 @@ +# vim: filetype=meson + +option('cpuid', type : 'feature', + description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)', +) diff --git a/src/libutil/package.nix b/src/libutil/package.nix new file mode 100644 index 000000000..28d7d8f0e --- /dev/null +++ b/src/libutil/package.nix @@ -0,0 +1,103 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, boost +, brotli +, libarchive +, libcpuid +, libsodium +, nlohmann_json +, openssl + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + ./linux/meson.build + ./unix/meson.build + ./windows/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + brotli + libsodium + openssl + ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + ; + + propagatedBuildInputs = [ + boost + libarchive + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + # + # TODO: change release process to add `pre` in `.version`, remove it + # before tagging, and restore after. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "cpuid" stdenv.hostPlatform.isx86_64) + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh index 168fcaa55..bbbe7dcab 100644 --- a/src/libutil/processes.hh +++ b/src/libutil/processes.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "error.hh" +#include "file-descriptor.hh" #include "logging.hh" #include "ansicolor.hh" @@ -23,26 +24,36 @@ namespace nix { struct Sink; struct Source; -#ifndef _WIN32 class Pid { +#ifndef _WIN32 pid_t pid = -1; bool separatePG = false; int killSignal = SIGKILL; +#else + AutoCloseFD pid = INVALID_DESCRIPTOR; +#endif public: Pid(); +#ifndef _WIN32 Pid(pid_t pid); - ~Pid(); void operator =(pid_t pid); operator pid_t(); +#else + Pid(AutoCloseFD pid); + void operator =(AutoCloseFD pid); +#endif + ~Pid(); int kill(); int wait(); + // TODO: Implement for Windows +#ifndef _WIN32 void setSeparatePG(bool separatePG); void setKillSignal(int signal); pid_t release(); -}; #endif +}; #ifndef _WIN32 diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index e45cfaf85..f3b2f55b5 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -67,6 +67,17 @@ int getArchiveFilterCodeByName(const std::string & method) return code; } +static void enableSupportedFormats(struct archive * archive) +{ + archive_read_support_format_tar(archive); + archive_read_support_format_zip(archive); + + /* Enable support for empty files so we don't throw an exception + for empty HTTP 304 "Not modified" responses. See + downloadTarball(). */ + archive_read_support_format_empty(archive); +} + TarArchive::TarArchive(Source & source, bool raw, std::optional compression_method) : archive{archive_read_new()} , source{&source} @@ -78,9 +89,9 @@ TarArchive::TarArchive(Source & source, bool raw, std::optional com archive_read_support_filter_by_code(archive, getArchiveFilterCodeByName(*compression_method)); } - if (!raw) { - archive_read_support_format_all(archive); - } else { + if (!raw) + enableSupportedFormats(archive); + else { archive_read_support_format_raw(archive); archive_read_support_format_empty(archive); } @@ -96,7 +107,7 @@ TarArchive::TarArchive(const Path & path) , buffer(defaultBufferSize) { archive_read_support_filter_all(archive); - archive_read_support_format_all(archive); + enableSupportedFormats(archive); archive_read_set_option(archive, NULL, "mac-ext", NULL); check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); } @@ -176,6 +187,7 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & auto path = archive_entry_pathname(entry); if (!path) throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + auto cpath = CanonPath{path}; if (r == ARCHIVE_WARN) warn(archive_error_string(archive.archive)); else @@ -191,11 +203,11 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & switch (auto type = archive_entry_filetype(entry)) { case AE_IFDIR: - parseSink.createDirectory(path); + parseSink.createDirectory(cpath); break; case AE_IFREG: { - parseSink.createRegularFile(path, [&](auto & crf) { + parseSink.createRegularFile(cpath, [&](auto & crf) { if (archive_entry_mode(entry) & S_IXUSR) crf.isExecutable(); @@ -219,7 +231,7 @@ time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & case AE_IFLNK: { auto target = archive_entry_symlink(entry); - parseSink.createSymlink(path, target); + parseSink.createSymlink(cpath, target); break; } diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc index 87914bb83..1707fdb75 100644 --- a/src/libutil/unix-domain-socket.cc +++ b/src/libutil/unix-domain-socket.cc @@ -24,7 +24,7 @@ AutoCloseFD createUnixDomainSocket() if (!fdSocket) throw SysError("cannot create Unix domain socket"); #ifndef _WIN32 - closeOnExec(fdSocket.get()); + unix::closeOnExec(fdSocket.get()); #endif return fdSocket; } diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index a74f16ce1..a3af1623f 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -110,8 +110,8 @@ void Pipe::create() if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); #else if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); + unix::closeOnExec(fds[0]); + unix::closeOnExec(fds[1]); #endif readSide = fds[0]; writeSide = fds[1]; @@ -120,7 +120,7 @@ void Pipe::create() ////////////////////////////////////////////////////////////////////// -void closeMostFDs(const std::set & exceptions) +void unix::closeMostFDs(const std::set & exceptions) { #if __linux__ try { @@ -139,14 +139,16 @@ void closeMostFDs(const std::set & exceptions) #endif int maxFD = 0; +#if HAVE_SYSCONF maxFD = sysconf(_SC_OPEN_MAX); +#endif for (int fd = 0; fd < maxFD; ++fd) if (!exceptions.count(fd)) close(fd); /* ignore result */ } -void closeOnExec(int fd) +void unix::closeOnExec(int fd) { int prev; if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build new file mode 100644 index 000000000..38e5cd3aa --- /dev/null +++ b/src/libutil/unix/meson.build @@ -0,0 +1,17 @@ +sources += files( + 'environment-variables.cc', + 'file-descriptor.cc', + 'file-path.cc', + 'file-system.cc', + 'muxable-pipe.cc', + 'processes.cc', + 'signals.cc', + 'users.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'monitor-fd.hh', + 'signals-impl.hh', +) diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index b5c21ad32..16773e3ea 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -122,7 +122,7 @@ void Pipe::create() #if _WIN32_WINNT >= 0x0600 -std::wstring handleToFileName(HANDLE handle) { +std::wstring windows::handleToFileName(HANDLE handle) { std::vector buf(0x100); DWORD dw = GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED); if (dw == 0) { @@ -141,7 +141,7 @@ std::wstring handleToFileName(HANDLE handle) { } -Path handleToPath(HANDLE handle) { +Path windows::handleToPath(HANDLE handle) { return os_string_to_string(handleToFileName(handle)); } diff --git a/src/libutil/windows/meson.build b/src/libutil/windows/meson.build new file mode 100644 index 000000000..00320877f --- /dev/null +++ b/src/libutil/windows/meson.build @@ -0,0 +1,19 @@ +sources += files( + 'environment-variables.cc', + 'file-descriptor.cc', + 'file-path.cc', + 'file-system.cc', + 'muxable-pipe.cc', + 'processes.cc', + 'users.cc', + 'windows-async-pipe.cc', + 'windows-error.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'signals-impl.hh', + 'windows-async-pipe.hh', + 'windows-error.hh', +) diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 44a32f6a1..9cd714f84 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -1,9 +1,15 @@ #include "current-process.hh" #include "environment-variables.hh" +#include "error.hh" +#include "file-descriptor.hh" +#include "file-path.hh" #include "signals.hh" #include "processes.hh" #include "finally.hh" #include "serialise.hh" +#include "file-system.hh" +#include "util.hh" +#include "windows-error.hh" #include #include @@ -16,25 +22,347 @@ #include #include +#define WIN32_LEAN_AND_MEAN +#include + namespace nix { -std::string runProgram(Path program, bool lookupPath, const Strings & args, - const std::optional & input, bool isInteractive) +using namespace nix::windows; + +Pid::Pid() {} + +Pid::Pid(AutoCloseFD pid) + : pid(std::move(pid)) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); } +Pid::~Pid() +{ + if (pid.get() != INVALID_DESCRIPTOR) + kill(); +} +void Pid::operator=(AutoCloseFD pid) +{ + if (this->pid.get() != INVALID_DESCRIPTOR && this->pid.get() != pid.get()) + kill(); + this->pid = std::move(pid); +} + +// TODO: Implement (not needed for process spawning yet) +int Pid::kill() +{ + assert(pid.get() != INVALID_DESCRIPTOR); + + debug("killing process %1%", pid.get()); + + throw UnimplementedError("Pid::kill unimplemented"); +} + +int Pid::wait() +{ + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L1938 + assert(pid.get() != INVALID_DESCRIPTOR); + DWORD status = WaitForSingleObject(pid.get(), INFINITE); + if (status != WAIT_OBJECT_0) { + debug("WaitForSingleObject returned %1%", status); + } + + DWORD exitCode = 0; + if (GetExitCodeProcess(pid.get(), &exitCode) == FALSE) { + debug("GetExitCodeProcess failed on pid %1%", pid.get()); + } + + pid.close(); + return exitCode; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. +std::string runProgram( + Path program, bool lookupPath, const Strings & args, const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions{ + .program = program, .lookupPath = lookupPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +std::optional getProgramInterpreter(const Path & program) +{ + // These extensions are automatically handled by Windows and don't require an interpreter. + static constexpr const char * exts[] = {".exe", ".cmd", ".bat"}; + for (const auto ext : exts) { + if (hasSuffix(program, ext)) { + return {}; + } + } + // TODO: Open file and read the shebang + throw UnimplementedError("getProgramInterpreter unimplemented"); +} + +// TODO: Not sure if this is needed in the unix version but it might be useful as a member func +void setFDInheritable(AutoCloseFD & fd, bool inherit) +{ + if (fd.get() != INVALID_DESCRIPTOR) { + if (!SetHandleInformation(fd.get(), HANDLE_FLAG_INHERIT, inherit ? HANDLE_FLAG_INHERIT : 0)) { + throw WinError("Couldn't disable inheriting of handle"); + } + } +} + +AutoCloseFD nullFD() +{ + // Create null handle to discard reads / writes + // https://stackoverflow.com/a/25609668 + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L2228 + AutoCloseFD nul = CreateFileW( + L"NUL", + GENERIC_READ | GENERIC_WRITE, + // We don't care who reads / writes / deletes this file since it's NUL anyways + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + 0, + NULL); + if (!nul.get()) { + throw WinError("Couldn't open NUL device"); + } + // Let this handle be inheritable by child processes + setFDInheritable(nul, true); + return nul; +} + +// Adapted from +// https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ +std::string windowsEscape(const std::string & str, bool cmd) +{ + // TODO: This doesn't handle cmd.exe escaping. + if (cmd) { + throw UnimplementedError("cmd.exe escaping is not implemented"); + } + + if (str.find_first_of(" \t\n\v\"") == str.npos && !str.empty()) { + // No need to escape this one, the nonempty contents don't have a special character + return str; + } + std::string buffer; + // Add the opening quote + buffer += '"'; + for (auto iter = str.begin();; ++iter) { + size_t backslashes = 0; + while (iter != str.end() && *iter == '\\') { + ++iter; + ++backslashes; + } + + // We only escape backslashes if: + // - They come immediately before the closing quote + // - They come immediately before a quote in the middle of the string + // Both of these cases break the escaping if not handled. Otherwise backslashes are fine as-is + if (iter == str.end()) { + // Need to escape each backslash + buffer.append(backslashes * 2, '\\'); + // Exit since we've reached the end of the string + break; + } else if (*iter == '"') { + // Need to escape each backslash and the intermediate quote character + buffer.append(backslashes * 2, '\\'); + buffer += "\\\""; + } else { + // Don't escape the backslashes since they won't break the delimiter + buffer.append(backslashes, '\\'); + buffer += *iter; + } + } + // Add the closing quote + return buffer + '"'; +} + +Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) +{ + // Setup pipes. + if (options.standardOut) { + // Don't inherit the read end of the output pipe + setFDInheritable(out.readSide, false); + } else { + out.writeSide = nullFD(); + } + if (options.standardIn) { + // Don't inherit the write end of the input pipe + setFDInheritable(in.writeSide, false); + } else { + in.readSide = nullFD(); + } + + STARTUPINFOW startInfo = {0}; + startInfo.cb = sizeof(startInfo); + startInfo.dwFlags = STARTF_USESTDHANDLES; + startInfo.hStdInput = in.readSide.get(); + startInfo.hStdOutput = out.writeSide.get(); + startInfo.hStdError = out.writeSide.get(); + + std::string envline; + // Retain the current processes' environment variables. + for (const auto & envVar : getEnv()) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + // Also add new ones specified in options. + if (options.environment) { + for (const auto & envVar : *options.environment) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + } + + std::string cmdline = windowsEscape(realProgram, false); + for (const auto & arg : options.args) { + // TODO: This isn't the right way to escape windows command + // See https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw + cmdline += ' ' + windowsEscape(arg, false); + } + + PROCESS_INFORMATION procInfo = {0}; + if (CreateProcessW( + // EXE path is provided in the cmdline + NULL, + string_to_os_string(cmdline).data(), + NULL, + NULL, + TRUE, + CREATE_UNICODE_ENVIRONMENT | CREATE_SUSPENDED, + string_to_os_string(envline).data(), + options.chdir.has_value() ? string_to_os_string(*options.chdir).data() : NULL, + &startInfo, + &procInfo) + == 0) { + throw WinError("CreateProcessW failed (%1%)", cmdline); + } + + // Convert these to use RAII + AutoCloseFD process = procInfo.hProcess; + AutoCloseFD thread = procInfo.hThread; + + // Add current process and child to job object so child terminates when parent terminates + // TODO: This spawns one job per child process. We can probably keep this as a global, and + // add children a single job so we don't use so many jobs at once. + Descriptor job = CreateJobObjectW(NULL, NULL); + if (job == NULL) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't create job object for child process"); + } + if (AssignProcessToJobObject(job, procInfo.hProcess) == FALSE) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't assign child process to job object"); + } + if (ResumeThread(procInfo.hThread) == (DWORD) -1) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't resume child process thread"); + } + + return process; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. // Output = error code + "standard out" output stream std::pair runProgram(RunOptions && options) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); -} + StringSink sink; + options.standardOut = &sink; + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} void runProgram2(const RunOptions & options) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + // TODO: I copied this from unix but this is handled again in spawnProcess, so might be weird to split it up like + // this + if (options.standardOut) + out.create(); + if (source) + in.create(); + + Path realProgram = options.program; + // TODO: Implement shebang / program interpreter lookup on Windows + auto interpreter = getProgramInterpreter(realProgram); + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace([]() { logger->resume(); }); + } + + Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out, in); + + // TODO: This is identical to unix, deduplicate? + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&] { + if (writerThread.joinable()) + writerThread.join(); + }); + + if (source) { + in.readSide.close(); + writerThread = std::thread([&] { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) + promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); } std::string statusToString(int status) @@ -45,10 +373,8 @@ std::string statusToString(int status) return "succeeded"; } - bool statusOk(int status) { return status == 0; } - } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 77df08edd..d37b16bdc 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -26,6 +26,7 @@ #include "legacy.hh" #include "users.hh" #include "network-proxy.hh" +#include "compatibility-settings.hh" using namespace nix; using namespace std::string_literals; @@ -90,24 +91,50 @@ static std::vector shellwords(const std::string & s) return res; } +/** + * Like `resolveExprPath`, but prefers `shell.nix` instead of `default.nix`, + * and if `path` was a directory, it checks eagerly whether `shell.nix` or + * `default.nix` exist, throwing an error if they don't. + */ +static SourcePath resolveShellExprPath(SourcePath path) +{ + auto resolvedOrDir = resolveExprPath(path, false); + if (resolvedOrDir.resolveSymlinks().lstat().type == SourceAccessor::tDirectory) { + if ((resolvedOrDir / "shell.nix").pathExists()) { + if (compatibilitySettings.nixShellAlwaysLooksForShellNix) { + return resolvedOrDir / "shell.nix"; + } else { + warn("Skipping '%1%', because the setting '%2%' is disabled. This is a deprecated behavior. Consider enabling '%2%'.", + resolvedOrDir / "shell.nix", + "nix-shell-always-looks-for-shell-nix"); + } + } + if ((resolvedOrDir / "default.nix").pathExists()) { + return resolvedOrDir / "default.nix"; + } + throw Error("neither '%s' nor '%s' found in '%s'", "shell.nix", "default.nix", resolvedOrDir); + } + return resolvedOrDir; +} + static void main_nix_build(int argc, char * * argv) { auto dryRun = false; - auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); + auto isNixShell = std::regex_search(argv[0], std::regex("nix-shell$")); auto pure = false; auto fromArgs = false; auto packages = false; // Same condition as bash uses for interactive shells auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); Strings attrPaths; - Strings left; + Strings remainingArgs; BuildMode buildMode = bmNormal; bool readStdin = false; std::string envCommand; // interactive shell Strings envExclude; - auto myName = runEnv ? "nix-shell" : "nix-build"; + auto myName = isNixShell ? "nix-shell" : "nix-build"; auto inShebang = false; std::string script; @@ -132,7 +159,7 @@ static void main_nix_build(int argc, char * * argv) // Heuristic to see if we're invoked as a shebang script, namely, // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". - if (runEnv && argc > 1) { + if (isNixShell && argc > 1) { script = argv[1]; try { auto lines = tokenizeString(readFile(script), "\n"); @@ -186,9 +213,9 @@ static void main_nix_build(int argc, char * * argv) dryRun = true; else if (*arg == "--run-env") // obsolete - runEnv = true; + isNixShell = true; - else if (runEnv && (*arg == "--command" || *arg == "--run")) { + else if (isNixShell && (*arg == "--command" || *arg == "--run")) { if (*arg == "--run") interactive = false; envCommand = getArg(*arg, arg, end) + "\nexit"; @@ -206,7 +233,7 @@ static void main_nix_build(int argc, char * * argv) else if (*arg == "--pure") pure = true; else if (*arg == "--impure") pure = false; - else if (runEnv && (*arg == "--packages" || *arg == "-p")) + else if (isNixShell && (*arg == "--packages" || *arg == "-p")) packages = true; else if (inShebang && *arg == "-i") { @@ -246,7 +273,7 @@ static void main_nix_build(int argc, char * * argv) return false; else - left.push_back(*arg); + remainingArgs.push_back(*arg); return true; }); @@ -259,14 +286,14 @@ static void main_nix_build(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; auto autoArgs = myArgs.getAutoArgs(*state); auto autoArgsWithInNixShell = autoArgs; - if (runEnv) { + if (isNixShell) { auto newArgs = state->buildBindings(autoArgsWithInNixShell->size() + 1); newArgs.alloc("inNixShell").mkBool(true); for (auto & i : *autoArgs) newArgs.insert(i); @@ -276,19 +303,26 @@ static void main_nix_build(int argc, char * * argv) if (packages) { std::ostringstream joined; joined << "{...}@args: with import args; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; - for (const auto & i : left) + for (const auto & i : remainingArgs) joined << '(' << i << ") "; joined << "]; } \"\""; fromArgs = true; - left = {joined.str()}; - } else if (!fromArgs) { - if (left.empty() && runEnv && pathExists("shell.nix")) - left = {"shell.nix"}; - if (left.empty()) - left = {"default.nix"}; + remainingArgs = {joined.str()}; + } else if (!fromArgs && remainingArgs.empty()) { + if (isNixShell && !compatibilitySettings.nixShellAlwaysLooksForShellNix && std::filesystem::exists("shell.nix")) { + // If we're in 2.3 compatibility mode, we need to look for shell.nix + // now, because it won't be done later. + remainingArgs = {"shell.nix"}; + } else { + remainingArgs = {"."}; + + // Instead of letting it throw later, we throw here to give a more relevant error message + if (isNixShell && !std::filesystem::exists("shell.nix") && !std::filesystem::exists("default.nix")) + throw Error("no argument specified and no '%s' or '%s' file found in the working directory", "shell.nix", "default.nix"); + } } - if (runEnv) + if (isNixShell) setEnv("IN_NIX_SHELL", pure ? "pure" : "impure"); PackageInfos drvs; @@ -299,7 +333,7 @@ static void main_nix_build(int argc, char * * argv) if (readStdin) exprs = {state->parseStdin()}; else - for (auto i : left) { + for (auto i : remainingArgs) { if (fromArgs) exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath("."))); else { @@ -310,14 +344,18 @@ static void main_nix_build(int argc, char * * argv) auto [path, outputNames] = parsePathWithOutputs(absolute); if (evalStore->isStorePath(path) && hasSuffix(path, ".drv")) drvs.push_back(PackageInfo(*state, evalStore, absolute)); - else + else { /* If we're in a #! script, interpret filenames relative to the script. */ - exprs.push_back( - state->parseExprFromFile( - resolveExprPath( - lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))); + auto baseDir = inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i; + + auto sourcePath = lookupFileArg(*state, + baseDir); + auto resolvedPath = + isNixShell ? resolveShellExprPath(sourcePath) : resolveExprPath(sourcePath); + + exprs.push_back(state->parseExprFromFile(resolvedPath)); + } } } @@ -330,7 +368,7 @@ static void main_nix_build(int argc, char * * argv) std::function takesNixShellAttr; takesNixShellAttr = [&](const Value & v) { - if (!runEnv) { + if (!isNixShell) { return false; } bool add = false; @@ -381,7 +419,7 @@ static void main_nix_build(int argc, char * * argv) store->buildPaths(paths, buildMode, evalStore); }; - if (runEnv) { + if (isNixShell) { if (drvs.size() != 1) throw UsageError("nix-shell requires a single derivation"); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index b5e13cc23..c72378cd5 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1525,7 +1525,7 @@ static int main_nix_env(int argc, char * * argv) auto store = openStore(); - globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store)); + globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, evalSettings)); globals.state->repair = myArgs.repair; globals.instSource.nixExprPath = std::make_shared( diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 5246b03e4..a24dd11d6 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -115,7 +115,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, std::string str2 = str.str(); StringSource source { str2 }; state.store->addToStoreFromDump( - source, "env-manifest.nix", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references); + source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references); }); /* Get the environment builder expression. */ diff --git a/src/nix-expr-test-support b/src/nix-expr-test-support new file mode 120000 index 000000000..427b80dff --- /dev/null +++ b/src/nix-expr-test-support @@ -0,0 +1 @@ +../tests/unit/libexpr-support \ No newline at end of file diff --git a/src/nix-expr-tests b/src/nix-expr-tests new file mode 120000 index 000000000..3af7110d3 --- /dev/null +++ b/src/nix-expr-tests @@ -0,0 +1 @@ +../tests/unit/libexpr \ No newline at end of file diff --git a/src/nix-fetchers-tests b/src/nix-fetchers-tests new file mode 120000 index 000000000..80e4b68ae --- /dev/null +++ b/src/nix-fetchers-tests @@ -0,0 +1 @@ +../tests/unit/libfetchers \ No newline at end of file diff --git a/src/nix-flake-tests b/src/nix-flake-tests new file mode 120000 index 000000000..bb2d49400 --- /dev/null +++ b/src/nix-flake-tests @@ -0,0 +1 @@ +../tests/unit/libflake \ No newline at end of file diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 35664374c..a4bfeebbb 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -157,7 +157,7 @@ static int main_nix_instantiate(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix-store-test-support b/src/nix-store-test-support new file mode 120000 index 000000000..af4befd90 --- /dev/null +++ b/src/nix-store-test-support @@ -0,0 +1 @@ +../tests/unit/libstore-support \ No newline at end of file diff --git a/src/nix-store-tests b/src/nix-store-tests new file mode 120000 index 000000000..fc9b910af --- /dev/null +++ b/src/nix-store-tests @@ -0,0 +1 @@ +../tests/unit/libstore \ No newline at end of file diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6d028e0a7..d0840a02e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -194,10 +194,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto method = FileIngestionMethod::Flat; + ContentAddressMethod method = ContentAddressMethod::Raw::Flat; for (auto & i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::Recursive; + if (i == "--recursive") method = ContentAddressMethod::Raw::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) @@ -223,7 +223,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -563,7 +563,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (!hashGiven) { HashResult hash = hashPath( {store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); info->narHash = hash.first; info->narSize = hash.second; } diff --git a/src/nix-util-test-support b/src/nix-util-test-support new file mode 120000 index 000000000..4b25930eb --- /dev/null +++ b/src/nix-util-test-support @@ -0,0 +1 @@ +../tests/unit/libutil-support \ No newline at end of file diff --git a/src/nix-util-tests b/src/nix-util-tests new file mode 120000 index 000000000..e1138411a --- /dev/null +++ b/src/nix-util-tests @@ -0,0 +1 @@ +../tests/unit/libutil \ No newline at end of file diff --git a/src/nix/.version b/src/nix/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/nix/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index af6743375..5c08f7616 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -12,7 +12,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - ContentAddressMethod caMethod = FileIngestionMethod::Recursive; + ContentAddressMethod caMethod = ContentAddressMethod::Raw::NixArchive; HashAlgorithm hashAlgo = HashAlgorithm::SHA256; CmdAddToStore() @@ -68,7 +68,7 @@ struct CmdAddFile : CmdAddToStore { CmdAddFile() { - caMethod = FileIngestionMethod::Flat; + caMethod = ContentAddressMethod::Raw::Flat; } std::string description() override diff --git a/src/nix/build-remote b/src/nix/build-remote new file mode 120000 index 000000000..2cea44d46 --- /dev/null +++ b/src/nix/build-remote @@ -0,0 +1 @@ +../build-remote \ No newline at end of file diff --git a/src/nix/build-utils-meson b/src/nix/build-utils-meson new file mode 120000 index 000000000..91937f183 --- /dev/null +++ b/src/nix/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson/ \ No newline at end of file diff --git a/src/nix/config.cc b/src/nix/config.cc index 52706afcf..07f975a00 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "config-global.hh" #include diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 27287a1a8..6bd3dc9ef 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,3 +1,4 @@ +#include "config-global.hh" #include "eval.hh" #include "installable-flake.hh" #include "command-installable-value.hh" @@ -238,7 +239,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore auto getEnvShPath = ({ StringSource source { getEnvSh }; evalStore->addToStoreFromDump( - source, "get-env.sh", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, {}); + source, "get-env.sh", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, {}); }); drv.args = {store->printStorePath(getEnvShPath)}; diff --git a/src/nix/doc b/src/nix/doc new file mode 120000 index 000000000..7e57b0f58 --- /dev/null +++ b/src/nix/doc @@ -0,0 +1 @@ +../../doc \ No newline at end of file diff --git a/src/nix/flake.cc b/src/nix/flake.cc index fb7ea6211..84c659023 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -233,6 +233,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["lastModified"] = *lastModified; j["path"] = storePath; j["locks"] = lockedFlake.lockFile.toJSON().first; + if (auto fingerprint = lockedFlake.getFingerprint(store)) + j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); logger->cout("%s", j.dump()); } else { logger->cout( @@ -265,6 +267,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); + if (auto fingerprint = lockedFlake.getFingerprint(store)) + logger->cout( + ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", + fingerprint->to_string(HashFormat::Base16, false)); if (!lockedFlake.lockFile.root->inputs.empty()) logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index f57b224d2..62266fda1 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -68,7 +68,7 @@ struct CmdHashBase : Command switch (mode) { case FileIngestionMethod::Flat: return "print cryptographic hash of a regular file"; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return "print cryptographic hash of the NAR serialisation of a path"; case FileIngestionMethod::Git: return "print cryptographic hash of the Git serialisation of a path"; @@ -91,7 +91,7 @@ struct CmdHashBase : Command Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ switch (mode) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { auto hashSink = makeSink(); dumpPath(path2, *hashSink, (FileSerialisationMethod) mode); @@ -126,7 +126,7 @@ struct CmdHashBase : Command struct CmdHashPath : CmdHashBase { CmdHashPath() - : CmdHashBase(FileIngestionMethod::Recursive) + : CmdHashBase(FileIngestionMethod::NixArchive) { addFlag(flag::hashAlgo("algo", &hashAlgo)); addFlag(flag::fileIngestionMethod(&mode)); @@ -311,7 +311,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); + CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::NixArchive); if (!hashAlgo.has_value()) hashAlgo = HashAlgorithm::MD5; cmd.hashAlgo = hashAlgo.value(); cmd.hashFormat = hashFormat; diff --git a/src/nix/help-stores.md b/src/nix/help-stores.md new file mode 120000 index 000000000..5c5624f5e --- /dev/null +++ b/src/nix/help-stores.md @@ -0,0 +1 @@ +../../doc/manual/src/store/types/index.md.in \ No newline at end of file diff --git a/src/nix/local.mk b/src/nix/local.mk index 9883509fb..28b30b586 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -24,9 +24,9 @@ ifdef HOST_UNIX INCLUDE_nix += -I $(d)/unix endif -nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix) +nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix) -nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd +nix_LIBS = libexpr libmain libfetchers libflake libstore libutil libcmd nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) @@ -42,27 +42,16 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh -src/nix/develop.cc: src/nix/get-env.sh.gen.hh +$(d)/develop.cc: $(d)/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: \ +$(d)/main.cc: \ doc/manual/generate-manpage.nix.gen.hh \ doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \ doc/manual/generate-store-info.nix.gen.hh \ - src/nix/generated-doc/help-stores.md + $(d)/help-stores.md.gen.hh -src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md - @mkdir -p $$(dirname $@) - @cp $< $@ +$(d)/profile.cc: $(d)/profile.md -src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh - -src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in - @mkdir -p $$(dirname $@) - @echo 'R"(' >> $@.tmp - @echo >> $@.tmp - @cat $^ >> $@.tmp - @echo >> $@.tmp - @echo ')"' >> $@.tmp - @mv $@.tmp $@ +$(d)/profile.md: $(d)/profiles.md.gen.hh diff --git a/src/nix/main.cc b/src/nix/main.cc index 81d368d6a..c90bb25a7 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -242,7 +242,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) evalSettings.restrictEval = false; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); + EvalState state({}, openStore("dummy://"), evalSettings); auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( @@ -333,7 +333,7 @@ struct CmdHelpStores : Command std::string doc() override { return - #include "generated-doc/help-stores.md" + #include "help-stores.md.gen.hh" ; } @@ -418,36 +418,29 @@ void mainWrapped(int argc, char * * argv) Xp::FetchTree, }; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); - auto res = nlohmann::json::object(); - res["builtins"] = ({ - auto builtinsJson = nlohmann::json::object(); - for (auto & builtin : *state.baseEnv.values[0]->attrs()) { - auto b = nlohmann::json::object(); - if (!builtin.value->isPrimOp()) continue; - auto primOp = builtin.value->primOp(); - if (!primOp->doc) continue; - b["arity"] = primOp->arity; - b["args"] = primOp->args; - b["doc"] = trim(stripIndentation(primOp->doc)); + EvalState state({}, openStore("dummy://"), evalSettings); + auto builtinsJson = nlohmann::json::object(); + for (auto & builtin : *state.baseEnv.values[0]->attrs()) { + auto b = nlohmann::json::object(); + if (!builtin.value->isPrimOp()) continue; + auto primOp = builtin.value->primOp(); + if (!primOp->doc) continue; + b["args"] = primOp->args; + b["doc"] = trim(stripIndentation(primOp->doc)); + if (primOp->experimentalFeature) b["experimental-feature"] = primOp->experimentalFeature; - builtinsJson[state.symbols[builtin.name]] = std::move(b); - } - std::move(builtinsJson); - }); - res["constants"] = ({ - auto constantsJson = nlohmann::json::object(); - for (auto & [name, info] : state.constantInfos) { - auto c = nlohmann::json::object(); - if (!info.doc) continue; - c["doc"] = trim(stripIndentation(info.doc)); - c["type"] = showType(info.type, false); - c["impure-only"] = info.impureOnly; - constantsJson[name] = std::move(c); - } - std::move(constantsJson); - }); - logger->cout("%s", res); + builtinsJson[state.symbols[builtin.name]] = std::move(b); + } + for (auto & [name, info] : state.constantInfos) { + auto b = nlohmann::json::object(); + if (!info.doc) continue; + b["doc"] = trim(stripIndentation(info.doc)); + b["type"] = showType(info.type, false); + if (info.impureOnly) + b["impure-only"] = true; + builtinsJson[name] = std::move(b); + } + logger->cout("%s", builtinsJson); return; } diff --git a/src/nix/meson.build b/src/nix/meson.build new file mode 100644 index 000000000..dd21c4b1b --- /dev/null +++ b/src/nix/meson.build @@ -0,0 +1,170 @@ +project('nix', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), + dependency('nix-fetchers'), + dependency('nix-main'), + dependency('nix-cmd'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/export-all-symbols') + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + #'-include', 'config-fetchers.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/generate-header') + +nix_sources = files( + 'add-to-store.cc', + 'app.cc', + 'build.cc', + 'bundle.cc', + 'cat.cc', + 'config-check.cc', + 'config.cc', + 'copy.cc', + 'derivation-add.cc', + 'derivation-show.cc', + 'derivation.cc', + 'develop.cc', + 'diff-closures.cc', + 'dump-path.cc', + 'edit.cc', + 'env.cc', + 'eval.cc', + 'flake.cc', + 'fmt.cc', + 'hash.cc', + 'log.cc', + 'ls.cc', + 'main.cc', + 'make-content-addressed.cc', + 'nar.cc', + 'optimise-store.cc', + 'path-from-hash-part.cc', + 'path-info.cc', + 'prefetch.cc', + 'profile.cc', + 'realisation.cc', + 'registry.cc', + 'repl.cc', + 'run.cc', + 'search.cc', + 'sigs.cc', + 'store-copy-log.cc', + 'store-delete.cc', + 'store-gc.cc', + 'store-info.cc', + 'store-repair.cc', + 'store.cc', + 'unix/daemon.cc', + 'upgrade-nix.cc', + 'verify.cc', + 'why-depends.cc', +) + +nix_sources += [ + gen_header.process('doc/manual/generate-manpage.nix'), + gen_header.process('doc/manual/generate-settings.nix'), + gen_header.process('doc/manual/generate-store-info.nix'), + gen_header.process('doc/manual/utils.nix'), + gen_header.process('get-env.sh'), + gen_header.process('profiles.md'), + gen_header.process('help-stores.md'), +] + +if host_machine.system() != 'windows' + nix_sources += files( + 'unix/daemon.cc', + ) +endif + +# The rest of the subdirectories aren't separate components, +# just source files in another directory, so we process them here. + +build_remote_sources = files( + 'build-remote/build-remote.cc', +) +nix_build_sources = files( + 'nix-build/nix-build.cc', +) +nix_channel_sources = files( + 'nix-channel/nix-channel.cc', +) +unpack_channel_gen = gen_header.process('nix-channel/unpack-channel.nix') +nix_collect_garbage_sources = files( + 'nix-collect-garbage/nix-collect-garbage.cc', +) +nix_copy_closure_sources = files( + 'nix-copy-closure/nix-copy-closure.cc', +) +nix_env_buildenv_gen = gen_header.process('nix-env/buildenv.nix') +nix_env_sources = files( + 'nix-env/nix-env.cc', + 'nix-env/user-env.cc', +) +nix_instantiate_sources = files( + 'nix-instantiate/nix-instantiate.cc', +) +nix_store_sources = files( + 'nix-store/dotgraph.cc', + 'nix-store/graphml.cc', + 'nix-store/nix-store.cc', +) + +# Hurray for Meson list flattening! +sources = [ + nix_sources, + build_remote_sources, + nix_build_sources, + nix_channel_sources, + unpack_channel_gen, + nix_collect_garbage_sources, + nix_copy_closure_sources, + nix_env_buildenv_gen, + nix_env_sources, + nix_instantiate_sources, + nix_store_sources, +] + +include_dirs = [include_directories('.')] + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + install : true, +) diff --git a/src/nix/nix-build b/src/nix/nix-build new file mode 120000 index 000000000..2954d8ac7 --- /dev/null +++ b/src/nix/nix-build @@ -0,0 +1 @@ +../nix-build \ No newline at end of file diff --git a/src/nix/nix-channel b/src/nix/nix-channel new file mode 120000 index 000000000..29b759473 --- /dev/null +++ b/src/nix/nix-channel @@ -0,0 +1 @@ +../nix-channel \ No newline at end of file diff --git a/src/nix/nix-collect-garbage b/src/nix/nix-collect-garbage new file mode 120000 index 000000000..b037fc1b0 --- /dev/null +++ b/src/nix/nix-collect-garbage @@ -0,0 +1 @@ +../nix-collect-garbage \ No newline at end of file diff --git a/src/nix/nix-copy-closure b/src/nix/nix-copy-closure new file mode 120000 index 000000000..9063c583a --- /dev/null +++ b/src/nix/nix-copy-closure @@ -0,0 +1 @@ +../nix-copy-closure \ No newline at end of file diff --git a/src/nix/nix-env b/src/nix/nix-env new file mode 120000 index 000000000..f2f19f580 --- /dev/null +++ b/src/nix/nix-env @@ -0,0 +1 @@ +../nix-env \ No newline at end of file diff --git a/src/nix/nix-instantiate b/src/nix/nix-instantiate new file mode 120000 index 000000000..2d7502ffa --- /dev/null +++ b/src/nix/nix-instantiate @@ -0,0 +1 @@ +../nix-instantiate \ No newline at end of file diff --git a/src/nix/nix-store b/src/nix/nix-store new file mode 120000 index 000000000..e6efcac42 --- /dev/null +++ b/src/nix/nix-store @@ -0,0 +1 @@ +../nix-store/ \ No newline at end of file diff --git a/src/nix/package.nix b/src/nix/package.nix new file mode 100644 index 000000000..fe83c6969 --- /dev/null +++ b/src/nix/package.nix @@ -0,0 +1,129 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store +, nix-expr +, nix-main +, nix-cmd + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix"; + inherit version; + + workDir = ./.; + fileset = fileset.unions ([ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + + # Symbolic links to other dirs + ./build-remote + ./doc + ./nix-build + ./nix-channel + ./nix-collect-garbage + ./nix-copy-closure + ./nix-env + ./nix-instantiate + ./nix-store + + # Doc nix files for --help + ../../doc/manual/generate-manpage.nix + ../../doc/manual/utils.nix + ../../doc/manual/generate-settings.nix + ../../doc/manual/generate-store-info.nix + + # Other files to be included as string literals + ../nix-channel/unpack-channel.nix + ../nix-env/buildenv.nix + ./get-env.sh + ./help-stores.md + ../../doc/manual/src/store/types/index.md.in + ./profiles.md + ../../doc/manual/src/command-ref/files/profiles.md + + # Files + ] ++ lib.concatMap + (dir: [ + (fileset.fileFilter (file: file.hasExt "cc") dir) + (fileset.fileFilter (file: file.hasExt "hh") dir) + (fileset.fileFilter (file: file.hasExt "md") dir) + ]) + [ + ./. + ../build-remote + ../nix-build + ../nix-channel + ../nix-collect-garbage + ../nix-copy-closure + ../nix-env + ../nix-instantiate + ../nix-store + ] + ); + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-store + nix-expr + nix-main + nix-cmd + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3ce52acc5..c6e60a53b 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -57,7 +57,9 @@ std::tuple prefetchFile( bool unpack, bool executable) { - auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + ContentAddressMethod method = unpack || executable + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Figure out a name in the Nix store. */ if (!name) { @@ -73,11 +75,10 @@ std::tuple prefetchFile( the store. */ if (expectedHash) { hashAlgo = expectedHash->algo; - storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - .method = ingestionMethod, - .hash = *expectedHash, - .references = {}, - }); + storePath = store->makeFixedOutputPathFromCA(*name, ContentAddressWithReferences::fromParts( + method, + *expectedHash, + {})); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -113,14 +114,15 @@ std::tuple prefetchFile( createDirs(unpacked); unpackTarfile(tmpFile.string(), unpacked); + auto entries = std::filesystem::directory_iterator{unpacked}; /* If the archive unpacks to a single file/directory, then use that as the top-level. */ - auto entries = std::filesystem::directory_iterator{unpacked}; - auto file_count = std::distance(entries, std::filesystem::directory_iterator{}); - if (file_count == 1) - tmpFile = entries->path(); - else + tmpFile = entries->path(); + auto fileCount = std::distance(entries, std::filesystem::directory_iterator{}); + if (fileCount != 1) { + /* otherwise, use the directory itself */ tmpFile = unpacked; + } } Activity act(*logger, lvlChatty, actUnknown, @@ -128,7 +130,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow( *name, PosixSourceAccessor::createAtRoot(tmpFile), - ingestionMethod, hashAlgo, {}, expectedHash); + method, hashAlgo, {}, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash; @@ -193,7 +195,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) startProgressBar(); auto store = openStore(); - auto state = std::make_unique(myArgs.lookupPath, store); + auto state = std::make_unique(myArgs.lookupPath, store, evalSettings); Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index a5a40e4f6..c89b8c9bd 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -258,7 +258,7 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = narHash, .references = { .others = std::move(references), diff --git a/src/nix/profile.md b/src/nix/profile.md index 9b2f86f4a..83a0b5f29 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. )"" -#include "generated-doc/files/profiles.md.gen.hh" +#include "profiles.md.gen.hh" R""( diff --git a/src/nix/profiles.md b/src/nix/profiles.md new file mode 120000 index 000000000..c67a86194 --- /dev/null +++ b/src/nix/profiles.md @@ -0,0 +1 @@ +../../doc/manual/src/command-ref/files/profiles.md \ No newline at end of file diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index de77a7b6b..4a7997b1f 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -10,6 +10,7 @@ #include "serialise.hh" #include "archive.hh" #include "globals.hh" +#include "config-global.hh" #include "derivations.hh" #include "finally.hh" #include "legacy.hh" @@ -202,7 +203,11 @@ static PeerInfo getPeerInfo(int remote) #if defined(SO_PEERCRED) - ucred cred; +# if defined(__OpenBSD__) + struct sockpeercred cred; +# else + ucred cred; +# endif socklen_t credLen = sizeof(cred); if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) throw SysError("getting peer credentials"); @@ -210,9 +215,9 @@ static PeerInfo getPeerInfo(int remote) #elif defined(LOCAL_PEERCRED) -#if !defined(SOL_LOCAL) -#define SOL_LOCAL 0 -#endif +# if !defined(SOL_LOCAL) +# define SOL_LOCAL 0 +# endif xucred cred; socklen_t credLen = sizeof(cred); @@ -295,7 +300,7 @@ static void daemonLoop(std::optional forceTrustClientOpt) if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") throw Error("unexpected systemd environment variables"); fdSocket = SD_LISTEN_FDS_START; - closeOnExec(fdSocket.get()); + unix::closeOnExec(fdSocket.get()); } // Otherwise, create and bind to a Unix domain socket. @@ -323,7 +328,7 @@ static void daemonLoop(std::optional forceTrustClientOpt) throw SysError("accepting connection"); } - closeOnExec(remote.get()); + unix::closeOnExec(remote.get()); PeerInfo peer { .pidKnown = false }; TrustedFlag trusted; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17d1edb97..29297253b 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -147,7 +147,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(LookupPath{}, store); + auto state = std::make_unique(LookupPath{}, store, evalSettings); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/src/perl/.version b/src/perl/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/perl/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/perl/.yath.rc.in b/src/perl/.yath.rc.in similarity index 100% rename from perl/.yath.rc.in rename to src/perl/.yath.rc.in diff --git a/perl/MANIFEST b/src/perl/MANIFEST similarity index 100% rename from perl/MANIFEST rename to src/perl/MANIFEST diff --git a/perl/lib/Nix/Config.pm.in b/src/perl/lib/Nix/Config.pm.in similarity index 100% rename from perl/lib/Nix/Config.pm.in rename to src/perl/lib/Nix/Config.pm.in diff --git a/perl/lib/Nix/CopyClosure.pm b/src/perl/lib/Nix/CopyClosure.pm similarity index 100% rename from perl/lib/Nix/CopyClosure.pm rename to src/perl/lib/Nix/CopyClosure.pm diff --git a/perl/lib/Nix/Manifest.pm b/src/perl/lib/Nix/Manifest.pm similarity index 100% rename from perl/lib/Nix/Manifest.pm rename to src/perl/lib/Nix/Manifest.pm diff --git a/perl/lib/Nix/SSH.pm b/src/perl/lib/Nix/SSH.pm similarity index 100% rename from perl/lib/Nix/SSH.pm rename to src/perl/lib/Nix/SSH.pm diff --git a/perl/lib/Nix/Store.pm b/src/perl/lib/Nix/Store.pm similarity index 100% rename from perl/lib/Nix/Store.pm rename to src/perl/lib/Nix/Store.pm diff --git a/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs similarity index 97% rename from perl/lib/Nix/Store.xs rename to src/perl/lib/Nix/Store.xs index e751c2be1..f951437c8 100644 --- a/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -1,4 +1,5 @@ -#include "nix/config.h" +#include "config-util.hh" +#include "config-store.hh" #include "EXTERN.h" #include "perl.h" @@ -258,7 +259,7 @@ hashPath(char * algo, int base32, char * path) try { Hash h = hashPath( PosixSourceAccessor::createAtRoot(path), - FileIngestionMethod::Recursive, parseHashAlgo(algo)).first; + FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -334,7 +335,7 @@ SV * StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; auto path = THIS->store->addToStore( std::string(baseNameOf(srcPath)), PosixSourceAccessor::createAtRoot(srcPath), @@ -350,7 +351,7 @@ StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char PPCODE: try { auto h = Hash::parseAny(hash, parseHashAlgo(algo)); - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { .method = method, .hash = h, diff --git a/perl/lib/Nix/Utils.pm b/src/perl/lib/Nix/Utils.pm similarity index 100% rename from perl/lib/Nix/Utils.pm rename to src/perl/lib/Nix/Utils.pm diff --git a/perl/lib/Nix/meson.build b/src/perl/lib/Nix/meson.build similarity index 96% rename from perl/lib/Nix/meson.build rename to src/perl/lib/Nix/meson.build index 9a79245cd..256e66096 100644 --- a/perl/lib/Nix/meson.build +++ b/src/perl/lib/Nix/meson.build @@ -43,6 +43,7 @@ nix_perl_store_lib = library( 'Store', sources : nix_perl_store_cc, name_prefix : '', + prelink : true, # For C++ static initializers install : true, install_mode : 'rwxr-xr-x', install_dir : join_paths(nix_perl_install_dir, 'auto', 'Nix', 'Store'), diff --git a/perl/meson.build b/src/perl/meson.build similarity index 95% rename from perl/meson.build rename to src/perl/meson.build index 350e5bd67..5fe7e1e28 100644 --- a/perl/meson.build +++ b/src/perl/meson.build @@ -7,27 +7,30 @@ project ( 'nix-perl', 'cpp', - meson_version : '>= 0.64.0', + version : files('.version'), + meson_version : '>= 1.1', license : 'LGPL-2.1-or-later', ) # setup env #------------------------------------------------- fs = import('fs') -nix_version = get_option('version') cpp = meson.get_compiler('cpp') nix_perl_conf = configuration_data() -nix_perl_conf.set('PACKAGE_VERSION', nix_version) +nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) # set error arguments #------------------------------------------------- error_args = [ + '-Werror=unused-result', + '-Wdeprecated-copy', + '-Wdeprecated-declarations', + '-Wignored-qualifiers', '-Wno-pedantic', '-Wno-non-virtual-dtor', '-Wno-unused-parameter', '-Wno-variadic-macros', - '-Wdeprecated-declarations', '-Wno-missing-field-initializers', '-Wno-unknown-warning-option', '-Wno-unused-variable', @@ -64,7 +67,7 @@ yath = find_program('yath', required : false) bzip2_dep = dependency('bzip2') curl_dep = dependency('libcurl') libsodium_dep = dependency('libsodium') -# nix_util_dep = dependency('nix-util') + nix_store_dep = dependency('nix-store') diff --git a/perl/meson_options.txt b/src/perl/meson.options similarity index 88% rename from perl/meson_options.txt rename to src/perl/meson.options index 82ca52f37..9b5b6b1d9 100644 --- a/perl/meson_options.txt +++ b/src/perl/meson.options @@ -5,11 +5,6 @@ # compiler args #============================================================================ -option( - 'version', - type : 'string', - description : 'nix-perl version') - option( 'tests', type : 'feature', diff --git a/perl/default.nix b/src/perl/package.nix similarity index 53% rename from perl/default.nix rename to src/perl/package.nix index 45682381e..26856e631 100644 --- a/perl/default.nix +++ b/src/perl/package.nix @@ -1,52 +1,53 @@ { lib -, fileset , stdenv +, mkMesonDerivation , perl , perlPackages , meson , ninja , pkg-config -, nix +, nix-store +, darwin +, version , curl , bzip2 -, xz -, boost , libsodium -, darwin }: -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; +let + inherit (lib) fileset; +in - src = fileset.toSource { - root = ./.; - fileset = fileset.unions ([ - ./MANIFEST - ./lib - ./meson.build - ./meson_options.txt - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc.in - ./t - ]); - }; +perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: { + pname = "nix-perl"; + inherit version; + + workDir = ./.; + fileset = fileset.unions ([ + ./.version + ../../.version + ./MANIFEST + ./lib + ./meson.build + ./meson.options + ] ++ lib.optionals finalAttrs.doCheck [ + ./.yath.rc.in + ./t + ]); nativeBuildInputs = [ meson ninja pkg-config + perl + curl ]; buildInputs = [ - nix - curl + nix-store bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + libsodium + ]; # `perlPackages.Test2Harness` is marked broken for Darwin doCheck = !stdenv.isDarwin; @@ -55,8 +56,14 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { perlPackages.Test2Harness ]; + preConfigure = + # "Inline" .version so its not a symlink, and includes the suffix + '' + chmod u+w .version + echo ${finalAttrs.version} > .version + ''; + mesonFlags = [ - (lib.mesonOption "version" (builtins.readFile ../.version)) (lib.mesonOption "dbi_path" "${perlPackages.DBI}/${perl.libPrefix}") (lib.mesonOption "dbd_sqlite_path" "${perlPackages.DBDSQLite}/${perl.libPrefix}") (lib.mesonEnable "tests" finalAttrs.doCheck) diff --git a/perl/t/init.t b/src/perl/t/init.t similarity index 100% rename from perl/t/init.t rename to src/perl/t/init.t diff --git a/perl/t/meson.build b/src/perl/t/meson.build similarity index 100% rename from perl/t/meson.build rename to src/perl/t/meson.build diff --git a/src/toml11/LICENSE b/src/toml11/LICENSE deleted file mode 100644 index f55c511d6..000000000 --- a/src/toml11/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 Toru Niina - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/src/toml11/README.md b/src/toml11/README.md deleted file mode 100644 index 62b586305..000000000 --- a/src/toml11/README.md +++ /dev/null @@ -1,1966 +0,0 @@ -toml11 -====== - -[![Build Status on GitHub Actions](https://github.com/ToruNiina/toml11/workflows/build/badge.svg)](https://github.com/ToruNiina/toml11/actions) -[![Build Status on TravisCI](https://travis-ci.org/ToruNiina/toml11.svg?branch=master)](https://travis-ci.org/ToruNiina/toml11) -[![Build status on Appveyor](https://ci.appveyor.com/api/projects/status/m2n08a926asvg5mg/branch/master?svg=true)](https://ci.appveyor.com/project/ToruNiina/toml11/branch/master) -[![Build status on CircleCI](https://circleci.com/gh/ToruNiina/toml11/tree/master.svg?style=svg)](https://circleci.com/gh/ToruNiina/toml11/tree/master) -[![Version](https://img.shields.io/github/release/ToruNiina/toml11.svg?style=flat)](https://github.com/ToruNiina/toml11/releases) -[![License](https://img.shields.io/github/license/ToruNiina/toml11.svg?style=flat)](LICENSE) -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1209136.svg)](https://doi.org/10.5281/zenodo.1209136) - -toml11 is a C++11 (or later) header-only toml parser/encoder depending only on C++ standard library. - -- It is compatible to the latest version of [TOML v1.0.0](https://toml.io/en/v1.0.0). -- It is one of the most TOML standard compliant libraries, tested with [the language agnostic test suite for TOML parsers by BurntSushi](https://github.com/BurntSushi/toml-test). -- It shows highly informative error messages. You can see the error messages about invalid files at [CircleCI](https://circleci.com/gh/ToruNiina/toml11). -- It has configurable container. You can use any random-access containers and key-value maps as backend containers. -- It optionally preserves comments without any overhead. -- It has configurable serializer that supports comments, inline tables, literal strings and multiline strings. -- It supports user-defined type conversion from/into toml values. -- It correctly handles UTF-8 sequences, with or without BOM, both on posix and Windows. - -## Example - -```cpp -#include -#include - -int main() -{ - // ```toml - // title = "an example toml file" - // nums = [3, 1, 4, 1, 5] - // ``` - auto data = toml::parse("example.toml"); - - // find a value with the specified type from a table - std::string title = toml::find(data, "title"); - - // convert the whole array into any container automatically - std::vector nums = toml::find>(data, "nums"); - - // access with STL-like manner - if(!data.contains("foo")) - { - data["foo"] = "bar"; - } - - // pass a fallback - std::string name = toml::find_or(data, "name", "not found"); - - // width-dependent formatting - std::cout << std::setw(80) << data << std::endl; - - return 0; -} -``` - -## Table of Contents - -- [Integration](#integration) -- [Decoding a toml file](#decoding-a-toml-file) - - [In the case of syntax error](#in-the-case-of-syntax-error) - - [Invalid UTF-8 Codepoints](#invalid-utf-8-codepoints) -- [Finding a toml value](#finding-a-toml-value) - - [Finding a value in a table](#finding-a-value-in-a-table) - - [In case of error](#in-case-of-error) - - [Dotted keys](#dotted-keys) -- [Casting a toml value](#casting-a-toml-value) -- [Checking value type](#checking-value-type) -- [More about conversion](#more-about-conversion) - - [Converting an array](#converting-an-array) - - [Converting a table](#converting-a-table) - - [Getting an array of tables](#getting-an-array-of-tables) - - [Cost of conversion](#cost-of-conversion) - - [Converting datetime and its variants](#converting-datetime-and-its-variants) -- [Getting with a fallback](#getting-with-a-fallback) -- [Expecting conversion](#expecting-conversion) -- [Visiting a toml::value](#visiting-a-tomlvalue) -- [Constructing a toml::value](#constructing-a-tomlvalue) -- [Preserving Comments](#preserving-comments) -- [Customizing containers](#customizing-containers) -- [TOML literal](#toml-literal) -- [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) -- [Formatting user-defined error messages](#formatting-user-defined-error-messages) -- [Obtaining location information](#obtaining-location-information) -- [Exceptions](#exceptions) -- [Colorize Error Messages](#colorize-error-messages) -- [Serializing TOML data](#serializing-toml-data) -- [Underlying types](#underlying-types) -- [Unreleased TOML features](#unreleased-toml-features) -- [Breaking Changes from v2](#breaking-changes-from-v2) -- [Running Tests](#running-tests) -- [Contributors](#contributors) -- [Licensing Terms](#licensing-terms) - -## Integration - -Just include the file after adding it to the include path. - -```cpp -#include // that's all! now you can use it. -#include - -int main() -{ - const auto data = toml::parse("example.toml"); - const auto title = toml::find(data, "title"); - std::cout << "the title is " << title << std::endl; - return 0; -} -``` - -The convenient way is to add this repository as a git-submodule or to install -it in your system by CMake. - -Note for MSVC: We recommend to set `/Zc:__cplusplus` to detect C++ version correctly. - -## Decoding a toml file - -To parse a toml file, the only thing you have to do is -to pass a filename to the `toml::parse` function. - -```cpp -const std::string fname("sample.toml"); -const toml::value data = toml::parse(fname); -``` - -As required by the TOML specification, the top-level value is always a table. -You can find a value inside it, cast it into a table explicitly, and insert it as a value into other `toml::value`. - -If it encounters an error while opening a file, it will throw `std::runtime_error`. - -You can also pass a `std::istream` to the `toml::parse` function. -To show a filename in an error message, however, it is recommended to pass the -filename with the stream. - -```cpp -std::ifstream ifs("sample.toml", std::ios_base::binary); -assert(ifs.good()); -const auto data = toml::parse(ifs, /*optional -> */ "sample.toml"); -``` - -**Note**: When you are **on Windows, open a file in binary mode**. -If a file is opened in text-mode, CRLF ("\r\n") will automatically be -converted to LF ("\n") and this causes inconsistency between file size -and the contents that would be read. This causes weird error. - -### In the case of syntax error - -If there is a syntax error in a toml file, `toml::parse` will throw -`toml::syntax_error` that inherits `std::exception`. - -toml11 has clean and informative error messages inspired by Rust and -it looks like the following. - -```console -terminate called after throwing an instance of 'toml::syntax_error' - what(): [error] toml::parse_table: invalid line format # error description - --> example.toml # file name - 3 | a = 42 = true # line num and content - | ^------ expected newline, but got '='. # error reason -``` - -If you (mistakenly) duplicate tables and got an error, it is helpful to see -where they are. toml11 shows both at the same time like the following. - -```console -terminate called after throwing an instance of 'toml::syntax_error' - what(): [error] toml::insert_value: table ("table") already exists. - --> duplicate-table.toml - 1 | [table] - | ~~~~~~~ table already exists here - ... - 3 | [table] - | ~~~~~~~ table defined twice -``` - -When toml11 encounters a malformed value, it tries to detect what type it is. -Then it shows hints to fix the format. An error message while reading one of -the malformed files in [the language agnostic test suite](https://github.com/BurntSushi/toml-test). -is shown below. - -```console -what(): [error] bad time: should be HH:MM:SS.subsec - --> ./datetime-malformed-no-secs.toml - 1 | no-secs = 1987-07-05T17:45Z - | ^------- HH:MM:SS.subsec - | -Hint: pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999 -Hint: fail: 1979-05-27T7:32:00, 1979-05-27 17:32 -``` - -You can find other examples in a job named `output_result` on -[CircleCI](https://circleci.com/gh/ToruNiina/toml11). - -Since the error message generation is generally a difficult task, the current -status is not ideal. If you encounter a weird error message, please let us know -and contribute to improve the quality! - -### Invalid UTF-8 codepoints - -It throws `syntax_error` if a value of an escape sequence -representing unicode character is not a valid UTF-8 codepoint. - -```console - what(): [error] toml::read_utf8_codepoint: input codepoint is too large. - --> utf8.toml - 1 | exceeds_unicode = "\U0011FFFF example" - | ^--------- should be in [0x00..0x10FFFF] -``` - -## Finding a toml value - -After parsing successfully, you can obtain the values from the result of -`toml::parse` using `toml::find` function. - -```toml -# sample.toml -answer = 42 -pi = 3.14 -numbers = [1,2,3] -time = 1979-05-27T07:32:00Z -``` - -``` cpp -const auto data = toml::parse("sample.toml"); -const auto answer = toml::find(data, "answer"); -const auto pi = toml::find(data, "pi"); -const auto numbers = toml::find>(data, "numbers"); -const auto timepoint = toml::find(data, "time"); -``` - -By default, `toml::find` returns a `toml::value`. - -```cpp -const toml::value& answer = toml::find(data, "answer"); -``` - -When you pass an exact TOML type that does not require type conversion, -`toml::find` returns a reference without copying the value. - -```cpp -const auto data = toml::parse("sample.toml"); -const auto& answer = toml::find(data, "answer"); -``` - -If the specified type requires conversion, you can't take a reference to the value. -See also [underlying types](#underlying-types). - -**NOTE**: For some technical reason, automatic conversion between `integer` and -`floating` is not supported. If you want to get a floating value even if a value -has integer value, you need to convert it manually after obtaining a value, -like the following. - -```cpp -const auto vx = toml::find(data, "x"); -double x = vx.is_floating() ? vx.as_floating(std::nothrow) : - static_cast(vx.as_integer()); // it throws if vx is neither - // floating nor integer. -``` - -### Finding a value in a table - -There are several way to get a value defined in a table. -First, you can get a table as a normal value and find a value from the table. - -```toml -[fruit] -name = "apple" -[fruit.physical] -color = "red" -shape = "round" -``` - -``` cpp -const auto data = toml::parse("fruit.toml"); -const auto& fruit = toml::find(data, "fruit"); -const auto name = toml::find(fruit, "name"); - -const auto& physical = toml::find(fruit, "physical"); -const auto color = toml::find(physical, "color"); -const auto shape = toml::find(physical, "shape"); -``` - -Here, variable `fruit` is a `toml::value` and can be used as the first argument -of `toml::find`. - -Second, you can pass as many arguments as the number of subtables to `toml::find`. - -```cpp -const auto data = toml::parse("fruit.toml"); -const auto color = toml::find(data, "fruit", "physical", "color"); -const auto shape = toml::find(data, "fruit", "physical", "shape"); -``` - -### Finding a value in an array - -You can find n-th value in an array by `toml::find`. - -```toml -values = ["foo", "bar", "baz"] -``` - -``` cpp -const auto data = toml::parse("sample.toml"); -const auto values = toml::find(data, "values"); -const auto bar = toml::find(values, 1); -``` - -`toml::find` can also search array recursively. - -```cpp -const auto data = toml::parse("fruit.toml"); -const auto bar = toml::find(data, "values", 1); -``` - -Before calling `toml::find`, you can check if a value corresponding to a key -exists. You can use both `bool toml::value::contains(const key&) const` and -`std::size_t toml::value::count(const key&) const`. Those behaves like the -`std::map::contains` and `std::map::count`. - -```cpp -const auto data = toml::parse("fruit.toml"); -if(data.contains("fruit") && data.at("fruit").count("physical") != 0) -{ - // ... -} -``` - -### In case of error - -If the value does not exist, `toml::find` throws `std::out_of_range` with the -location of the table. - -```console -terminate called after throwing an instance of 'std::out_of_range' - what(): [error] key "answer" not found - --> example.toml - 6 | [tab] - | ~~~~~ in this table -``` - ----- - -If the specified type differs from the actual value contained, it throws -`toml::type_error` that inherits `std::exception`. - -Similar to the case of syntax error, toml11 also displays clean error messages. -The error message when you choose `int` to get `string` value would be like this. - -```console -terminate called after throwing an instance of 'toml::type_error' - what(): [error] toml::value bad_cast to integer - --> example.toml - 3 | title = "TOML Example" - | ~~~~~~~~~~~~~~ the actual type is string -``` - -**NOTE**: In order to show this kind of error message, all the toml values have -a pointer to represent its range in a file. The entire contents of a file is -shared by `toml::value`s and remains on the heap memory. It is recommended to -destruct all the `toml::value` classes after configuring your application -if you have a large TOML file compared to the memory resource. - -### Dotted keys - -TOML v0.5.0 has a new feature named "dotted keys". -You can chain keys to represent the structure of the data. - -```toml -physical.color = "orange" -physical.shape = "round" -``` - -This is equivalent to the following. - -```toml -[physical] -color = "orange" -shape = "round" -``` - -You can get both of the above tables with the same c++ code. - -```cpp -const auto physical = toml::find(data, "physical"); -const auto color = toml::find(physical, "color"); -``` - -The following code does not work for the above toml file. - -```cpp -// XXX this does not work! -const auto color = toml::find(data, "physical.color"); -``` - -The above code works with the following toml file. - -```toml -"physical.color" = "orange" -# equivalent to {"physical.color": "orange"}, -# NOT {"physical": {"color": "orange"}}. -``` - - -## Casting a toml value - -### `toml::get` - -`toml::parse` returns `toml::value`. `toml::value` is a union type that can -contain one of the following types. - -- `toml::boolean` (`bool`) -- `toml::integer` (`std::int64_t`) -- `toml::floating` (`double`) -- `toml::string` (a type convertible to std::string) -- `toml::local_date` -- `toml::local_time` -- `toml::local_datetime` -- `toml::offset_datetime` -- `toml::array` (by default, `std::vector`) - - It depends. See [customizing containers](#customizing-containers) for detail. -- `toml::table` (by default, `std::unordered_map`) - - It depends. See [customizing containers](#customizing-containers) for detail. - -To get a value inside, you can use `toml::get()`. The usage is the same as -`toml::find` (actually, `toml::find` internally uses `toml::get` after casting -a value to `toml::table`). - -``` cpp -const toml::value data = toml::parse("sample.toml"); -const toml::value answer_ = toml::get(data).at("answer"); -const std::int64_t answer = toml::get(answer_); -``` - -When you pass an exact TOML type that does not require type conversion, -`toml::get` returns a reference through which you can modify the content -(if the `toml::value` is `const`, it returns `const` reference). - -```cpp -toml::value data = toml::parse("sample.toml"); -toml::value answer_ = toml::get(data).at("answer"); -toml::integer& answer = toml::get(answer_); -answer = 6 * 9; // write to data.answer. now `answer_` contains 54. -``` - -If the specified type requires conversion, you can't take a reference to the value. -See also [underlying types](#underlying-types). - -It also throws a `toml::type_error` if the type differs. - -### `as_xxx` - -You can also use a member function to cast a value. - -```cpp -const std::int64_t answer = data.as_table().at("answer").as_integer(); -``` - -It also throws a `toml::type_error` if the type differs. If you are sure that -the value `v` contains a value of the specified type, you can suppress checking -by passing `std::nothrow`. - -```cpp -const auto& answer = data.as_table().at("answer"); -if(answer.is_integer() && answer.as_integer(std::nothrow) == 42) -{ - std::cout << "value is 42" << std::endl; -} -``` - -If `std::nothrow` is passed, the functions are marked as noexcept. - -By casting a `toml::value` into an array or a table, you can iterate over the -elements. - -```cpp -const auto data = toml::parse("example.toml"); -std::cout << "keys in the top-level table are the following: \n"; -for(const auto& [k, v] : data.as_table()) -{ - std::cout << k << '\n'; -} - -const auto& fruits = toml::find(data, "fruits"); -for(const auto& v : fruits.as_array()) -{ - std::cout << toml::find(v, "name") << '\n'; -} -``` - -The full list of the functions is below. - -```cpp -namespace toml { -class value { - // ... - const boolean& as_boolean() const&; - const integer& as_integer() const&; - const floating& as_floating() const&; - const string& as_string() const&; - const offset_datetime& as_offset_datetime() const&; - const local_datetime& as_local_datetime() const&; - const local_date& as_local_date() const&; - const local_time& as_local_time() const&; - const array& as_array() const&; - const table& as_table() const&; - // -------------------------------------------------------- - // non-const version - boolean& as_boolean() &; - // ditto... - // -------------------------------------------------------- - // rvalue version - boolean&& as_boolean() &&; - // ditto... - - // -------------------------------------------------------- - // noexcept versions ... - const boolean& as_boolean(const std::nothrow_t&) const& noexcept; - boolean& as_boolean(const std::nothrow_t&) & noexcept; - boolean&& as_boolean(const std::nothrow_t&) && noexcept; - // ditto... -}; -} // toml -``` - -### `at()` - -You can access to the element of a table and an array by `toml::basic_value::at`. - -```cpp -const toml::value v{1,2,3,4,5}; -std::cout << v.at(2).as_integer() << std::endl; // 3 - -const toml::value v{{"foo", 42}, {"bar", 3.14}}; -std::cout << v.at("foo").as_integer() << std::endl; // 42 -``` - -If an invalid key (integer for a table, string for an array), it throws -`toml::type_error` for the conversion. If the provided key is out-of-range, -it throws `std::out_of_range`. - -Note that, although `std::string` has `at()` member function, `toml::value::at` -throws if the contained type is a string. Because `std::string` does not -contain `toml::value`. - -### `operator[]` - -You can also access to the element of a table and an array by -`toml::basic_value::operator[]`. - -```cpp -const toml::value v{1,2,3,4,5}; -std::cout << v[2].as_integer() << std::endl; // 3 - -const toml::value v{{"foo", 42}, {"bar", 3.14}}; -std::cout << v["foo"].as_integer() << std::endl; // 42 -``` - -When you access to a `toml::value` that is not initialized yet via -`operator[](const std::string&)`, the `toml::value` will be a table, -just like the `std::map`. - -```cpp -toml::value v; // not initialized as a table. -v["foo"] = 42; // OK. `v` will be a table. -``` - -Contrary, if you access to a `toml::value` that contains an array via `operator[]`, -it does not check anything. It converts `toml::value` without type check and then -access to the n-th element without boundary check, just like the `std::vector::operator[]`. - -```cpp -toml::value v; // not initialized as an array -v[2] = 42; // error! UB -``` - -Please make sure that the `toml::value` has an array inside when you access to -its element via `operator[]`. - -## Checking value type - -You can check the type of a value by `is_xxx` function. - -```cpp -const toml::value v = /* ... */; -if(v.is_integer()) -{ - std::cout << "value is an integer" << std::endl; -} -``` - -The complete list of the functions is below. - -```cpp -namespace toml { -class value { - // ... - bool is_boolean() const noexcept; - bool is_integer() const noexcept; - bool is_floating() const noexcept; - bool is_string() const noexcept; - bool is_offset_datetime() const noexcept; - bool is_local_datetime() const noexcept; - bool is_local_date() const noexcept; - bool is_local_time() const noexcept; - bool is_array() const noexcept; - bool is_table() const noexcept; - bool is_uninitialized() const noexcept; - // ... -}; -} // toml -``` - -Also, you can get `enum class value_t` from `toml::value::type()`. - -```cpp -switch(data.at("something").type()) -{ - case toml::value_t::integer: /*do some stuff*/ ; break; - case toml::value_t::floating: /*do some stuff*/ ; break; - case toml::value_t::string : /*do some stuff*/ ; break; - default : throw std::runtime_error( - "unexpected type : " + toml::stringize(data.at("something").type())); -} -``` - -The complete list of the `enum`s can be found in the section -[underlying types](#underlying-types). - -The `enum`s can be used as a parameter of `toml::value::is` function like the following. - -```cpp -toml::value v = /* ... */; -if(v.is(toml::value_t::boolean)) // ... -``` - -## More about conversion - -Since `toml::find` internally uses `toml::get`, all the following examples work -with both `toml::get` and `toml::find`. - -### Converting an array - -You can get any kind of `container` class from a `toml::array` -except for `map`-like classes. - -``` cpp -// # sample.toml -// numbers = [1,2,3] - -const auto numbers = toml::find(data, "numbers"); - -const auto vc = toml::get >(numbers); -const auto ls = toml::get >(numbers); -const auto dq = toml::get >(numbers); -const auto ar = toml::get>(numbers); -// if the size of data.at("numbers") is larger than that of std::array, -// it will throw toml::type_error because std::array is not resizable. -``` - -Surprisingly, you can convert `toml::array` into `std::pair` and `std::tuple`. - -```cpp -// numbers = [1,2,3] -const auto tp = toml::get>(numbers); -``` - -This functionality is helpful when you have a toml file like the following. - -```toml -array_of_arrays = [[1, 2, 3], ["foo", "bar", "baz"]] # toml allows this -``` - -What is the corresponding C++ type? -Obviously, it is a `std::pair` of `std::vector`s. - -```cpp -const auto array_of_arrays = toml::find(data, "array_of_arrays"); -const auto aofa = toml::get< - std::pair, std::vector> - >(array_of_arrays); -``` - -If you don't know the type of the elements, you can use `toml::array`, -which is a `std::vector` of `toml::value`, instead. - -```cpp -const auto a_of_a = toml::get(array_of_arrays); -const auto first = toml::get>(a_of_a.at(0)); -``` - -You can change the implementation of `toml::array` with `std::deque` or some -other array-like container. See [Customizing containers](#customizing-containers) -for detail. - -### Converting a table - -When all the values of the table have the same type, toml11 allows you to -convert a `toml::table` to a `map` that contains the convertible type. - -```toml -[tab] -key1 = "foo" # all the values are -key2 = "bar" # toml String -``` - -```cpp -const auto data = toml::parse("sample.toml"); -const auto tab = toml::find>(data, "tab"); -std::cout << tab["key1"] << std::endl; // foo -std::cout << tab["key2"] << std::endl; // bar -``` - -But since `toml::table` is just an alias of `std::unordered_map`, -normally you don't need to convert it because it has all the functionalities that -`std::unordered_map` has (e.g. `operator[]`, `count`, and `find`). In most cases -`toml::table` is sufficient. - -```cpp -toml::table tab = toml::get(data); -if(data.count("title") != 0) -{ - data["title"] = std::string("TOML example"); -} -``` - -You can change the implementation of `toml::table` with `std::map` or some -other map-like container. See [Customizing containers](#customizing-containers) -for detail. - -### Getting an array of tables - -An array of tables is just an array of tables. -You can get it in completely the same way as the other arrays and tables. - -```toml -# sample.toml -array_of_inline_tables = [{key = "value1"}, {key = "value2"}, {key = "value3"}] - -[[array_of_tables]] -key = "value4" -[[array_of_tables]] -key = "value5" -[[array_of_tables]] -key = "value6" -``` - -```cpp -const auto data = toml::parse("sample.toml"); -const auto aot1 = toml::find>(data, "array_of_inline_tables"); -const auto aot2 = toml::find>(data, "array_of_tables"); -``` - -### Cost of conversion - -Although conversion through `toml::(get|find)` is convenient, it has additional -copy-cost because it copies data contained in `toml::value` to the -user-specified type. Of course in some cases this overhead is not ignorable. - -```cpp -// the following code constructs a std::vector. -// it requires heap allocation for vector and element conversion. -const auto array = toml::find>(data, "foo"); -``` - -By passing the exact types, `toml::get` returns reference that has no overhead. - -``` cpp -const auto& tab = toml::find(data, "tab"); -const auto& numbers = toml::find(data, "numbers"); -``` - -Also, `as_xxx` are zero-overhead because they always return a reference. - -``` cpp -const auto& tab = toml::find(data, "tab" ).as_table(); -const auto& numbers = toml::find(data, "numbers").as_array(); -``` - -In this case you need to call `toml::get` each time you access to -the element of `toml::array` because `toml::array` is an array of `toml::value`. - -```cpp -const auto& num0 = toml::get(numbers.at(0)); -const auto& num1 = toml::get(numbers.at(1)); -const auto& num2 = toml::get(numbers.at(2)); -``` - -### Converting datetime and its variants - -TOML v0.5.0 has 4 different datetime objects, `local_date`, `local_time`, -`local_datetime`, and `offset_datetime`. - -Since `local_date`, `local_datetime`, and `offset_datetime` represent a time -point, you can convert them to `std::chrono::system_clock::time_point`. - -Contrary, `local_time` does not represents a time point because they lack a -date information, but it can be converted to `std::chrono::duration` that -represents a duration from the beginning of the day, `00:00:00.000`. - -```toml -# sample.toml -date = 2018-12-23 -time = 12:30:00 -l_dt = 2018-12-23T12:30:00 -o_dt = 2018-12-23T12:30:00+09:30 -``` - -```cpp -const auto data = toml::parse("sample.toml"); - -const auto date = toml::get(data.at("date")); -const auto l_dt = toml::get(data.at("l_dt")); -const auto o_dt = toml::get(data.at("o_dt")); - -const auto time = toml::get(data.at("time")); // 12 * 60 + 30 min -``` - -`local_date` and `local_datetime` are assumed to be in the local timezone when -they are converted into `time_point`. On the other hand, `offset_datetime` only -uses the offset part of the data and it does not take local timezone into account. - -To contain datetime data, toml11 defines its own datetime types. -For more detail, you can see the definitions in [toml/datetime.hpp](toml/datetime.hpp). - -## Getting with a fallback - -`toml::find_or` returns a default value if the value is not found or has a -different type. - -```cpp -const auto data = toml::parse("example.toml"); -const auto num = toml::find_or(data, "num", 42); -``` - -It works recursively if you pass several keys for subtables. -In that case, the last argument is considered to be the optional value. -All other arguments between `toml::value` and the optinoal value are considered as keys. - -```cpp -// [fruit.physical] -// color = "red" -auto data = toml::parse("fruit.toml"); -auto color = toml::find_or(data, "fruit", "physical", "color", "red"); -// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ -// arguments optional value -``` - -Also, `toml::get_or` returns a default value if `toml::get` failed. - -```cpp -toml::value v("foo"); // v contains String -const int value = toml::get_or(v, 42); // conversion fails. it returns 42. -``` - -These functions automatically deduce what type you want to get -from the default value you passed. - -To get a reference through this function, take care about the default value. - -```cpp -toml::value v("foo"); // v contains String -toml::integer& i = toml::get_or(v, 42); // does not work because binding `42` - // to `integer&` is invalid -toml::integer opt = 42; -toml::integer& i = toml::get_or(v, opt); // this works. -``` - -## Expecting conversion - -By using `toml::expect`, you will get your expected value or an error message -without throwing `toml::type_error`. - -```cpp -const auto value = toml::expect(data.at("title")); -if(value.is_ok()) { - std::cout << value.unwrap() << std::endl; -} else { - std::cout << value.unwrap_err() << std::endl; -} -``` - -Also, you can pass a function object to modify the expected value. - -```cpp -const auto value = toml::expect(data.at("number")) - .map(// function that receives expected type (here, int) - [](const int number) -> double { - return number * 1.5 + 1.0; - }).unwrap_or(/*default value =*/ 3.14); -``` - -## Visiting a toml::value - -toml11 provides `toml::visit` to apply a function to `toml::value` in the -same way as `std::variant`. - -```cpp -const toml::value v(3.14); -toml::visit([](const auto& val) -> void { - std::cout << val << std::endl; - }, v); -``` - -The function object that would be passed to `toml::visit` must be able to -receive all the possible TOML types. Also, the result types should be the same -each other. - -## Constructing a toml::value - -`toml::value` can be constructed in various ways. - -```cpp -toml::value v(true); // boolean -toml::value v(42); // integer -toml::value v(3.14); // floating -toml::value v("foobar"); // string -toml::value v(toml::local_date(2019, toml::month_t::Apr, 1)); // date -toml::value v{1, 2, 3, 4, 5}; // array -toml::value v{{"foo", 42}, {"bar", 3.14}, {"baz", "qux"}}; // table -``` - -When constructing a string, you can choose to use either literal or basic string. -By default, it will be a basic string. - -```cpp -toml::value v("foobar", toml::string_t::basic ); -toml::value v("foobar", toml::string_t::literal); -``` - -Datetime objects can be constructed from `std::tm` and -`std::chrono::system_clock::time_point`. But you need to specify what type -you use to avoid ambiguity. - -```cpp -const auto now = std::chrono::system_clock::now(); -toml::value v(toml::local_date(now)); -toml::value v(toml::local_datetime(now)); -toml::value v(toml::offset_datetime(now)); -``` - -Since local time is not equivalent to a time point, because it lacks date -information, it will be constructed from `std::chrono::duration`. - -```cpp -toml::value v(toml::local_time(std::chrono::hours(10))); -``` - -You can construct an array object not only from `initializer_list`, but also -from STL containers. In that case, the element type must be convertible to -`toml::value`. - -```cpp -std::vector vec{1,2,3,4,5}; -toml::value v(vec); -``` - -When you construct an array value, all the elements of `initializer_list` -must be convertible into `toml::value`. - -If a `toml::value` has an array, you can `push_back` an element in it. - -```cpp -toml::value v{1,2,3,4,5}; -v.push_back(6); -``` - -`emplace_back` also works. - -## Preserving comments - -toml11 v3 or later allows you yo choose whether comments are preserved or not via template parameter - -```cpp -const auto data1 = toml::parse("example.toml"); -const auto data2 = toml::parse("example.toml"); -``` - -or macro definition. - -```cpp -#define TOML11_PRESERVE_COMMENTS_BY_DEFAULT -#include -``` - -This feature is controlled by template parameter in `toml::basic_value<...>`. -`toml::value` is an alias of `toml::basic_value<...>`. - -If template parameter is explicitly specified, the return value of `toml::parse` -will be `toml::basic_value`. -If the macro is defined, the alias `toml::value` will be -`toml::basic_value`. - -Comments related to a value can be obtained by `toml::value::comments()`. -The return value has the same interface as `std::vector`. - -```cpp -const auto& com = v.comments(); -for(const auto& c : com) -{ - std::cout << c << std::endl; -} -``` - -Comments just before and just after (within the same line) a value are kept in a value. - -```toml -# this is a comment for v1. -v1 = "foo" - -v2 = "bar" # this is a comment for v2. -# Note that this comment is NOT a comment for v2. - -# this comment is not related to any value -# because there are empty lines between v3. -# this comment will be ignored even if you set `preserve_comments`. - -# this is a comment for v3 -# this is also a comment for v3. -v3 = "baz" # ditto. -``` - -Each comment line becomes one element of a `std::vector`. - -Hash signs will be removed, but spaces after hash sign will not be removed. - -```cpp -v1.comments().at(0) == " this is a comment for v1."s; - -v2.comments().at(1) == " this is a comment for v1."s; - -v3.comments().at(0) == " this is a comment for v3."s; -v3.comments().at(1) == " this is also a comment for v3."s; -v3.comments().at(2) == " ditto."s; -``` - -Note that a comment just after an opening brace of an array will not be a -comment for the array. - -```toml -# this is a comment for a. -a = [ # this is not a comment for a. this will be ignored. - 1, 2, 3, - # this is a comment for `42`. - 42, # this is also a comment for `42`. - 5 -] # this is a comment for a. -``` - -You can also append and modify comments. -The interfaces are the same as `std::vector`. - -```cpp -toml::basic_value v(42); -v.comments().push_back(" add this comment."); -// # add this comment. -// i = 42 -``` - -Also, you can pass a `std::vector` when constructing a -`toml::basic_value`. - -```cpp -std::vector comments{"comment 1", "comment 2"}; -const toml::basic_value v1(42, std::move(comments)); -const toml::basic_value v2(42, {"comment 1", "comment 2"}); -``` - -When `toml::discard_comments` is chosen, comments will not be contained in a value. -`value::comments()` will always be kept empty. -All the modification on comments would be ignored. -All the element access in a `discard_comments` causes the same error as accessing -an element of an empty `std::vector`. - -The comments will also be serialized. If comments exist, those comments will be -added just before the values. - -__NOTE__: Result types from `toml::parse(...)` and -`toml::parse(...)` are different. - -## Customizing containers - -Actually, `toml::basic_value` has 3 template arguments. - -```cpp -template class Table = std::unordered_map, - template class Array = std::vector> -class basic_value; -``` - -This enables you to change the containers used inside. E.g. you can use -`std::map` to contain a table object instead of `std::unordered_map`. -And also can use `std::deque` as a array object instead of `std::vector`. - -You can set these parameters while calling `toml::parse` function. - -```cpp -const auto data = toml::parse< - toml::preserve_comments, std::map, std::deque - >("example.toml"); -``` - -Needless to say, the result types from `toml::parse(...)` and -`toml::parse(...)` are different (unless you specify the same -types as default). - -Note that, since `toml::table` and `toml::array` is an alias for a table and an -array of a default `toml::value`, so it is different from the types actually -contained in a `toml::basic_value` when you customize containers. -To get the actual type in a generic way, use -`typename toml::basic_type::table_type` and -`typename toml::basic_type::array_type`. - -## TOML literal - -toml11 supports `"..."_toml` literal. -It accept both a bare value and a file content. - -```cpp -using namespace toml::literals::toml_literals; - -// `_toml` can convert a bare value without key -const toml::value v = u8"0xDEADBEEF"_toml; -// v is an Integer value containing 0xDEADBEEF. - -// raw string literal (`R"(...)"` is useful for this purpose) -const toml::value t = u8R"( - title = "this is TOML literal" - [table] - key = "value" -)"_toml; -// the literal will be parsed and the result will be contained in t -``` - -The literal function is defined in the same way as the standard library literals -such as `std::literals::string_literals::operator""s`. - -```cpp -namespace toml -{ -inline namespace literals -{ -inline namespace toml_literals -{ -toml::value operator"" _toml(const char* str, std::size_t len); -} // toml_literals -} // literals -} // toml -``` - -Access to the operator can be gained with `using namespace toml::literals;`, -`using namespace toml::toml_literals`, and `using namespace toml::literals::toml_literals`. - -Note that a key that is composed only of digits is allowed in TOML. -And, unlike the file parser, toml-literal allows a bare value without a key. -Thus it is difficult to distinguish arrays having integers and definitions of -tables that are named as digits. -Currently, literal `[1]` becomes a table named "1". -To ensure a literal to be considered as an array with one element, you need to -add a comma after the first element (like `[1,]`). - -```cpp -"[1,2,3]"_toml; // This is an array -"[table]"_toml; // This is a table that has an empty table named "table" inside. -"[[1,2,3]]"_toml; // This is an array of arrays -"[[table]]"_toml; // This is a table that has an array of tables inside. - -"[[1]]"_toml; // This literal is ambiguous. - // Currently, it becomes a table that has array of table "1". -"1 = [{}]"_toml; // This is a table that has an array of table named 1. -"[[1,]]"_toml; // This is an array of arrays. -"[[1],]"_toml; // ditto. -``` - -NOTE: `_toml` literal returns a `toml::value` that does not have comments. - -## Conversion between toml value and arbitrary types - -You can also use `toml::get` and other related functions with the types -you defined after you implement a way to convert it. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -const auto data = toml::parse("example.toml"); - -// to do this -const foo f = toml::find(data, "foo"); -``` - -There are 3 ways to use `toml::get` with the types that you defined. - -The first one is to implement `from_toml(const toml::value&)` member function. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; - - void from_toml(const toml::value& v) - { - this->a = toml::find(v, "a"); - this->b = toml::find(v, "b"); - this->c = toml::find(v, "c"); - return; - } -}; -} // ext -``` - -In this way, because `toml::get` first constructs `foo` without arguments, -the type should be default-constructible. - -The second is to implement `constructor(const toml::value&)`. - -```cpp -namespace ext -{ -struct foo -{ - explicit foo(const toml::value& v) - : a(toml::find(v, "a")), b(toml::find(v, "b")), - c(toml::find(v, "c")) - {} - - int a; - double b; - std::string c; -}; -} // ext -``` - -Note that implicit default constructor declaration will be suppressed -when a constructor is defined. If you want to use the struct (here, `foo`) -in a container (e.g. `std::vector`), you may need to define default -constructor explicitly. - -The third is to implement specialization of `toml::from` for your type. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -namespace toml -{ -template<> -struct from -{ - static ext::foo from_toml(const value& v) - { - ext::foo f; - f.a = find(v, "a"); - f.b = find(v, "b"); - f.c = find(v, "c"); - return f; - } -}; -} // toml -``` - -In this way, since the conversion function is defined outside of the class, -you can add conversion between `toml::value` and classes defined in another library. - -In some cases, a class has a templatized constructor that takes a template, `T`. -It confuses `toml::get/find` because it makes the class "constructible" from -`toml::value`. To avoid this problem, `toml::from` and `from_toml` always -precede constructor. It makes easier to implement conversion between -`toml::value` and types defined in other libraries because it skips constructor. - -But, importantly, you cannot define `toml::from` and `T.from_toml` at the same -time because it causes ambiguity in the overload resolution of `toml::get` and `toml::find`. - -So the precedence is `toml::from` == `T.from_toml()` > `T(toml::value)`. - -If you want to convert any versions of `toml::basic_value`, -you need to templatize the conversion function as follows. - -```cpp -struct foo -{ - template class M, template class A> - void from_toml(const toml::basic_value& v) - { - this->a = toml::find(v, "a"); - this->b = toml::find(v, "b"); - this->c = toml::find(v, "c"); - return; - } -}; -// or -namespace toml -{ -template<> -struct from -{ - template class M, template class A> - static ext::foo from_toml(const basic_value& v) - { - ext::foo f; - f.a = find(v, "a"); - f.b = find(v, "b"); - f.c = find(v, "c"); - return f; - } -}; -} // toml -``` - ----- - -The opposite direction is also supported in a similar way. You can directly -pass your type to `toml::value`'s constructor by introducing `into_toml` or -`toml::into`. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; - - toml::value into_toml() const // you need to mark it const. - { - return toml::value{{"a", this->a}, {"b", this->b}, {"c", this->c}}; - } -}; -} // ext - -ext::foo f{42, 3.14, "foobar"}; -toml::value v(f); -``` - -The definition of `toml::into` is similar to `toml::from`. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -namespace toml -{ -template<> -struct into -{ - static toml::value into_toml(const ext::foo& f) - { - return toml::value{{"a", f.a}, {"b", f.b}, {"c", f.c}}; - } -}; -} // toml - -ext::foo f{42, 3.14, "foobar"}; -toml::value v(f); -``` - -Any type that can be converted to `toml::value`, e.g. `int`, `toml::table` and -`toml::array` are okay to return from `into_toml`. - -You can also return a custom `toml::basic_value` from `toml::into`. - -```cpp -namespace toml -{ -template<> -struct into -{ - static toml::basic_value into_toml(const ext::foo& f) - { - toml::basic_value v{{"a", f.a}, {"b", f.b}, {"c", f.c}}; - v.comments().push_back(" comment"); - return v; - } -}; -} // toml -``` - -But note that, if this `basic_value` would be assigned into other `toml::value` -that discards `comments`, the comments would be dropped. - -### Macro to automatically define conversion functions - -There is a helper macro that automatically generates conversion functions `from` and `into` for a simple struct. - -```cpp -namespace foo -{ -struct Foo -{ - std::string s; - double d; - int i; -}; -} // foo - -TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) - -int main() -{ - const auto file = toml::parse("example.toml"); - auto f = toml::find(file, "foo"); -} -``` - -And then you can use `toml::find(file, "foo");` - -**Note** that, because of a slight difference in implementation of preprocessor between gcc/clang and MSVC, [you need to define `/Zc:preprocessor`](https://github.com/ToruNiina/toml11/issues/139#issuecomment-803683682) to use it in MSVC (Thank you @glebm !). - -## Formatting user-defined error messages - -When you encounter an error after you read the toml value, you may want to -show the error with the value. - -toml11 provides you a function that formats user-defined error message with -related values. With a code like the following, - -```cpp -const auto value = toml::find(data, "num"); -if(value < 0) -{ - std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required") - << std::endl; -} -``` - -you will get an error message like this. - -```console -[error] value should be positive - --> example.toml - 3 | num = -42 - | ~~~ positive number required -``` - -When you pass two values to `toml::format_error`, - -```cpp -const auto min = toml::find(range, "min"); -const auto max = toml::find(range, "max"); -if(max < min) -{ - std::cerr << toml::format_error("[error] max should be larger than min", - data.at("min"), "minimum number here", - data.at("max"), "maximum number here"); - << std::endl; -} -``` - -you will get an error message like this. - -```console -[error] max should be larger than min - --> example.toml - 3 | min = 54 - | ~~ minimum number here - ... - 4 | max = 42 - | ~~ maximum number here -``` - -You can print hints at the end of the message. - -```cpp -std::vector hints; -hints.push_back("positive number means n >= 0."); -hints.push_back("negative number is not positive."); -std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required", hints) - << std::endl; -``` - -```console -[error] value should be positive - --> example.toml - 2 | num = 42 - | ~~ positive number required - | -Hint: positive number means n >= 0. -Hint: negative number is not positive. -``` - -## Obtaining location information - -You can also format error messages in your own way by using `source_location`. - -```cpp -struct source_location -{ - std::uint_least32_t line() const noexcept; - std::uint_least32_t column() const noexcept; - std::uint_least32_t region() const noexcept; - std::string const& file_name() const noexcept; - std::string const& line_str() const noexcept; -}; -// +-- line() +--- length of the region (here, region() == 9) -// v .---+---. -// 12 | value = "foo bar" <- line_str() returns the line itself. -// ^-------- column() points here -``` - -You can get this by -```cpp -const toml::value v = /*...*/; -const toml::source_location loc = v.location(); -``` - -## Exceptions - -The following `exception` classes inherits `toml::exception` that inherits -`std::exception`. - -```cpp -namespace toml { -struct exception : public std::exception {/**/}; -struct syntax_error : public toml::exception {/**/}; -struct type_error : public toml::exception {/**/}; -struct internal_error : public toml::exception {/**/}; -} // toml -``` - -`toml::exception` has `toml::exception::location()` member function that returns -`toml::source_location`, in addition to `what()`. - -```cpp -namespace toml { -struct exception : public std::exception -{ - // ... - source_location const& location() const noexcept; -}; -} // toml -``` - -It represents where the error occurs. - -`syntax_error` will be thrown from `toml::parse` and `_toml` literal. -`type_error` will be thrown from `toml::get/find`, `toml::value::as_xxx()`, and -other functions that takes a content inside of `toml::value`. - -Note that, currently, from `toml::value::at()` and `toml::find(value, key)` -may throw an `std::out_of_range` that does not inherits `toml::exception`. - -Also, in some cases, most likely in the file open error, it will throw an -`std::runtime_error`. - -## Colorize Error Messages - -By defining `TOML11_COLORIZE_ERROR_MESSAGE`, the error messages from -`toml::parse` and `toml::find|get` will be colorized. By default, this feature -is turned off. - -With the following toml file taken from `toml-lang/toml/tests/hard_example.toml`, - -```toml -[error] -array = [ - "This might most likely happen in multiline arrays", - Like here, - "or here, - and here" - ] End of array comment, forgot the # -``` - -the error message would be like this. - -![error-message-1](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-1.png) - -With the following, - -```toml -[error] -# array = [ -# "This might most likely happen in multiline arrays", -# Like here, -# "or here, -# and here" -# ] End of array comment, forgot the # -number = 3.14 pi <--again forgot the # -``` - -the error message would be like this. - -![error-message-2](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-2.png) - -The message would be messy when it is written to a file, not a terminal because -it uses [ANSI escape code](https://en.wikipedia.org/wiki/ANSI_escape_code). - -Without `TOML11_COLORIZE_ERROR_MESSAGE`, you can still colorize user-defined -error message by passing `true` to the `toml::format_error` function. -If you define `TOML11_COLORIZE_ERROR_MESSAGE`, the value is `true` by default. -If not, the default value would be `false`. - -```cpp -std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required", - hints, /*colorize = */ true) << std::endl; -``` - -Note: It colorize `[error]` in red. That means that it detects `[error]` prefix -at the front of the error message. If there is no `[error]` prefix, -`format_error` adds it to the error message. - -## Serializing TOML data - -toml11 enables you to serialize data into toml format. - -```cpp -const toml::value data{{"foo", 42}, {"bar", "baz"}}; -std::cout << data << std::endl; -// bar = "baz" -// foo = 42 -``` - -toml11 automatically makes a small table and small array inline. -You can specify the width to make them inline by `std::setw` for streams. - -```cpp -const toml::value data{ - {"qux", {{"foo", 42}, {"bar", "baz"}}}, - {"quux", {"small", "array", "of", "strings"}}, - {"foobar", {"this", "array", "of", "strings", "is", "too", "long", - "to", "print", "into", "single", "line", "isn't", "it?"}}, -}; - -// the threshold becomes 80. -std::cout << std::setw(80) << data << std::endl; -// foobar = [ -// "this","array","of","strings","is","too","long","to","print","into", -// "single","line","isn't","it?", -// ] -// quux = ["small","array","of","strings"] -// qux = {bar="baz",foo=42} - - -// the width is 0. nothing become inline. -std::cout << std::setw(0) << data << std::endl; -// foobar = [ -// "this", -// ... (snip) -// "it?", -// ] -// quux = [ -// "small", -// "array", -// "of", -// "strings", -// ] -// [qux] -// bar = "baz" -// foo = 42 -``` - -It is recommended to set width before printing data. Some I/O functions changes -width to 0, and it makes all the stuff (including `toml::array`) multiline. -The resulting files becomes too long. - -To control the precision of floating point numbers, you need to pass -`std::setprecision` to stream. - -```cpp -const toml::value data{ - {"pi", 3.141592653589793}, - {"e", 2.718281828459045} -}; -std::cout << std::setprecision(17) << data << std::endl; -// e = 2.7182818284590451 -// pi = 3.1415926535897931 -std::cout << std::setprecision( 7) << data << std::endl; -// e = 2.718282 -// pi = 3.141593 -``` - -There is another way to format toml values, `toml::format()`. -It returns `std::string` that represents a value. - -```cpp -const toml::value v{{"a", 42}}; -const std::string fmt = toml::format(v); -// a = 42 -``` - -Note that since `toml::format` formats a value, the resulting string may lack -the key value. - -```cpp -const toml::value v{3.14}; -const std::string fmt = toml::format(v); -// 3.14 -``` - -To control the width and precision, `toml::format` receives optional second and -third arguments to set them. By default, the width is 80 and the precision is -`std::numeric_limits::max_digit10`. - -```cpp -const auto serial = toml::format(data, /*width = */ 0, /*prec = */ 17); -``` - -When you pass a comment-preserving-value, the comment will also be serialized. -An array or a table containing a value that has a comment would not be inlined. - -## Underlying types - -The toml types (can be used as `toml::*` in this library) and corresponding `enum` names are listed in the table below. - -| TOML type | underlying c++ type | enum class | -| -------------- | ---------------------------------- | -------------------------------- | -| Boolean | `bool` | `toml::value_t::boolean` | -| Integer | `std::int64_t` | `toml::value_t::integer` | -| Float | `double` | `toml::value_t::floating` | -| String | `toml::string` | `toml::value_t::string` | -| LocalDate | `toml::local_date` | `toml::value_t::local_date` | -| LocalTime | `toml::local_time` | `toml::value_t::local_time` | -| LocalDatetime | `toml::local_datetime` | `toml::value_t::local_datetime` | -| OffsetDatetime | `toml::offset_datetime` | `toml::value_t::offset_datetime` | -| Array | `array-like` | `toml::value_t::array` | -| Table | `map-like` | `toml::value_t::table` | - -`array-like` and `map-like` are the STL containers that works like a `std::vector` and -`std::unordered_map`, respectively. By default, `std::vector` and `std::unordered_map` -are used. See [Customizing containers](#customizing-containers) for detail. - -`toml::string` is effectively the same as `std::string` but has an additional -flag that represents a kind of a string, `string_t::basic` and `string_t::literal`. -Although `std::string` is not an exact toml type, still you can get a reference -that points to internal `std::string` by using `toml::get()` for convenience. -The most important difference between `std::string` and `toml::string` is that -`toml::string` will be formatted as a TOML string when outputted with `ostream`. -This feature is introduced to make it easy to write a custom serializer. - -`Datetime` variants are `struct` that are defined in this library. -Because `std::chrono::system_clock::time_point` is a __time point__, -not capable of representing a Local Time independent from a specific day. - -## Unreleased TOML features - -Since TOML v1.0.0-rc.1 has been released, those features are now activated by -default. We no longer need to define `TOML11_USE_UNRELEASED_FEATURES`. - -- Leading zeroes in exponent parts of floats are permitted. - - e.g. `1.0e+01`, `5e+05` - - [toml-lang/toml/PR/656](https://github.com/toml-lang/toml/pull/656) -- Allow raw tab characters in basic strings and multi-line basic strings. - - [toml-lang/toml/PR/627](https://github.com/toml-lang/toml/pull/627) -- Allow heterogeneous arrays - - [toml-lang/toml/PR/676](https://github.com/toml-lang/toml/pull/676) - -## Note about heterogeneous arrays - -Although `toml::parse` allows heterogeneous arrays, constructor of `toml::value` -does not. Here the reason is explained. - -```cpp -// this won't be compiled -toml::value v{ - "foo", 3.14, 42, {1,2,3,4,5}, {{"key", "value"}} -} -``` - -There is a workaround for this. By explicitly converting values into -`toml::value`, you can initialize `toml::value` with a heterogeneous array. -Also, you can first initialize a `toml::value` with an array and then -`push_back` into it. - -```cpp -// OK! -toml::value v{ - toml::value("foo"), toml::value(3.14), toml::value(42), - toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} -} - -// OK! -toml::value v(toml::array{}); -v.push_back("foo"); -v.push_back(3.14); - -// OK! -toml::array a; -a.push_back("foo"); -a.push_back(3.14); -toml::value v(std::move(a)); -``` - -The reason why the first example is not allowed is the following. -Let's assume that you are initializing a `toml::value` with a table. - -```cpp - // # expecting TOML table. -toml::value v{ // [v] - {"answer", 42}, // answer = 42 - {"pi", 3.14}, // pi = 3.14 - {"foo", "bar"} // foo = "bar" -}; -``` - -This is indistinguishable from a (heterogeneous) TOML array definition. - -```toml -v = [ - ["answer", 42], - ["pi", 3.14], - ["foo", "bar"], -] -``` - -This means that the above C++ code makes constructor's overload resolution -ambiguous. So a constructor that allows both "table as an initializer-list" and -"heterogeneous array as an initializer-list" cannot be implemented. - -Thus, although it is painful, we need to explicitly cast values into -`toml::value` when you initialize heterogeneous array in a C++ code. - -```cpp -toml::value v{ - toml::value("foo"), toml::value(3.14), toml::value(42), - toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} -}; -``` - -## Breaking Changes from v2 - -Although toml11 is relatively new library (it's three years old now), it had -some confusing and inconvenient user-interfaces because of historical reasons. - -Between v2 and v3, those interfaces are rearranged. - -- `toml::parse` now returns a `toml::value`, not `toml::table`. -- `toml::value` is now an alias of `toml::basic_value`. - - See [Customizing containers](#customizing-containers) for detail. -- The elements of `toml::value_t` are renamed as `snake_case`. - - See [Underlying types](#underlying-types) for detail. -- Supports for the CamelCaseNames are dropped. - - See [Underlying types](#underlying-types) for detail. -- `(is|as)_float` has been removed to make the function names consistent with others. - - Since `float` is a keyword, toml11 named a float type as `toml::floating`. - - Also a `value_t` corresponds to `toml::floating` is named `value_t::floating`. - - So `(is|as)_floating` is introduced and `is_float` has been removed. - - See [Casting a toml::value](#casting-a-tomlvalue) and [Checking value type](#checking-value-type) for detail. -- An overload of `toml::find` for `toml::table` has been dropped. Use `toml::value` version instead. - - Because type conversion between a table and a value causes ambiguity while overload resolution - - Since `toml::parse` now returns a `toml::value`, this feature becomes less important. - - Also because `toml::table` is a normal STL container, implementing utility function is easy. - - See [Finding a toml::value](#finding-a-toml-value) for detail. -- An overload of `operator<<` and `toml::format` for `toml::table`s are dropped. - - Use `toml::value` instead. - - See [Serializing TOML data](#serializing-toml-data) for detail. -- Interface around comments. - - See [Preserving Comments](#preserving-comments) for detail. -- An ancient `from_toml/into_toml` has been removed. Use arbitrary type conversion support. - - See [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) for detail. - -Such a big change will not happen in the coming years. - -## Running Tests - -After cloning this repository, run the following command (thank you @jwillikers -for automating test set fetching!). - -```sh -$ mkdir build -$ cd build -$ cmake .. -Dtoml11_BUILD_TEST=ON -$ make -$ make test -``` - -To run the language agnostic test suite, you need to compile -`tests/check_toml_test.cpp` and pass it to the tester. - -## Contributors - -I appreciate the help of the contributors who introduced the great feature to this library. - -- Guillaume Fraux (@Luthaf) - - Windows support and CI on Appvayor - - Intel Compiler support -- Quentin Khan (@xaxousis) - - Found & Fixed a bug around ODR - - Improved error messages for invalid keys to show the location where the parser fails -- Petr Beneš (@wbenny) - - Fixed warnings on MSVC -- Ivan Shynkarenka (@chronoxor) - - Fixed Visual Studio 2019 warnings -- @khoitd1997 - - Fixed warnings while type conversion -- @KerstinKeller - - Added installation script to CMake -- J.C. Moyer (@jcmoyer) - - Fixed an example code in the documentation -- Jt Freeman (@blockparty-sh) - - Fixed feature test macro around `localtime_s` - - Suppress warnings in Debug mode -- OGAWA Kenichi (@kenichiice) - - Suppress warnings on intel compiler -- Jordan Williams (@jwillikers) - - Fixed clang range-loop-analysis warnings - - Fixed feature test macro to suppress -Wundef - - Use cache variables in CMakeLists.txt - - Automate test set fetching, update and refactor CMakeLists.txt -- Scott McCaskill - - Parse 9 digits (nanoseconds) of fractional seconds in a `local_time` -- Shu Wang (@halfelf) - - fix "Finding a value in an array" example in README -- @maass-tv and @SeverinLeonhardt - - Fix MSVC warning C4866 -- OGAWA KenIchi (@kenichiice) - - Fix include path in README -- Mohammed Alyousef (@MoAlyousef) - - Made testing optional in CMake -- Ivan Shynkarenka (@chronoxor) - - Fix compilation error in `` with MinGW -- Alex Merry (@amerry) - - Add missing include files -- sneakypete81 (@sneakypete81) - - Fix typo in error message -- Oliver Kahrmann (@founderio) - - Fix missing filename in error message if parsed file is empty -- Karl Nilsson (@karl-nilsson) - - Fix many spelling errors -- ohdarling88 (@ohdarling) - - Fix a bug in a constructor of serializer -- estshorter (@estshorter) - - Fix MSVC warning C26478 -- Philip Top (@phlptp) - - Improve checking standard library feature availability check -- Louis Marascio (@marascio) - - Fix free-nonheap-object warning - - -## Licensing terms - -This product is licensed under the terms of the [MIT License](LICENSE). - -- Copyright (c) 2017-2021 Toru Niina - -All rights reserved. diff --git a/src/toml11/toml.hpp b/src/toml11/toml.hpp deleted file mode 100644 index f34cfccca..000000000 --- a/src/toml11/toml.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2017 Toru Niina - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TOML_FOR_MODERN_CPP -#define TOML_FOR_MODERN_CPP - -#ifndef __cplusplus -# error "__cplusplus is not defined" -#endif - -#if __cplusplus < 201103L && _MSC_VER < 1900 -# error "toml11 requires C++11 or later." -#endif - -#define TOML11_VERSION_MAJOR 3 -#define TOML11_VERSION_MINOR 7 -#define TOML11_VERSION_PATCH 0 - -#include "toml/parser.hpp" -#include "toml/literal.hpp" -#include "toml/serializer.hpp" -#include "toml/get.hpp" -#include "toml/macros.hpp" - -#endif// TOML_FOR_MODERN_CPP diff --git a/src/toml11/toml/color.hpp b/src/toml11/toml/color.hpp deleted file mode 100644 index 4cb572cb0..000000000 --- a/src/toml11/toml/color.hpp +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef TOML11_COLOR_HPP -#define TOML11_COLOR_HPP -#include -#include - -#ifdef TOML11_COLORIZE_ERROR_MESSAGE -#define TOML11_ERROR_MESSAGE_COLORIZED true -#else -#define TOML11_ERROR_MESSAGE_COLORIZED false -#endif - -namespace toml -{ - -// put ANSI escape sequence to ostream -namespace color_ansi -{ -namespace detail -{ -inline int colorize_index() -{ - static const int index = std::ios_base::xalloc(); - return index; -} -} // detail - -inline std::ostream& colorize(std::ostream& os) -{ - // by default, it is zero. - os.iword(detail::colorize_index()) = 1; - return os; -} -inline std::ostream& nocolorize(std::ostream& os) -{ - os.iword(detail::colorize_index()) = 0; - return os; -} -inline std::ostream& reset (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[00m";} return os;} -inline std::ostream& bold (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[01m";} return os;} -inline std::ostream& grey (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[30m";} return os;} -inline std::ostream& red (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[31m";} return os;} -inline std::ostream& green (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[32m";} return os;} -inline std::ostream& yellow (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[33m";} return os;} -inline std::ostream& blue (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[34m";} return os;} -inline std::ostream& magenta(std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[35m";} return os;} -inline std::ostream& cyan (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[36m";} return os;} -inline std::ostream& white (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[37m";} return os;} -} // color_ansi - -// ANSI escape sequence is the only and default colorization method currently -namespace color = color_ansi; - -} // toml -#endif// TOML11_COLOR_HPP diff --git a/src/toml11/toml/combinator.hpp b/src/toml11/toml/combinator.hpp deleted file mode 100644 index 33ecca1eb..000000000 --- a/src/toml11/toml/combinator.hpp +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_COMBINATOR_HPP -#define TOML11_COMBINATOR_HPP -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "region.hpp" -#include "result.hpp" -#include "traits.hpp" -#include "utility.hpp" - -// they scans characters and returns region if it matches to the condition. -// when they fail, it does not change the location. -// in lexer.hpp, these are used. - -namespace toml -{ -namespace detail -{ - -// to output character as an error message. -inline std::string show_char(const char c) -{ - // It suppresses an error that occurs only in Debug mode of MSVC++ on Windows. - // I'm not completely sure but they check the value of char to be in the - // range [0, 256) and some of the COMPLETELY VALID utf-8 character sometimes - // has negative value (if char has sign). So here it re-interprets c as - // unsigned char through pointer. In general, converting pointer to a - // pointer that has different type cause UB, but `(signed|unsigned)?char` - // are one of the exceptions. Converting pointer only to char and std::byte - // (c++17) are valid. - if(std::isgraph(*reinterpret_cast(std::addressof(c)))) - { - return std::string(1, c); - } - else - { - std::array buf; - buf.fill('\0'); - const auto r = std::snprintf( - buf.data(), buf.size(), "0x%02x", static_cast(c) & 0xFF); - (void) r; // Unused variable warning - assert(r == static_cast(buf.size()) - 1); - return std::string(buf.data()); - } -} - -template -struct character -{ - static constexpr char target = C; - - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - const auto first = loc.iter(); - - const char c = *(loc.iter()); - if(c != target) - { - return none(); - } - loc.advance(); // update location - - return ok(region(loc, first, loc.iter())); - } -}; -template -constexpr char character::target; - -// closed interval [Low, Up]. both Low and Up are included. -template -struct in_range -{ - // assuming ascii part of UTF-8... - static_assert(Low <= Up, "lower bound should be less than upper bound."); - - static constexpr char upper = Up; - static constexpr char lower = Low; - - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - const auto first = loc.iter(); - - const char c = *(loc.iter()); - if(c < lower || upper < c) - { - return none(); - } - - loc.advance(); - return ok(region(loc, first, loc.iter())); - } -}; -template constexpr char in_range::upper; -template constexpr char in_range::lower; - -// keep iterator if `Combinator` matches. otherwise, increment `iter` by 1 char. -// for detecting invalid characters, like control sequences in toml string. -template -struct exclude -{ - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - auto first = loc.iter(); - - auto rslt = Combinator::invoke(loc); - if(rslt.is_ok()) - { - loc.reset(first); - return none(); - } - loc.reset(std::next(first)); // XXX maybe loc.advance() is okay but... - return ok(region(loc, first, loc.iter())); - } -}; - -// increment `iter`, if matches. otherwise, just return empty string. -template -struct maybe -{ - static result - invoke(location& loc) - { - const auto rslt = Combinator::invoke(loc); - if(rslt.is_ok()) - { - return rslt; - } - return ok(region(loc)); - } -}; - -template -struct sequence; - -template -struct sequence -{ - static result - invoke(location& loc) - { - const auto first = loc.iter(); - auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - return sequence::invoke(loc, std::move(rslt.unwrap()), first); - } - - // called from the above function only, recursively. - template - static result - invoke(location& loc, region reg, Iterator first) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - reg += rslt.unwrap(); // concat regions - return sequence::invoke(loc, std::move(reg), first); - } -}; - -template -struct sequence -{ - // would be called from sequence::invoke only. - template - static result - invoke(location& loc, region reg, Iterator first) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - reg += rslt.unwrap(); // concat regions - return ok(reg); - } -}; - -template -struct either; - -template -struct either -{ - static result - invoke(location& loc) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_ok()) {return rslt;} - return either::invoke(loc); - } -}; -template -struct either -{ - static result - invoke(location& loc) - { - return Head::invoke(loc); - } -}; - -template -struct repeat; - -template struct exactly{}; -template struct at_least{}; -struct unlimited{}; - -template -struct repeat> -{ - static result - invoke(location& loc) - { - region retval(loc); - const auto first = loc.iter(); - for(std::size_t i=0; i -struct repeat> -{ - static result - invoke(location& loc) - { - region retval(loc); - - const auto first = loc.iter(); - for(std::size_t i=0; i -struct repeat -{ - static result - invoke(location& loc) - { - region retval(loc); - while(true) - { - auto rslt = T::invoke(loc); - if(rslt.is_err()) - { - return ok(std::move(retval)); - } - retval += rslt.unwrap(); - } - } -}; - -} // detail -} // toml -#endif// TOML11_COMBINATOR_HPP diff --git a/src/toml11/toml/comments.hpp b/src/toml11/toml/comments.hpp deleted file mode 100644 index ec2504117..000000000 --- a/src/toml11/toml/comments.hpp +++ /dev/null @@ -1,472 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_COMMENTS_HPP -#define TOML11_COMMENTS_HPP -#include -#include -#include -#include -#include -#include -#include - -#ifdef TOML11_PRESERVE_COMMENTS_BY_DEFAULT -# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::preserve_comments -#else -# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::discard_comments -#endif - -// This file provides mainly two classes, `preserve_comments` and `discard_comments`. -// Those two are a container that have the same interface as `std::vector` -// but bahaves in the opposite way. `preserve_comments` is just the same as -// `std::vector` and each `std::string` corresponds to a comment line. -// Conversely, `discard_comments` discards all the strings and ignores everything -// assigned in it. `discard_comments` is always empty and you will encounter an -// error whenever you access to the element. -namespace toml -{ -struct discard_comments; // forward decl - -// use it in the following way -// -// const toml::basic_value data = -// toml::parse("example.toml"); -// -// the interface is almost the same as std::vector. -struct preserve_comments -{ - // `container_type` is not provided in discard_comments. - // do not use this inner-type in a generic code. - using container_type = std::vector; - - using size_type = container_type::size_type; - using difference_type = container_type::difference_type; - using value_type = container_type::value_type; - using reference = container_type::reference; - using const_reference = container_type::const_reference; - using pointer = container_type::pointer; - using const_pointer = container_type::const_pointer; - using iterator = container_type::iterator; - using const_iterator = container_type::const_iterator; - using reverse_iterator = container_type::reverse_iterator; - using const_reverse_iterator = container_type::const_reverse_iterator; - - preserve_comments() = default; - ~preserve_comments() = default; - preserve_comments(preserve_comments const&) = default; - preserve_comments(preserve_comments &&) = default; - preserve_comments& operator=(preserve_comments const&) = default; - preserve_comments& operator=(preserve_comments &&) = default; - - explicit preserve_comments(const std::vector& c): comments(c){} - explicit preserve_comments(std::vector&& c) - : comments(std::move(c)) - {} - preserve_comments& operator=(const std::vector& c) - { - comments = c; - return *this; - } - preserve_comments& operator=(std::vector&& c) - { - comments = std::move(c); - return *this; - } - - explicit preserve_comments(const discard_comments&) {} - - explicit preserve_comments(size_type n): comments(n) {} - preserve_comments(size_type n, const std::string& x): comments(n, x) {} - preserve_comments(std::initializer_list x): comments(x) {} - template - preserve_comments(InputIterator first, InputIterator last) - : comments(first, last) - {} - - template - void assign(InputIterator first, InputIterator last) {comments.assign(first, last);} - void assign(std::initializer_list ini) {comments.assign(ini);} - void assign(size_type n, const std::string& val) {comments.assign(n, val);} - - // Related to the issue #97. - // - // It is known that `std::vector::insert` and `std::vector::erase` in - // the standard library implementation included in GCC 4.8.5 takes - // `std::vector::iterator` instead of `std::vector::const_iterator`. - // Because of the const-correctness, we cannot convert a `const_iterator` to - // an `iterator`. It causes compilation error in GCC 4.8.5. -#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && !defined(__clang__) -# if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) <= 40805 -# define TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION -# endif -#endif - -#ifdef TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION - iterator insert(iterator p, const std::string& x) - { - return comments.insert(p, x); - } - iterator insert(iterator p, std::string&& x) - { - return comments.insert(p, std::move(x)); - } - void insert(iterator p, size_type n, const std::string& x) - { - return comments.insert(p, n, x); - } - template - void insert(iterator p, InputIterator first, InputIterator last) - { - return comments.insert(p, first, last); - } - void insert(iterator p, std::initializer_list ini) - { - return comments.insert(p, ini); - } - - template - iterator emplace(iterator p, Ts&& ... args) - { - return comments.emplace(p, std::forward(args)...); - } - - iterator erase(iterator pos) {return comments.erase(pos);} - iterator erase(iterator first, iterator last) - { - return comments.erase(first, last); - } -#else - iterator insert(const_iterator p, const std::string& x) - { - return comments.insert(p, x); - } - iterator insert(const_iterator p, std::string&& x) - { - return comments.insert(p, std::move(x)); - } - iterator insert(const_iterator p, size_type n, const std::string& x) - { - return comments.insert(p, n, x); - } - template - iterator insert(const_iterator p, InputIterator first, InputIterator last) - { - return comments.insert(p, first, last); - } - iterator insert(const_iterator p, std::initializer_list ini) - { - return comments.insert(p, ini); - } - - template - iterator emplace(const_iterator p, Ts&& ... args) - { - return comments.emplace(p, std::forward(args)...); - } - - iterator erase(const_iterator pos) {return comments.erase(pos);} - iterator erase(const_iterator first, const_iterator last) - { - return comments.erase(first, last); - } -#endif - - void swap(preserve_comments& other) {comments.swap(other.comments);} - - void push_back(const std::string& v) {comments.push_back(v);} - void push_back(std::string&& v) {comments.push_back(std::move(v));} - void pop_back() {comments.pop_back();} - - template - void emplace_back(Ts&& ... args) {comments.emplace_back(std::forward(args)...);} - - void clear() {comments.clear();} - - size_type size() const noexcept {return comments.size();} - size_type max_size() const noexcept {return comments.max_size();} - size_type capacity() const noexcept {return comments.capacity();} - bool empty() const noexcept {return comments.empty();} - - void reserve(size_type n) {comments.reserve(n);} - void resize(size_type n) {comments.resize(n);} - void resize(size_type n, const std::string& c) {comments.resize(n, c);} - void shrink_to_fit() {comments.shrink_to_fit();} - - reference operator[](const size_type n) noexcept {return comments[n];} - const_reference operator[](const size_type n) const noexcept {return comments[n];} - reference at(const size_type n) {return comments.at(n);} - const_reference at(const size_type n) const {return comments.at(n);} - reference front() noexcept {return comments.front();} - const_reference front() const noexcept {return comments.front();} - reference back() noexcept {return comments.back();} - const_reference back() const noexcept {return comments.back();} - - pointer data() noexcept {return comments.data();} - const_pointer data() const noexcept {return comments.data();} - - iterator begin() noexcept {return comments.begin();} - iterator end() noexcept {return comments.end();} - const_iterator begin() const noexcept {return comments.begin();} - const_iterator end() const noexcept {return comments.end();} - const_iterator cbegin() const noexcept {return comments.cbegin();} - const_iterator cend() const noexcept {return comments.cend();} - - reverse_iterator rbegin() noexcept {return comments.rbegin();} - reverse_iterator rend() noexcept {return comments.rend();} - const_reverse_iterator rbegin() const noexcept {return comments.rbegin();} - const_reverse_iterator rend() const noexcept {return comments.rend();} - const_reverse_iterator crbegin() const noexcept {return comments.crbegin();} - const_reverse_iterator crend() const noexcept {return comments.crend();} - - friend bool operator==(const preserve_comments&, const preserve_comments&); - friend bool operator!=(const preserve_comments&, const preserve_comments&); - friend bool operator< (const preserve_comments&, const preserve_comments&); - friend bool operator<=(const preserve_comments&, const preserve_comments&); - friend bool operator> (const preserve_comments&, const preserve_comments&); - friend bool operator>=(const preserve_comments&, const preserve_comments&); - - friend void swap(preserve_comments&, std::vector&); - friend void swap(std::vector&, preserve_comments&); - - private: - - container_type comments; -}; - -inline bool operator==(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments == rhs.comments;} -inline bool operator!=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments != rhs.comments;} -inline bool operator< (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments < rhs.comments;} -inline bool operator<=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments <= rhs.comments;} -inline bool operator> (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments > rhs.comments;} -inline bool operator>=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments >= rhs.comments;} - -inline void swap(preserve_comments& lhs, preserve_comments& rhs) -{ - lhs.swap(rhs); - return; -} -inline void swap(preserve_comments& lhs, std::vector& rhs) -{ - lhs.comments.swap(rhs); - return; -} -inline void swap(std::vector& lhs, preserve_comments& rhs) -{ - lhs.swap(rhs.comments); - return; -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const preserve_comments& com) -{ - for(const auto& c : com) - { - os << '#' << c << '\n'; - } - return os; -} - -namespace detail -{ - -// To provide the same interface with `preserve_comments`, `discard_comments` -// should have an iterator. But it does not contain anything, so we need to -// add an iterator that points nothing. -// -// It always points null, so DO NOT unwrap this iterator. It always crashes -// your program. -template -struct empty_iterator -{ - using value_type = T; - using reference_type = typename std::conditional::type; - using pointer_type = typename std::conditional::type; - using difference_type = std::ptrdiff_t; - using iterator_category = std::random_access_iterator_tag; - - empty_iterator() = default; - ~empty_iterator() = default; - empty_iterator(empty_iterator const&) = default; - empty_iterator(empty_iterator &&) = default; - empty_iterator& operator=(empty_iterator const&) = default; - empty_iterator& operator=(empty_iterator &&) = default; - - // DO NOT call these operators. - reference_type operator*() const noexcept {std::terminate();} - pointer_type operator->() const noexcept {return nullptr;} - reference_type operator[](difference_type) const noexcept {return this->operator*();} - - // These operators do nothing. - empty_iterator& operator++() noexcept {return *this;} - empty_iterator operator++(int) noexcept {return *this;} - empty_iterator& operator--() noexcept {return *this;} - empty_iterator operator--(int) noexcept {return *this;} - - empty_iterator& operator+=(difference_type) noexcept {return *this;} - empty_iterator& operator-=(difference_type) noexcept {return *this;} - - empty_iterator operator+(difference_type) const noexcept {return *this;} - empty_iterator operator-(difference_type) const noexcept {return *this;} -}; - -template -bool operator==(const empty_iterator&, const empty_iterator&) noexcept {return true;} -template -bool operator!=(const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator< (const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator<=(const empty_iterator&, const empty_iterator&) noexcept {return true;} -template -bool operator> (const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator>=(const empty_iterator&, const empty_iterator&) noexcept {return true;} - -template -typename empty_iterator::difference_type -operator-(const empty_iterator&, const empty_iterator&) noexcept {return 0;} - -template -empty_iterator -operator+(typename empty_iterator::difference_type, const empty_iterator& rhs) noexcept {return rhs;} -template -empty_iterator -operator+(const empty_iterator& lhs, typename empty_iterator::difference_type) noexcept {return lhs;} - -} // detail - -// The default comment type. It discards all the comments. It requires only one -// byte to contain, so the memory footprint is smaller than preserve_comments. -// -// It just ignores `push_back`, `insert`, `erase`, and any other modifications. -// IT always returns size() == 0, the iterator taken by `begin()` is always the -// same as that of `end()`, and accessing through `operator[]` or iterators -// always causes a segmentation fault. DO NOT access to the element of this. -// -// Why this is chose as the default type is because the last version (2.x.y) -// does not contain any comments in a value. To minimize the impact on the -// efficiency, this is chosen as a default. -// -// To reduce the memory footprint, later we can try empty base optimization (EBO). -struct discard_comments -{ - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using value_type = std::string; - using reference = std::string&; - using const_reference = std::string const&; - using pointer = std::string*; - using const_pointer = std::string const*; - using iterator = detail::empty_iterator; - using const_iterator = detail::empty_iterator; - using reverse_iterator = detail::empty_iterator; - using const_reverse_iterator = detail::empty_iterator; - - discard_comments() = default; - ~discard_comments() = default; - discard_comments(discard_comments const&) = default; - discard_comments(discard_comments &&) = default; - discard_comments& operator=(discard_comments const&) = default; - discard_comments& operator=(discard_comments &&) = default; - - explicit discard_comments(const std::vector&) noexcept {} - explicit discard_comments(std::vector&&) noexcept {} - discard_comments& operator=(const std::vector&) noexcept {return *this;} - discard_comments& operator=(std::vector&&) noexcept {return *this;} - - explicit discard_comments(const preserve_comments&) noexcept {} - - explicit discard_comments(size_type) noexcept {} - discard_comments(size_type, const std::string&) noexcept {} - discard_comments(std::initializer_list) noexcept {} - template - discard_comments(InputIterator, InputIterator) noexcept {} - - template - void assign(InputIterator, InputIterator) noexcept {} - void assign(std::initializer_list) noexcept {} - void assign(size_type, const std::string&) noexcept {} - - iterator insert(const_iterator, const std::string&) {return iterator{};} - iterator insert(const_iterator, std::string&&) {return iterator{};} - iterator insert(const_iterator, size_type, const std::string&) {return iterator{};} - template - iterator insert(const_iterator, InputIterator, InputIterator) {return iterator{};} - iterator insert(const_iterator, std::initializer_list) {return iterator{};} - - template - iterator emplace(const_iterator, Ts&& ...) {return iterator{};} - iterator erase(const_iterator) {return iterator{};} - iterator erase(const_iterator, const_iterator) {return iterator{};} - - void swap(discard_comments&) {return;} - - void push_back(const std::string&) {return;} - void push_back(std::string&& ) {return;} - void pop_back() {return;} - - template - void emplace_back(Ts&& ...) {return;} - - void clear() {return;} - - size_type size() const noexcept {return 0;} - size_type max_size() const noexcept {return 0;} - size_type capacity() const noexcept {return 0;} - bool empty() const noexcept {return true;} - - void reserve(size_type) {return;} - void resize(size_type) {return;} - void resize(size_type, const std::string&) {return;} - void shrink_to_fit() {return;} - - // DO NOT access to the element of this container. This container is always - // empty, so accessing through operator[], front/back, data causes address - // error. - - reference operator[](const size_type) noexcept {return *data();} - const_reference operator[](const size_type) const noexcept {return *data();} - reference at(const size_type) {throw std::out_of_range("toml::discard_comment is always empty.");} - const_reference at(const size_type) const {throw std::out_of_range("toml::discard_comment is always empty.");} - reference front() noexcept {return *data();} - const_reference front() const noexcept {return *data();} - reference back() noexcept {return *data();} - const_reference back() const noexcept {return *data();} - - pointer data() noexcept {return nullptr;} - const_pointer data() const noexcept {return nullptr;} - - iterator begin() noexcept {return iterator{};} - iterator end() noexcept {return iterator{};} - const_iterator begin() const noexcept {return const_iterator{};} - const_iterator end() const noexcept {return const_iterator{};} - const_iterator cbegin() const noexcept {return const_iterator{};} - const_iterator cend() const noexcept {return const_iterator{};} - - reverse_iterator rbegin() noexcept {return iterator{};} - reverse_iterator rend() noexcept {return iterator{};} - const_reverse_iterator rbegin() const noexcept {return const_iterator{};} - const_reverse_iterator rend() const noexcept {return const_iterator{};} - const_reverse_iterator crbegin() const noexcept {return const_iterator{};} - const_reverse_iterator crend() const noexcept {return const_iterator{};} -}; - -inline bool operator==(const discard_comments&, const discard_comments&) noexcept {return true;} -inline bool operator!=(const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator< (const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator<=(const discard_comments&, const discard_comments&) noexcept {return true;} -inline bool operator> (const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator>=(const discard_comments&, const discard_comments&) noexcept {return true;} - -inline void swap(const discard_comments&, const discard_comments&) noexcept {return;} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const discard_comments&) -{ - return os; -} - -} // toml11 -#endif// TOML11_COMMENTS_HPP diff --git a/src/toml11/toml/datetime.hpp b/src/toml11/toml/datetime.hpp deleted file mode 100644 index d8127c150..000000000 --- a/src/toml11/toml/datetime.hpp +++ /dev/null @@ -1,631 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_DATETIME_HPP -#define TOML11_DATETIME_HPP -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace toml -{ - -// To avoid non-threadsafe std::localtime. In C11 (not C++11!), localtime_s is -// provided in the absolutely same purpose, but C++11 is actually not compatible -// with C11. We need to dispatch the function depending on the OS. -namespace detail -{ -// TODO: find more sophisticated way to handle this -#if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_POSIX_SOURCE) -inline std::tm localtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::localtime_r(src, &dst); - if (!result) { throw std::runtime_error("localtime_r failed."); } - return dst; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::gmtime_r(src, &dst); - if (!result) { throw std::runtime_error("gmtime_r failed."); } - return dst; -} -#elif defined(_MSC_VER) -inline std::tm localtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::localtime_s(&dst, src); - if (result) { throw std::runtime_error("localtime_s failed."); } - return dst; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::gmtime_s(&dst, src); - if (result) { throw std::runtime_error("gmtime_s failed."); } - return dst; -} -#else // fallback. not threadsafe -inline std::tm localtime_s(const std::time_t* src) -{ - const auto result = std::localtime(src); - if (!result) { throw std::runtime_error("localtime failed."); } - return *result; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - const auto result = std::gmtime(src); - if (!result) { throw std::runtime_error("gmtime failed."); } - return *result; -} -#endif -} // detail - -enum class month_t : std::uint8_t -{ - Jan = 0, - Feb = 1, - Mar = 2, - Apr = 3, - May = 4, - Jun = 5, - Jul = 6, - Aug = 7, - Sep = 8, - Oct = 9, - Nov = 10, - Dec = 11 -}; - -struct local_date -{ - std::int16_t year; // A.D. (like, 2018) - std::uint8_t month; // [0, 11] - std::uint8_t day; // [1, 31] - - local_date(int y, month_t m, int d) - : year (static_cast(y)), - month(static_cast(m)), - day (static_cast(d)) - {} - - explicit local_date(const std::tm& t) - : year (static_cast(t.tm_year + 1900)), - month(static_cast(t.tm_mon)), - day (static_cast(t.tm_mday)) - {} - - explicit local_date(const std::chrono::system_clock::time_point& tp) - { - const auto t = std::chrono::system_clock::to_time_t(tp); - const auto time = detail::localtime_s(&t); - *this = local_date(time); - } - - explicit local_date(const std::time_t t) - : local_date(std::chrono::system_clock::from_time_t(t)) - {} - - operator std::chrono::system_clock::time_point() const - { - // std::mktime returns date as local time zone. no conversion needed - std::tm t; - t.tm_sec = 0; - t.tm_min = 0; - t.tm_hour = 0; - t.tm_mday = static_cast(this->day); - t.tm_mon = static_cast(this->month); - t.tm_year = static_cast(this->year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - return std::chrono::system_clock::from_time_t(std::mktime(&t)); - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - local_date() = default; - ~local_date() = default; - local_date(local_date const&) = default; - local_date(local_date&&) = default; - local_date& operator=(local_date const&) = default; - local_date& operator=(local_date&&) = default; -}; - -inline bool operator==(const local_date& lhs, const local_date& rhs) -{ - return std::make_tuple(lhs.year, lhs.month, lhs.day) == - std::make_tuple(rhs.year, rhs.month, rhs.day); -} -inline bool operator!=(const local_date& lhs, const local_date& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_date& lhs, const local_date& rhs) -{ - return std::make_tuple(lhs.year, lhs.month, lhs.day) < - std::make_tuple(rhs.year, rhs.month, rhs.day); -} -inline bool operator<=(const local_date& lhs, const local_date& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_date& lhs, const local_date& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_date& lhs, const local_date& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_date& date) -{ - os << std::setfill('0') << std::setw(4) << static_cast(date.year ) << '-'; - os << std::setfill('0') << std::setw(2) << static_cast(date.month) + 1 << '-'; - os << std::setfill('0') << std::setw(2) << static_cast(date.day ) ; - return os; -} - -struct local_time -{ - std::uint8_t hour; // [0, 23] - std::uint8_t minute; // [0, 59] - std::uint8_t second; // [0, 60] - std::uint16_t millisecond; // [0, 999] - std::uint16_t microsecond; // [0, 999] - std::uint16_t nanosecond; // [0, 999] - - local_time(int h, int m, int s, - int ms = 0, int us = 0, int ns = 0) - : hour (static_cast(h)), - minute(static_cast(m)), - second(static_cast(s)), - millisecond(static_cast(ms)), - microsecond(static_cast(us)), - nanosecond (static_cast(ns)) - {} - - explicit local_time(const std::tm& t) - : hour (static_cast(t.tm_hour)), - minute(static_cast(t.tm_min)), - second(static_cast(t.tm_sec)), - millisecond(0), microsecond(0), nanosecond(0) - {} - - template - explicit local_time(const std::chrono::duration& t) - { - const auto h = std::chrono::duration_cast(t); - this->hour = static_cast(h.count()); - const auto t2 = t - h; - const auto m = std::chrono::duration_cast(t2); - this->minute = static_cast(m.count()); - const auto t3 = t2 - m; - const auto s = std::chrono::duration_cast(t3); - this->second = static_cast(s.count()); - const auto t4 = t3 - s; - const auto ms = std::chrono::duration_cast(t4); - this->millisecond = static_cast(ms.count()); - const auto t5 = t4 - ms; - const auto us = std::chrono::duration_cast(t5); - this->microsecond = static_cast(us.count()); - const auto t6 = t5 - us; - const auto ns = std::chrono::duration_cast(t6); - this->nanosecond = static_cast(ns.count()); - } - - operator std::chrono::nanoseconds() const - { - return std::chrono::nanoseconds (this->nanosecond) + - std::chrono::microseconds(this->microsecond) + - std::chrono::milliseconds(this->millisecond) + - std::chrono::seconds(this->second) + - std::chrono::minutes(this->minute) + - std::chrono::hours(this->hour); - } - - local_time() = default; - ~local_time() = default; - local_time(local_time const&) = default; - local_time(local_time&&) = default; - local_time& operator=(local_time const&) = default; - local_time& operator=(local_time&&) = default; -}; - -inline bool operator==(const local_time& lhs, const local_time& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) == - std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); -} -inline bool operator!=(const local_time& lhs, const local_time& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_time& lhs, const local_time& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) < - std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); -} -inline bool operator<=(const local_time& lhs, const local_time& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_time& lhs, const local_time& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_time& lhs, const local_time& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_time& time) -{ - os << std::setfill('0') << std::setw(2) << static_cast(time.hour ) << ':'; - os << std::setfill('0') << std::setw(2) << static_cast(time.minute) << ':'; - os << std::setfill('0') << std::setw(2) << static_cast(time.second); - if(time.millisecond != 0 || time.microsecond != 0 || time.nanosecond != 0) - { - os << '.'; - os << std::setfill('0') << std::setw(3) << static_cast(time.millisecond); - if(time.microsecond != 0 || time.nanosecond != 0) - { - os << std::setfill('0') << std::setw(3) << static_cast(time.microsecond); - if(time.nanosecond != 0) - { - os << std::setfill('0') << std::setw(3) << static_cast(time.nanosecond); - } - } - } - return os; -} - -struct time_offset -{ - std::int8_t hour; // [-12, 12] - std::int8_t minute; // [-59, 59] - - time_offset(int h, int m) - : hour (static_cast(h)), - minute(static_cast(m)) - {} - - operator std::chrono::minutes() const - { - return std::chrono::minutes(this->minute) + - std::chrono::hours(this->hour); - } - - time_offset() = default; - ~time_offset() = default; - time_offset(time_offset const&) = default; - time_offset(time_offset&&) = default; - time_offset& operator=(time_offset const&) = default; - time_offset& operator=(time_offset&&) = default; -}; - -inline bool operator==(const time_offset& lhs, const time_offset& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute) == - std::make_tuple(rhs.hour, rhs.minute); -} -inline bool operator!=(const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const time_offset& lhs, const time_offset& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute) < - std::make_tuple(rhs.hour, rhs.minute); -} -inline bool operator<=(const time_offset& lhs, const time_offset& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const time_offset& offset) -{ - if(offset.hour == 0 && offset.minute == 0) - { - os << 'Z'; - return os; - } - int minute = static_cast(offset.hour) * 60 + offset.minute; - if(minute < 0){os << '-'; minute = std::abs(minute);} else {os << '+';} - os << std::setfill('0') << std::setw(2) << minute / 60 << ':'; - os << std::setfill('0') << std::setw(2) << minute % 60; - return os; -} - -struct local_datetime -{ - local_date date; - local_time time; - - local_datetime(local_date d, local_time t): date(d), time(t) {} - - explicit local_datetime(const std::tm& t): date(t), time(t){} - - explicit local_datetime(const std::chrono::system_clock::time_point& tp) - { - const auto t = std::chrono::system_clock::to_time_t(tp); - std::tm ltime = detail::localtime_s(&t); - - this->date = local_date(ltime); - this->time = local_time(ltime); - - // std::tm lacks subsecond information, so diff between tp and tm - // can be used to get millisecond & microsecond information. - const auto t_diff = tp - - std::chrono::system_clock::from_time_t(std::mktime(<ime)); - this->time.millisecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - this->time.microsecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - this->time.nanosecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - } - - explicit local_datetime(const std::time_t t) - : local_datetime(std::chrono::system_clock::from_time_t(t)) - {} - - operator std::chrono::system_clock::time_point() const - { - using internal_duration = - typename std::chrono::system_clock::time_point::duration; - - // Normally DST begins at A.M. 3 or 4. If we re-use conversion operator - // of local_date and local_time independently, the conversion fails if - // it is the day when DST begins or ends. Since local_date considers the - // time is 00:00 A.M. and local_time does not consider DST because it - // does not have any date information. We need to consider both date and - // time information at the same time to convert it correctly. - - std::tm t; - t.tm_sec = static_cast(this->time.second); - t.tm_min = static_cast(this->time.minute); - t.tm_hour = static_cast(this->time.hour); - t.tm_mday = static_cast(this->date.day); - t.tm_mon = static_cast(this->date.month); - t.tm_year = static_cast(this->date.year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - - // std::mktime returns date as local time zone. no conversion needed - auto dt = std::chrono::system_clock::from_time_t(std::mktime(&t)); - dt += std::chrono::duration_cast( - std::chrono::milliseconds(this->time.millisecond) + - std::chrono::microseconds(this->time.microsecond) + - std::chrono::nanoseconds (this->time.nanosecond)); - return dt; - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - local_datetime() = default; - ~local_datetime() = default; - local_datetime(local_datetime const&) = default; - local_datetime(local_datetime&&) = default; - local_datetime& operator=(local_datetime const&) = default; - local_datetime& operator=(local_datetime&&) = default; -}; - -inline bool operator==(const local_datetime& lhs, const local_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time) == - std::make_tuple(rhs.date, rhs.time); -} -inline bool operator!=(const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_datetime& lhs, const local_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time) < - std::make_tuple(rhs.date, rhs.time); -} -inline bool operator<=(const local_datetime& lhs, const local_datetime& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_datetime& dt) -{ - os << dt.date << 'T' << dt.time; - return os; -} - -struct offset_datetime -{ - local_date date; - local_time time; - time_offset offset; - - offset_datetime(local_date d, local_time t, time_offset o) - : date(d), time(t), offset(o) - {} - offset_datetime(const local_datetime& dt, time_offset o) - : date(dt.date), time(dt.time), offset(o) - {} - explicit offset_datetime(const local_datetime& ld) - : date(ld.date), time(ld.time), offset(get_local_offset(nullptr)) - // use the current local timezone offset - {} - explicit offset_datetime(const std::chrono::system_clock::time_point& tp) - : offset(0, 0) // use gmtime - { - const auto timet = std::chrono::system_clock::to_time_t(tp); - const auto tm = detail::gmtime_s(&timet); - this->date = local_date(tm); - this->time = local_time(tm); - } - explicit offset_datetime(const std::time_t& t) - : offset(0, 0) // use gmtime - { - const auto tm = detail::gmtime_s(&t); - this->date = local_date(tm); - this->time = local_time(tm); - } - explicit offset_datetime(const std::tm& t) - : offset(0, 0) // assume gmtime - { - this->date = local_date(t); - this->time = local_time(t); - } - - operator std::chrono::system_clock::time_point() const - { - // get date-time - using internal_duration = - typename std::chrono::system_clock::time_point::duration; - - // first, convert it to local date-time information in the same way as - // local_datetime does. later we will use time_t to adjust time offset. - std::tm t; - t.tm_sec = static_cast(this->time.second); - t.tm_min = static_cast(this->time.minute); - t.tm_hour = static_cast(this->time.hour); - t.tm_mday = static_cast(this->date.day); - t.tm_mon = static_cast(this->date.month); - t.tm_year = static_cast(this->date.year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - const std::time_t tp_loc = std::mktime(std::addressof(t)); - - auto tp = std::chrono::system_clock::from_time_t(tp_loc); - tp += std::chrono::duration_cast( - std::chrono::milliseconds(this->time.millisecond) + - std::chrono::microseconds(this->time.microsecond) + - std::chrono::nanoseconds (this->time.nanosecond)); - - // Since mktime uses local time zone, it should be corrected. - // `12:00:00+09:00` means `03:00:00Z`. So mktime returns `03:00:00Z` if - // we are in `+09:00` timezone. To represent `12:00:00Z` there, we need - // to add `+09:00` to `03:00:00Z`. - // Here, it uses the time_t converted from date-time info to handle - // daylight saving time. - const auto ofs = get_local_offset(std::addressof(tp_loc)); - tp += std::chrono::hours (ofs.hour); - tp += std::chrono::minutes(ofs.minute); - - // We got `12:00:00Z` by correcting local timezone applied by mktime. - // Then we will apply the offset. Let's say `12:00:00-08:00` is given. - // And now, we have `12:00:00Z`. `12:00:00-08:00` means `20:00:00Z`. - // So we need to subtract the offset. - tp -= std::chrono::minutes(this->offset); - return tp; - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - offset_datetime() = default; - ~offset_datetime() = default; - offset_datetime(offset_datetime const&) = default; - offset_datetime(offset_datetime&&) = default; - offset_datetime& operator=(offset_datetime const&) = default; - offset_datetime& operator=(offset_datetime&&) = default; - - private: - - static time_offset get_local_offset(const std::time_t* tp) - { - // get local timezone with the same date-time information as mktime - const auto t = detail::localtime_s(tp); - - std::array buf; - const auto result = std::strftime(buf.data(), 6, "%z", &t); // +hhmm\0 - if(result != 5) - { - throw std::runtime_error("toml::offset_datetime: cannot obtain " - "timezone information of current env"); - } - const int ofs = std::atoi(buf.data()); - const int ofs_h = ofs / 100; - const int ofs_m = ofs - (ofs_h * 100); - return time_offset(ofs_h, ofs_m); - } -}; - -inline bool operator==(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time, lhs.offset) == - std::make_tuple(rhs.date, rhs.time, rhs.offset); -} -inline bool operator!=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const offset_datetime& lhs, const offset_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time, lhs.offset) < - std::make_tuple(rhs.date, rhs.time, rhs.offset); -} -inline bool operator<=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const offset_datetime& dt) -{ - os << dt.date << 'T' << dt.time << dt.offset; - return os; -} - -}//toml -#endif// TOML11_DATETIME diff --git a/src/toml11/toml/exception.hpp b/src/toml11/toml/exception.hpp deleted file mode 100644 index c64651d0a..000000000 --- a/src/toml11/toml/exception.hpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_EXCEPTION_HPP -#define TOML11_EXCEPTION_HPP -#include -#include - -#include "source_location.hpp" - -namespace toml -{ - -struct exception : public std::exception -{ - public: - explicit exception(const source_location& loc): loc_(loc) {} - virtual ~exception() noexcept override = default; - virtual const char* what() const noexcept override {return "";} - virtual source_location const& location() const noexcept {return loc_;} - - protected: - source_location loc_; -}; - -struct syntax_error : public toml::exception -{ - public: - explicit syntax_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~syntax_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -struct type_error : public toml::exception -{ - public: - explicit type_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~type_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -struct internal_error : public toml::exception -{ - public: - explicit internal_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~internal_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -} // toml -#endif // TOML_EXCEPTION diff --git a/src/toml11/toml/from.hpp b/src/toml11/toml/from.hpp deleted file mode 100644 index 10815caf5..000000000 --- a/src/toml11/toml/from.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_FROM_HPP -#define TOML11_FROM_HPP - -namespace toml -{ - -template -struct from; -// { -// static T from_toml(const toml::value& v) -// { -// // User-defined conversions ... -// } -// }; - -} // toml -#endif // TOML11_FROM_HPP diff --git a/src/toml11/toml/get.hpp b/src/toml11/toml/get.hpp deleted file mode 100644 index d7fdf553b..000000000 --- a/src/toml11/toml/get.hpp +++ /dev/null @@ -1,1117 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_GET_HPP -#define TOML11_GET_HPP -#include - -#include "from.hpp" -#include "result.hpp" -#include "value.hpp" - -namespace toml -{ - -// ============================================================================ -// exact toml::* type - -template class M, template class V> -detail::enable_if_t>::value, T> & -get(basic_value& v) -{ - return v.template cast>::value>(); -} - -template class M, template class V> -detail::enable_if_t>::value, T> const& -get(const basic_value& v) -{ - return v.template cast>::value>(); -} - -template class M, template class V> -detail::enable_if_t>::value, T> -get(basic_value&& v) -{ - return T(std::move(v).template cast>::value>()); -} - -// ============================================================================ -// T == toml::value; identity transformation. - -template class M, template class V> -inline detail::enable_if_t>::value, T>& -get(basic_value& v) -{ - return v; -} - -template class M, template class V> -inline detail::enable_if_t>::value, T> const& -get(const basic_value& v) -{ - return v; -} - -template class M, template class V> -inline detail::enable_if_t>::value, T> -get(basic_value&& v) -{ - return basic_value(std::move(v)); -} - -// ============================================================================ -// T == toml::basic_value; basic_value -> basic_value - -template class M, template class V> -inline detail::enable_if_t, - detail::negation>> - >::value, T> -get(const basic_value& v) -{ - return T(v); -} - -// ============================================================================ -// integer convertible from toml::Integer - -template class M, template class V> -inline detail::enable_if_t, // T is integral - detail::negation>, // but not bool - detail::negation< // but not toml::integer - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - return static_cast(v.as_integer()); -} - -// ============================================================================ -// floating point convertible from toml::Float - -template class M, template class V> -inline detail::enable_if_t, // T is floating_point - detail::negation< // but not toml::floating - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - return static_cast(v.as_floating()); -} - -// ============================================================================ -// std::string; toml uses its own toml::string, but it should be convertible to -// std::string seamlessly - -template class M, template class V> -inline detail::enable_if_t::value, std::string>& -get(basic_value& v) -{ - return v.as_string().str; -} - -template class M, template class V> -inline detail::enable_if_t::value, std::string> const& -get(const basic_value& v) -{ - return v.as_string().str; -} - -template class M, template class V> -inline detail::enable_if_t::value, std::string> -get(basic_value&& v) -{ - return std::string(std::move(v.as_string().str)); -} - -// ============================================================================ -// std::string_view - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 -template class M, template class V> -inline detail::enable_if_t::value, std::string_view> -get(const basic_value& v) -{ - return std::string_view(v.as_string().str); -} -#endif - -// ============================================================================ -// std::chrono::duration from toml::local_time. - -template class M, template class V> -inline detail::enable_if_t::value, T> -get(const basic_value& v) -{ - return std::chrono::duration_cast( - std::chrono::nanoseconds(v.as_local_time())); -} - -// ============================================================================ -// std::chrono::system_clock::time_point from toml::datetime variants - -template class M, template class V> -inline detail::enable_if_t< - std::is_same::value, T> -get(const basic_value& v) -{ - switch(v.type()) - { - case value_t::local_date: - { - return std::chrono::system_clock::time_point(v.as_local_date()); - } - case value_t::local_datetime: - { - return std::chrono::system_clock::time_point(v.as_local_datetime()); - } - case value_t::offset_datetime: - { - return std::chrono::system_clock::time_point(v.as_offset_datetime()); - } - default: - { - throw type_error(detail::format_underline("toml::value: " - "bad_cast to std::chrono::system_clock::time_point", { - {v.location(), concat_to_string("the actual type is ", v.type())} - }), v.location()); - } - } -} - -// ============================================================================ -// forward declaration to use this recursively. ignore this and go ahead. - -// array-like type with push_back(value) method -template class M, template class V> -detail::enable_if_t, // T is a container - detail::has_push_back_method, // T::push_back(value) works - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// array-like type without push_back(value) method -template class M, template class V> -detail::enable_if_t, // T is a container - detail::negation>, // w/o push_back(...) - detail::negation< // not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// std::pair -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// std::tuple -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// map-like classes -template class M, template class V> -detail::enable_if_t, // T is map - detail::negation< // but not toml::table - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// T.from_toml(v) -template class M, template class V> -detail::enable_if_t>>, - detail::has_from_toml_method, // but has from_toml(toml::value) - std::is_default_constructible // and default constructible - >::value, T> -get(const basic_value&); - -// toml::from::from_toml(v) -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// T(const toml::value&) and T is not toml::basic_value, -// and it does not have `from` nor `from_toml`. -template class M, template class V> -detail::enable_if_t>, - std::is_constructible&>, - detail::negation>, - detail::negation> - >::value, T> -get(const basic_value&); - -// ============================================================================ -// array-like types; most likely STL container, like std::vector, etc. - -template class M, template class V> -detail::enable_if_t, // T is a container - detail::has_push_back_method, // container.push_back(elem) works - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - const auto& ary = v.as_array(); - - T container; - try_reserve(container, ary.size()); - - for(const auto& elem : ary) - { - container.push_back(get(elem)); - } - return container; -} - -// ============================================================================ -// std::forward_list does not have push_back, insert, or emplace. -// It has insert_after, emplace_after, push_front. - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - T container; - for(const auto& elem : v.as_array()) - { - container.push_front(get(elem)); - } - container.reverse(); - return container; -} - -// ============================================================================ -// array-like types, without push_back(). most likely [std|boost]::array. - -template class M, template class V> -detail::enable_if_t, // T is a container - detail::negation>, // w/o push_back - detail::negation< // T is not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - const auto& ar = v.as_array(); - - T container; - if(ar.size() != container.size()) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified container size is ", container.size(), - " but there are ", ar.size(), " elements in toml array."), { - {v.location(), "here"} - })); - } - for(std::size_t i=0; i(ar[i]); - } - return container; -} - -// ============================================================================ -// std::pair. - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - using first_type = typename T::first_type; - using second_type = typename T::second_type; - - const auto& ar = v.as_array(); - if(ar.size() != 2) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified std::pair but there are ", ar.size(), - " elements in toml array."), {{v.location(), "here"}})); - } - return std::make_pair(::toml::get(ar.at(0)), - ::toml::get(ar.at(1))); -} - -// ============================================================================ -// std::tuple. - -namespace detail -{ -template -T get_tuple_impl(const Array& a, index_sequence) -{ - return std::make_tuple( - ::toml::get::type>(a.at(I))...); -} -} // detail - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - const auto& ar = v.as_array(); - if(ar.size() != std::tuple_size::value) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified std::tuple with ", - std::tuple_size::value, " elements, but there are ", ar.size(), - " elements in toml array."), {{v.location(), "here"}})); - } - return detail::get_tuple_impl(ar, - detail::make_index_sequence::value>{}); -} - -// ============================================================================ -// map-like types; most likely STL map, like std::map or std::unordered_map. - -template class M, template class V> -detail::enable_if_t, // T is map - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using key_type = typename T::key_type; - using mapped_type = typename T::mapped_type; - static_assert(std::is_convertible::value, - "toml::get only supports map type of which key_type is " - "convertible from std::string."); - T map; - for(const auto& kv : v.as_table()) - { - map.emplace(key_type(kv.first), get(kv.second)); - } - return map; -} - -// ============================================================================ -// user-defined, but compatible types. - -template class M, template class V> -detail::enable_if_t>>, - detail::has_from_toml_method, // but has from_toml(toml::value) memfn - std::is_default_constructible // and default constructible - >::value, T> -get(const basic_value& v) -{ - T ud; - ud.from_toml(v); - return ud; -} -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - return ::toml::from::from_toml(v); -} - -template class M, template class V> -detail::enable_if_t>, // T is not a toml::value - std::is_constructible&>, // T is constructible from toml::value - detail::negation>, // and T does not have T.from_toml(v); - detail::negation> // and T does not have toml::from{}; - >::value, T> -get(const basic_value& v) -{ - return T(v); -} - -// ============================================================================ -// find - -// ---------------------------------------------------------------------------- -// these overloads do not require to set T. and returns value itself. -template class M, template class V> -basic_value const& find(const basic_value& v, const key& ky) -{ - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return tab.at(ky); -} -template class M, template class V> -basic_value& find(basic_value& v, const key& ky) -{ - auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return tab.at(ky); -} -template class M, template class V> -basic_value find(basic_value&& v, const key& ky) -{ - typename basic_value::table_type tab = std::move(v).as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return basic_value(std::move(tab.at(ky))); -} - -// ---------------------------------------------------------------------------- -// find(value, idx) -template class M, template class V> -basic_value const& -find(const basic_value& v, const std::size_t idx) -{ - const auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ary.at(idx); -} -template class M, template class V> -basic_value& find(basic_value& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ary.at(idx); -} -template class M, template class V> -basic_value find(basic_value&& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return basic_value(std::move(ary.at(idx))); -} - -// ---------------------------------------------------------------------------- -// find(value, key); - -template class M, template class V> -decltype(::toml::get(std::declval const&>())) -find(const basic_value& v, const key& ky) -{ - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(tab.at(ky)); -} - -template class M, template class V> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, const key& ky) -{ - auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(tab.at(ky)); -} - -template class M, template class V> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, const key& ky) -{ - typename basic_value::table_type tab = std::move(v).as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(std::move(tab.at(ky))); -} - -// ---------------------------------------------------------------------------- -// find(value, idx) -template class M, template class V> -decltype(::toml::get(std::declval const&>())) -find(const basic_value& v, const std::size_t idx) -{ - const auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(ary.at(idx)); -} -template class M, template class V> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(ary.at(idx)); -} -template class M, template class V> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, const std::size_t idx) -{ - typename basic_value::array_type ary = std::move(v).as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(std::move(ary.at(idx))); -} - -// -------------------------------------------------------------------------- -// toml::find(toml::value, toml::key, Ts&& ... keys) - -namespace detail -{ -// It suppresses warnings by -Wsign-conversion. Let's say we have the following -// code. -// ```cpp -// const auto x = toml::find(data, "array", 0); -// ``` -// Here, the type of literal number `0` is `int`. `int` is a signed integer. -// `toml::find` takes `std::size_t` as an index. So it causes implicit sign -// conversion and `-Wsign-conversion` warns about it. Using `0u` instead of `0` -// suppresses the warning, but it makes user code messy. -// To suppress this warning, we need to be aware of type conversion caused -// by `toml::find(v, key1, key2, ... keys)`. But the thing is that the types of -// keys can be any combination of {string-like, size_t-like}. Of course we can't -// write down all the combinations. Thus we need to use some function that -// recognize the type of argument and cast it into `std::string` or -// `std::size_t` depending on the context. -// `key_cast` does the job. It has 2 overloads. One is invoked when the -// argument type is an integer and cast the argument into `std::size_t`. The -// other is invoked when the argument type is not an integer, possibly one of -// std::string, const char[N] or const char*, and construct std::string from -// the argument. -// `toml::find(v, k1, k2, ... ks)` uses `key_cast` before passing `ks` to -// `toml::find(v, k)` to suppress -Wsign-conversion. - -template -enable_if_t>, - negation, bool>>>::value, std::size_t> -key_cast(T&& v) noexcept -{ - return std::size_t(v); -} -template -enable_if_t>, - negation, bool>>>>::value, std::string> -key_cast(T&& v) noexcept -{ - return std::string(std::forward(v)); -} -} // detail - -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -const basic_value& -find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -basic_value& -find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -basic_value -find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(std::move(v), std::forward(k1)), - detail::key_cast(k2), std::forward(keys)...); -} - -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&>())) -find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(std::move(v), detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} - -// ============================================================================ -// get_or(value, fallback) - -template class M, template class V> -basic_value const& -get_or(const basic_value& v, const basic_value&) -{ - return v; -} -template class M, template class V> -basic_value& -get_or(basic_value& v, basic_value&) -{ - return v; -} -template class M, template class V> -basic_value -get_or(basic_value&& v, basic_value&&) -{ - return v; -} - -// ---------------------------------------------------------------------------- -// specialization for the exact toml types (return type becomes lvalue ref) - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T> const& -get_or(const basic_value& v, const T& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T>& -get_or(basic_value& v, T& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t, - basic_value>::value, detail::remove_cvref_t> -get_or(basic_value&& v, T&& opt) -{ - try - { - return get>(std::move(v)); - } - catch(...) - { - return detail::remove_cvref_t(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// specialization for std::string (return type becomes lvalue ref) - -template class M, template class V> -detail::enable_if_t, std::string>::value, - std::string> const& -get_or(const basic_value& v, const T& opt) -{ - try - { - return v.as_string().str; - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t::value, std::string>& -get_or(basic_value& v, T& opt) -{ - try - { - return v.as_string().str; - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t< - std::is_same, std::string>::value, std::string> -get_or(basic_value&& v, T&& opt) -{ - try - { - return std::move(v.as_string().str); - } - catch(...) - { - return std::string(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// specialization for string literal - -template class M, template class V> -detail::enable_if_t::type>::value, std::string> -get_or(const basic_value& v, T&& opt) -{ - try - { - return std::move(v.as_string().str); - } - catch(...) - { - return std::string(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// others (require type conversion and return type cannot be lvalue reference) - -template class M, template class V> -detail::enable_if_t, - basic_value>>, - detail::negation>>, - detail::negation::type>> - >::value, detail::remove_cvref_t> -get_or(const basic_value& v, T&& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return detail::remove_cvref_t(std::forward(opt)); - } -} - -// =========================================================================== -// find_or(value, key, fallback) - -template class M, template class V> -basic_value const& -find_or(const basic_value& v, const key& ky, - const basic_value& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return tab.at(ky); -} - -template class M, template class V> -basic_value& -find_or(basic_value& v, const toml::key& ky, basic_value& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return tab.at(ky); -} - -template class M, template class V> -basic_value -find_or(basic_value&& v, const toml::key& ky, basic_value&& opt) -{ - if(!v.is_table()) {return opt;} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return opt;} - return basic_value(std::move(tab.at(ky))); -} - -// --------------------------------------------------------------------------- -// exact types (return type can be a reference) -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T> const& -find_or(const basic_value& v, const key& ky, const T& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T>& -find_or(basic_value& v, const toml::key& ky, T& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, - detail::remove_cvref_t> -find_or(basic_value&& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(std::move(tab.at(ky)), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// std::string (return type can be a reference) - -template class M, template class V> -detail::enable_if_t::value, std::string> const& -find_or(const basic_value& v, const key& ky, const T& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} -template class M, template class V> -detail::enable_if_t::value, std::string>& -find_or(basic_value& v, const toml::key& ky, T& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} -template class M, template class V> -detail::enable_if_t::value, std::string> -find_or(basic_value&& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(std::move(tab.at(ky)), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// string literal (deduced as std::string) -template class M, template class V> -detail::enable_if_t< - detail::is_string_literal::type>::value, - std::string> -find_or(const basic_value& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::string(opt);} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return std::string(opt);} - return get_or(tab.at(ky), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// others (require type conversion and return type cannot be lvalue reference) -template class M, template class V> -detail::enable_if_t, basic_value>>, - // T is not std::string - detail::negation>>, - // T is not a string literal - detail::negation::type>> - >::value, detail::remove_cvref_t> -find_or(const basic_value& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(tab.at(ky), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// recursive find-or with type deduction (find_or(value, keys, opt)) - -template 1), std::nullptr_t> = nullptr> - // here we need to add SFINAE in the template parameter to avoid - // infinite recursion in type deduction on gcc -auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) - -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) -{ - if(!v.is_table()) - { - return detail::last_one(std::forward(keys)...); - } - auto&& tab = std::forward(v).as_table(); - if(tab.count(ky) == 0) - { - return detail::last_one(std::forward(keys)...); - } - return find_or(std::forward(tab).at(ky), std::forward(keys)...); -} - -// --------------------------------------------------------------------------- -// recursive find_or with explicit type specialization, find_or(value, keys...) - -template 1), std::nullptr_t> = nullptr> - // here we need to add SFINAE in the template parameter to avoid - // infinite recursion in type deduction on gcc -auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) - -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) -{ - if(!v.is_table()) - { - return detail::last_one(std::forward(keys)...); - } - auto&& tab = std::forward(v).as_table(); - if(tab.count(ky) == 0) - { - return detail::last_one(std::forward(keys)...); - } - return find_or(std::forward(tab).at(ky), std::forward(keys)...); -} - -// ============================================================================ -// expect - -template class M, template class V> -result expect(const basic_value& v) noexcept -{ - try - { - return ok(get(v)); - } - catch(const std::exception& e) - { - return err(e.what()); - } -} -template class M, template class V> -result -expect(const basic_value& v, const toml::key& k) noexcept -{ - try - { - return ok(find(v, k)); - } - catch(const std::exception& e) - { - return err(e.what()); - } -} - -} // toml -#endif// TOML11_GET diff --git a/src/toml11/toml/into.hpp b/src/toml11/toml/into.hpp deleted file mode 100644 index 74495560e..000000000 --- a/src/toml11/toml/into.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_INTO_HPP -#define TOML11_INTO_HPP - -namespace toml -{ - -template -struct into; -// { -// static toml::value into_toml(const T& user_defined_type) -// { -// // User-defined conversions ... -// } -// }; - -} // toml -#endif // TOML11_INTO_HPP diff --git a/src/toml11/toml/lexer.hpp b/src/toml11/toml/lexer.hpp deleted file mode 100644 index ea5050b8d..000000000 --- a/src/toml11/toml/lexer.hpp +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_LEXER_HPP -#define TOML11_LEXER_HPP -#include -#include -#include -#include - -#include "combinator.hpp" - -namespace toml -{ -namespace detail -{ - -// these scans contents from current location in a container of char -// and extract a region that matches their own pattern. -// to see the implementation of each component, see combinator.hpp. - -using lex_wschar = either, character<'\t'>>; -using lex_ws = repeat>; -using lex_newline = either, - sequence, character<'\n'>>>; -using lex_lower = in_range<'a', 'z'>; -using lex_upper = in_range<'A', 'Z'>; -using lex_alpha = either; -using lex_digit = in_range<'0', '9'>; -using lex_nonzero = in_range<'1', '9'>; -using lex_oct_dig = in_range<'0', '7'>; -using lex_bin_dig = in_range<'0', '1'>; -using lex_hex_dig = either, in_range<'a', 'f'>>; - -using lex_hex_prefix = sequence, character<'x'>>; -using lex_oct_prefix = sequence, character<'o'>>; -using lex_bin_prefix = sequence, character<'b'>>; -using lex_underscore = character<'_'>; -using lex_plus = character<'+'>; -using lex_minus = character<'-'>; -using lex_sign = either; - -// digit | nonzero 1*(digit | _ digit) -using lex_unsigned_dec_int = either>, at_least<1>>>, - lex_digit>; -// (+|-)? unsigned_dec_int -using lex_dec_int = sequence, lex_unsigned_dec_int>; - -// hex_prefix hex_dig *(hex_dig | _ hex_dig) -using lex_hex_int = sequence>, unlimited>>>; -// oct_prefix oct_dig *(oct_dig | _ oct_dig) -using lex_oct_int = sequence>, unlimited>>>; -// bin_prefix bin_dig *(bin_dig | _ bin_dig) -using lex_bin_int = sequence>, unlimited>>>; - -// (dec_int | hex_int | oct_int | bin_int) -using lex_integer = either; - -// =========================================================================== - -using lex_inf = sequence, character<'n'>, character<'f'>>; -using lex_nan = sequence, character<'a'>, character<'n'>>; -using lex_special_float = sequence, either>; - -using lex_zero_prefixable_int = sequence>, unlimited>>; - -using lex_fractional_part = sequence, lex_zero_prefixable_int>; - -using lex_exponent_part = sequence, character<'E'>>, - maybe, lex_zero_prefixable_int>; - -using lex_float = either>>>>; - -// =========================================================================== - -using lex_true = sequence, character<'r'>, - character<'u'>, character<'e'>>; -using lex_false = sequence, character<'a'>, character<'l'>, - character<'s'>, character<'e'>>; -using lex_boolean = either; - -// =========================================================================== - -using lex_date_fullyear = repeat>; -using lex_date_month = repeat>; -using lex_date_mday = repeat>; -using lex_time_delim = either, character<'t'>, character<' '>>; -using lex_time_hour = repeat>; -using lex_time_minute = repeat>; -using lex_time_second = repeat>; -using lex_time_secfrac = sequence, - repeat>>; - -using lex_time_numoffset = sequence, character<'-'>>, - sequence, - lex_time_minute>>; -using lex_time_offset = either, character<'z'>, - lex_time_numoffset>; - -using lex_partial_time = sequence, - lex_time_minute, character<':'>, - lex_time_second, maybe>; -using lex_full_date = sequence, - lex_date_month, character<'-'>, - lex_date_mday>; -using lex_full_time = sequence; - -using lex_offset_date_time = sequence; -using lex_local_date_time = sequence; -using lex_local_date = lex_full_date; -using lex_local_time = lex_partial_time; - -// =========================================================================== - -using lex_quotation_mark = character<'"'>; -using lex_basic_unescaped = exclude, // 0x09 (tab) is allowed - in_range<0x0A, 0x1F>, - character<0x22>, character<0x5C>, - character<0x7F>>>; - -using lex_escape = character<'\\'>; -using lex_escape_unicode_short = sequence, - repeat>>; -using lex_escape_unicode_long = sequence, - repeat>>; -using lex_escape_seq_char = either, character<'\\'>, - character<'b'>, character<'f'>, - character<'n'>, character<'r'>, - character<'t'>, - lex_escape_unicode_short, - lex_escape_unicode_long - >; -using lex_escaped = sequence; -using lex_basic_char = either; -using lex_basic_string = sequence, - lex_quotation_mark>; - -// After toml post-v0.5.0, it is explicitly clarified how quotes in ml-strings -// are allowed to be used. -// After this, the following strings are *explicitly* allowed. -// - One or two `"`s in a multi-line basic string is allowed wherever it is. -// - Three consecutive `"`s in a multi-line basic string is considered as a delimiter. -// - One or two `"`s can appear just before or after the delimiter. -// ```toml -// str4 = """Here are two quotation marks: "". Simple enough.""" -// str5 = """Here are three quotation marks: ""\".""" -// str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" -// str7 = """"This," she said, "is just a pointless statement."""" -// ``` -// In the current implementation (v3.3.0), it is difficult to parse `str7` in -// the above example. It is difficult to recognize `"` at the end of string body -// collectly. It will be misunderstood as a `"""` delimiter and an additional, -// invalid `"`. Like this: -// ```console -// what(): [error] toml::parse_table: invalid line format -// --> hoge.toml -// | -// 13 | str7 = """"This," she said, "is just a pointless statement."""" -// | ^- expected newline, but got '"'. -// ``` -// As a quick workaround for this problem, `lex_ml_basic_string_delim` was -// split into two, `lex_ml_basic_string_open` and `lex_ml_basic_string_close`. -// `lex_ml_basic_string_open` allows only `"""`. `_close` allows 3-5 `"`s. -// In parse_ml_basic_string() function, the trailing `"`s will be attached to -// the string body. -// -using lex_ml_basic_string_delim = repeat>; -using lex_ml_basic_string_open = lex_ml_basic_string_delim; -using lex_ml_basic_string_close = sequence< - repeat>, - maybe, maybe - >; - -using lex_ml_basic_unescaped = exclude, // 0x09 is tab - in_range<0x0A, 0x1F>, - character<0x5C>, // backslash - character<0x7F>, // DEL - lex_ml_basic_string_delim>>; - -using lex_ml_basic_escaped_newline = sequence< - lex_escape, maybe, lex_newline, - repeat, unlimited>>; - -using lex_ml_basic_char = either; -using lex_ml_basic_body = repeat, - unlimited>; -using lex_ml_basic_string = sequence; - -using lex_literal_char = exclude, in_range<0x0A, 0x1F>, - character<0x7F>, character<0x27>>>; -using lex_apostrophe = character<'\''>; -using lex_literal_string = sequence, - lex_apostrophe>; - -// the same reason as above. -using lex_ml_literal_string_delim = repeat>; -using lex_ml_literal_string_open = lex_ml_literal_string_delim; -using lex_ml_literal_string_close = sequence< - repeat>, - maybe, maybe - >; - -using lex_ml_literal_char = exclude, - in_range<0x0A, 0x1F>, - character<0x7F>, - lex_ml_literal_string_delim>>; -using lex_ml_literal_body = repeat, - unlimited>; -using lex_ml_literal_string = sequence; - -using lex_string = either; - -// =========================================================================== -using lex_dot_sep = sequence, character<'.'>, maybe>; - -using lex_unquoted_key = repeat, character<'_'>>, - at_least<1>>; -using lex_quoted_key = either; -using lex_simple_key = either; -using lex_dotted_key = sequence, - at_least<1> - > - >; -using lex_key = either; - -using lex_keyval_sep = sequence, - character<'='>, - maybe>; - -using lex_std_table_open = character<'['>; -using lex_std_table_close = character<']'>; -using lex_std_table = sequence, - lex_key, - maybe, - lex_std_table_close>; - -using lex_array_table_open = sequence; -using lex_array_table_close = sequence; -using lex_array_table = sequence, - lex_key, - maybe, - lex_array_table_close>; - -using lex_utf8_1byte = in_range<0x00, 0x7F>; -using lex_utf8_2byte = sequence< - in_range(0xC2), static_cast(0xDF)>, - in_range(0x80), static_cast(0xBF)> - >; -using lex_utf8_3byte = sequence(0xE0)>, in_range(0xA0), static_cast(0xBF)>>, - sequence(0xE1), static_cast(0xEC)>, in_range(0x80), static_cast(0xBF)>>, - sequence(0xED)>, in_range(0x80), static_cast(0x9F)>>, - sequence(0xEE), static_cast(0xEF)>, in_range(0x80), static_cast(0xBF)>> - >, in_range(0x80), static_cast(0xBF)>>; -using lex_utf8_4byte = sequence(0xF0)>, in_range(0x90), static_cast(0xBF)>>, - sequence(0xF1), static_cast(0xF3)>, in_range(0x80), static_cast(0xBF)>>, - sequence(0xF4)>, in_range(0x80), static_cast(0x8F)>> - >, in_range(0x80), static_cast(0xBF)>, - in_range(0x80), static_cast(0xBF)>>; -using lex_utf8_code = either< - lex_utf8_1byte, - lex_utf8_2byte, - lex_utf8_3byte, - lex_utf8_4byte - >; - -using lex_comment_start_symbol = character<'#'>; -using lex_non_eol_ascii = either, in_range<0x20, 0x7E>>; -using lex_comment = sequence, unlimited>>; - -} // detail -} // toml -#endif // TOML_LEXER_HPP diff --git a/src/toml11/toml/literal.hpp b/src/toml11/toml/literal.hpp deleted file mode 100644 index 04fbbc13e..000000000 --- a/src/toml11/toml/literal.hpp +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_LITERAL_HPP -#define TOML11_LITERAL_HPP -#include "parser.hpp" - -namespace toml -{ -inline namespace literals -{ -inline namespace toml_literals -{ - -// implementation -inline ::toml::basic_value -literal_internal_impl(::toml::detail::location loc) -{ - using value_type = ::toml::basic_value< - TOML11_DEFAULT_COMMENT_STRATEGY, std::unordered_map, std::vector>; - // if there are some comments or empty lines, skip them. - using skip_line = ::toml::detail::repeat, - ::toml::detail::maybe<::toml::detail::lex_comment>, - ::toml::detail::lex_newline - >, ::toml::detail::at_least<1>>; - skip_line::invoke(loc); - - // if there are some whitespaces before a value, skip them. - using skip_ws = ::toml::detail::repeat< - ::toml::detail::lex_ws, ::toml::detail::at_least<1>>; - skip_ws::invoke(loc); - - // to distinguish arrays and tables, first check it is a table or not. - // - // "[1,2,3]"_toml; // this is an array - // "[table]"_toml; // a table that has an empty table named "table" inside. - // "[[1,2,3]]"_toml; // this is an array of arrays - // "[[table]]"_toml; // this is a table that has an array of tables inside. - // - // "[[1]]"_toml; // this can be both... (currently it becomes a table) - // "1 = [{}]"_toml; // this is a table that has an array of table named 1. - // "[[1,]]"_toml; // this is an array of arrays. - // "[[1],]"_toml; // this also. - - const auto the_front = loc.iter(); - - const bool is_table_key = ::toml::detail::lex_std_table::invoke(loc); - loc.reset(the_front); - - const bool is_aots_key = ::toml::detail::lex_array_table::invoke(loc); - loc.reset(the_front); - - // If it is neither a table-key or a array-of-table-key, it may be a value. - if(!is_table_key && !is_aots_key) - { - if(auto data = ::toml::detail::parse_value(loc)) - { - return data.unwrap(); - } - } - - // Note that still it can be a table, because the literal might be something - // like the following. - // ```cpp - // R"( // c++11 raw string literals - // key = "value" - // int = 42 - // )"_toml; - // ``` - // It is a valid toml file. - // It should be parsed as if we parse a file with this content. - - if(auto data = ::toml::detail::parse_toml_file(loc)) - { - return data.unwrap(); - } - else // none of them. - { - throw ::toml::syntax_error(data.unwrap_err(), source_location(loc)); - } - -} - -inline ::toml::basic_value -operator"" _toml(const char* str, std::size_t len) -{ - ::toml::detail::location loc( - std::string("TOML literal encoded in a C++ code"), - std::vector(str, str + len)); - // literal length does not include the null character at the end. - return literal_internal_impl(std::move(loc)); -} - -// value of __cplusplus in C++2a/20 mode is not fixed yet along compilers. -// So here we use the feature test macro for `char8_t` itself. -#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L -// value of u8"" literal has been changed from char to char8_t and char8_t is -// NOT compatible to char -inline ::toml::basic_value -operator"" _toml(const char8_t* str, std::size_t len) -{ - ::toml::detail::location loc( - std::string("TOML literal encoded in a C++ code"), - std::vector(reinterpret_cast(str), - reinterpret_cast(str) + len)); - return literal_internal_impl(std::move(loc)); -} -#endif - -} // toml_literals -} // literals -} // toml -#endif//TOML11_LITERAL_HPP diff --git a/src/toml11/toml/macros.hpp b/src/toml11/toml/macros.hpp deleted file mode 100644 index e8f91aecd..000000000 --- a/src/toml11/toml/macros.hpp +++ /dev/null @@ -1,121 +0,0 @@ -#ifndef TOML11_MACROS_HPP -#define TOML11_MACROS_HPP - -#define TOML11_STRINGIZE_AUX(x) #x -#define TOML11_STRINGIZE(x) TOML11_STRINGIZE_AUX(x) - -#define TOML11_CONCATENATE_AUX(x, y) x##y -#define TOML11_CONCATENATE(x, y) TOML11_CONCATENATE_AUX(x, y) - -// ============================================================================ -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE - -#ifndef TOML11_WITHOUT_DEFINE_NON_INTRUSIVE - -// ---------------------------------------------------------------------------- -// TOML11_ARGS_SIZE - -#define TOML11_INDEX_RSEQ() \ - 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, \ - 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -#define TOML11_ARGS_SIZE_IMPL(\ - ARG1, ARG2, ARG3, ARG4, ARG5, ARG6, ARG7, ARG8, ARG9, ARG10, \ - ARG11, ARG12, ARG13, ARG14, ARG15, ARG16, ARG17, ARG18, ARG19, ARG20, \ - ARG21, ARG22, ARG23, ARG24, ARG25, ARG26, ARG27, ARG28, ARG29, ARG30, \ - ARG31, ARG32, N, ...) N -#define TOML11_ARGS_SIZE_AUX(...) TOML11_ARGS_SIZE_IMPL(__VA_ARGS__) -#define TOML11_ARGS_SIZE(...) TOML11_ARGS_SIZE_AUX(__VA_ARGS__, TOML11_INDEX_RSEQ()) - -// ---------------------------------------------------------------------------- -// TOML11_FOR_EACH_VA_ARGS - -#define TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, ARG1 ) FUNCTOR(ARG1) -#define TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_32(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, __VA_ARGS__) - -#define TOML11_FOR_EACH_VA_ARGS(FUNCTOR, ...)\ - TOML11_CONCATENATE(TOML11_FOR_EACH_VA_ARGS_AUX_, TOML11_ARGS_SIZE(__VA_ARGS__))(FUNCTOR, __VA_ARGS__) - -// ---------------------------------------------------------------------------- -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE - -// use it in the following way. -// ```cpp -// namespace foo -// { -// struct Foo -// { -// std::string s; -// double d; -// int i; -// }; -// } // foo -// -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) -// ``` -// And then you can use `toml::find(file, "foo");` -// -#define TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE(VAR_NAME)\ - obj.VAR_NAME = toml::find(v, TOML11_STRINGIZE(VAR_NAME)); - -#define TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE(VAR_NAME)\ - v[TOML11_STRINGIZE(VAR_NAME)] = obj.VAR_NAME; - -#define TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(NAME, ...)\ - namespace toml { \ - template<> \ - struct from \ - { \ - template class T, \ - template class A> \ - static NAME from_toml(const basic_value& v) \ - { \ - NAME obj; \ - TOML11_FOR_EACH_VA_ARGS(TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE, __VA_ARGS__) \ - return obj; \ - } \ - }; \ - template<> \ - struct into \ - { \ - static value into_toml(const NAME& obj) \ - { \ - ::toml::value v = ::toml::table{}; \ - TOML11_FOR_EACH_VA_ARGS(TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE, __VA_ARGS__) \ - return v; \ - } \ - }; \ - } /* toml */ - -#endif// TOML11_WITHOUT_DEFINE_NON_INTRUSIVE - -#endif// TOML11_MACROS_HPP diff --git a/src/toml11/toml/parser.hpp b/src/toml11/toml/parser.hpp deleted file mode 100644 index e31179918..000000000 --- a/src/toml11/toml/parser.hpp +++ /dev/null @@ -1,2364 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_PARSER_HPP -#define TOML11_PARSER_HPP -#include -#include -#include - -#include "combinator.hpp" -#include "lexer.hpp" -#include "region.hpp" -#include "result.hpp" -#include "types.hpp" -#include "value.hpp" - -#ifndef TOML11_DISABLE_STD_FILESYSTEM -#ifdef __cpp_lib_filesystem -#if __has_include() -#define TOML11_HAS_STD_FILESYSTEM -#include -#endif // has_include() -#endif // __cpp_lib_filesystem -#endif // TOML11_DISABLE_STD_FILESYSTEM - -namespace toml -{ -namespace detail -{ - -inline result, std::string> -parse_boolean(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_boolean::invoke(loc)) - { - const auto reg = token.unwrap(); - if (reg.str() == "true") {return ok(std::make_pair(true, reg));} - else if(reg.str() == "false") {return ok(std::make_pair(false, reg));} - else // internal error. - { - throw internal_error(format_underline( - "toml::parse_boolean: internal error", - {{source_location(reg), "invalid token"}}), - source_location(reg)); - } - } - loc.reset(first); //rollback - return err(format_underline("toml::parse_boolean: ", - {{source_location(loc), "the next token is not a boolean"}})); -} - -inline result, std::string> -parse_binary_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_bin_int::invoke(loc)) - { - auto str = token.unwrap().str(); - assert(str.size() > 2); // minimum -> 0b1 - integer retval(0), base(1); - for(auto i(str.rbegin()), e(str.rend() - 2); i!=e; ++i) - { - if (*i == '1'){retval += base; base *= 2;} - else if(*i == '0'){base *= 2;} - else if(*i == '_'){/* do nothing. */} - else // internal error. - { - throw internal_error(format_underline( - "toml::parse_integer: internal error", - {{source_location(token.unwrap()), "invalid token"}}), - source_location(loc)); - } - } - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_binary_integer:", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_octal_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_oct_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - str.erase(str.begin()); str.erase(str.begin()); // remove `0o` prefix - - std::istringstream iss(str); - integer retval(0); - iss >> std::oct >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_octal_integer:", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_hexadecimal_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_hex_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - str.erase(str.begin()); str.erase(str.begin()); // remove `0x` prefix - - std::istringstream iss(str); - integer retval(0); - iss >> std::hex >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_hexadecimal_integer", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_integer(location& loc) -{ - const auto first = loc.iter(); - if(first != loc.end() && *first == '0') - { - const auto second = std::next(first); - if(second == loc.end()) // the token is just zero. - { - loc.advance(); - return ok(std::make_pair(0, region(loc, first, second))); - } - - if(*second == 'b') {return parse_binary_integer (loc);} // 0b1100 - if(*second == 'o') {return parse_octal_integer (loc);} // 0o775 - if(*second == 'x') {return parse_hexadecimal_integer(loc);} // 0xC0FFEE - - if(std::isdigit(*second)) - { - return err(format_underline("toml::parse_integer: " - "leading zero in an Integer is not allowed.", - {{source_location(loc), "leading zero"}})); - } - else if(std::isalpha(*second)) - { - return err(format_underline("toml::parse_integer: " - "unknown integer prefix appeared.", - {{source_location(loc), "none of 0x, 0o, 0b"}})); - } - } - - if(const auto token = lex_dec_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - - std::istringstream iss(str); - integer retval(0); - iss >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_integer: ", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_floating(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_float::invoke(loc)) - { - auto str = token.unwrap().str(); - if(str == "inf" || str == "+inf") - { - if(std::numeric_limits::has_infinity) - { - return ok(std::make_pair( - std::numeric_limits::infinity(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: inf value found" - " but the current environment does not support inf. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "-inf") - { - if(std::numeric_limits::has_infinity) - { - return ok(std::make_pair( - -std::numeric_limits::infinity(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: inf value found" - " but the current environment does not support inf. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "nan" || str == "+nan") - { - if(std::numeric_limits::has_quiet_NaN) - { - return ok(std::make_pair( - std::numeric_limits::quiet_NaN(), token.unwrap())); - } - else if(std::numeric_limits::has_signaling_NaN) - { - return ok(std::make_pair( - std::numeric_limits::signaling_NaN(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: NaN value found" - " but the current environment does not support NaN. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "-nan") - { - if(std::numeric_limits::has_quiet_NaN) - { - return ok(std::make_pair( - -std::numeric_limits::quiet_NaN(), token.unwrap())); - } - else if(std::numeric_limits::has_signaling_NaN) - { - return ok(std::make_pair( - -std::numeric_limits::signaling_NaN(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: NaN value found" - " but the current environment does not support NaN. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - std::istringstream iss(str); - floating v(0.0); - iss >> v; - return ok(std::make_pair(v, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_floating: ", - {{source_location(loc), "the next token is not a float"}})); -} - -inline std::string read_utf8_codepoint(const region& reg, const location& loc) -{ - const auto str = reg.str().substr(1); - std::uint_least32_t codepoint; - std::istringstream iss(str); - iss >> std::hex >> codepoint; - - const auto to_char = [](const std::uint_least32_t i) noexcept -> char { - const auto uc = static_cast(i); - return *reinterpret_cast(std::addressof(uc)); - }; - - std::string character; - if(codepoint < 0x80) // U+0000 ... U+0079 ; just an ASCII. - { - character += static_cast(codepoint); - } - else if(codepoint < 0x800) //U+0080 ... U+07FF - { - // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 - character += to_char(0xC0| codepoint >> 6); - character += to_char(0x80|(codepoint & 0x3F)); - } - else if(codepoint < 0x10000) // U+0800...U+FFFF - { - if(0xD800 <= codepoint && codepoint <= 0xDFFF) - { - throw syntax_error(format_underline( - "toml::read_utf8_codepoint: codepoints in the range " - "[0xD800, 0xDFFF] are not valid UTF-8.", {{ - source_location(loc), "not a valid UTF-8 codepoint" - }}), source_location(loc)); - } - assert(codepoint < 0xD800 || 0xDFFF < codepoint); - // 1110yyyy 10yxxxxx 10xxxxxx - character += to_char(0xE0| codepoint >> 12); - character += to_char(0x80|(codepoint >> 6 & 0x3F)); - character += to_char(0x80|(codepoint & 0x3F)); - } - else if(codepoint < 0x110000) // U+010000 ... U+10FFFF - { - // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx - character += to_char(0xF0| codepoint >> 18); - character += to_char(0x80|(codepoint >> 12 & 0x3F)); - character += to_char(0x80|(codepoint >> 6 & 0x3F)); - character += to_char(0x80|(codepoint & 0x3F)); - } - else // out of UTF-8 region - { - throw syntax_error(format_underline("toml::read_utf8_codepoint:" - " input codepoint is too large.", - {{source_location(loc), "should be in [0x00..0x10FFFF]"}}), - source_location(loc)); - } - return character; -} - -inline result parse_escape_sequence(location& loc) -{ - const auto first = loc.iter(); - if(first == loc.end() || *first != '\\') - { - return err(format_underline("toml::parse_escape_sequence: ", {{ - source_location(loc), "the next token is not a backslash \"\\\""}})); - } - loc.advance(); - switch(*loc.iter()) - { - case '\\':{loc.advance(); return ok(std::string("\\"));} - case '"' :{loc.advance(); return ok(std::string("\""));} - case 'b' :{loc.advance(); return ok(std::string("\b"));} - case 't' :{loc.advance(); return ok(std::string("\t"));} - case 'n' :{loc.advance(); return ok(std::string("\n"));} - case 'f' :{loc.advance(); return ok(std::string("\f"));} - case 'r' :{loc.advance(); return ok(std::string("\r"));} - case 'u' : - { - if(const auto token = lex_escape_unicode_short::invoke(loc)) - { - return ok(read_utf8_codepoint(token.unwrap(), loc)); - } - else - { - return err(format_underline("parse_escape_sequence: " - "invalid token found in UTF-8 codepoint uXXXX.", - {{source_location(loc), "here"}})); - } - } - case 'U': - { - if(const auto token = lex_escape_unicode_long::invoke(loc)) - { - return ok(read_utf8_codepoint(token.unwrap(), loc)); - } - else - { - return err(format_underline("parse_escape_sequence: " - "invalid token found in UTF-8 codepoint Uxxxxxxxx", - {{source_location(loc), "here"}})); - } - } - } - - const auto msg = format_underline("parse_escape_sequence: " - "unknown escape sequence appeared.", {{source_location(loc), - "escape sequence is one of \\, \", b, t, n, f, r, uxxxx, Uxxxxxxxx"}}, - /* Hints = */{"if you want to write backslash as just one backslash, " - "use literal string like: regex = '<\\i\\c*\\s*>'"}); - loc.reset(first); - return err(msg); -} - -inline std::ptrdiff_t check_utf8_validity(const std::string& reg) -{ - location loc("tmp", reg); - const auto u8 = repeat::invoke(loc); - if(!u8 || loc.iter() != loc.end()) - { - const auto error_location = std::distance(loc.begin(), loc.iter()); - assert(0 <= error_location); - return error_location; - } - return -1; -} - -inline result, std::string> -parse_ml_basic_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_ml_basic_string::invoke(loc)) - { - auto inner_loc = loc; - inner_loc.reset(first); - - std::string retval; - retval.reserve(token.unwrap().size()); - - auto delim = lex_ml_basic_string_open::invoke(inner_loc); - if(!delim) - { - throw internal_error(format_underline( - "parse_ml_basic_string: invalid token", - {{source_location(inner_loc), "should be \"\"\""}}), - source_location(inner_loc)); - } - // immediate newline is ignored (if exists) - /* discard return value */ lex_newline::invoke(inner_loc); - - delim = none(); - while(!delim) - { - using lex_unescaped_seq = repeat< - either, unlimited>; - if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) - { - retval += unescaped.unwrap().str(); - } - if(auto escaped = parse_escape_sequence(inner_loc)) - { - retval += escaped.unwrap(); - } - if(auto esc_nl = lex_ml_basic_escaped_newline::invoke(inner_loc)) - { - // ignore newline after escape until next non-ws char - } - if(inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "parse_ml_basic_string: unexpected end of region", - {{source_location(inner_loc), "not sufficient token"}}), - source_location(inner_loc)); - } - delim = lex_ml_basic_string_close::invoke(inner_loc); - } - // `lex_ml_basic_string_close` allows 3 to 5 `"`s to allow 1 or 2 `"`s - // at just before the delimiter. Here, we need to attach `"`s at the - // end of the string body, if it exists. - // For detail, see the definition of `lex_ml_basic_string_close`. - assert(std::all_of(delim.unwrap().first(), delim.unwrap().last(), - [](const char c) noexcept {return c == '\"';})); - switch(delim.unwrap().size()) - { - case 3: {break;} - case 4: {retval += "\""; break;} - case 5: {retval += "\"\""; break;} - default: - { - throw internal_error(format_underline( - "parse_ml_basic_string: closing delimiter has invalid length", - {{source_location(inner_loc), "end of this"}}), - source_location(inner_loc)); - } - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval), token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_ml_basic_string: " - "the next token is not a valid multiline string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_basic_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_basic_string::invoke(loc)) - { - auto inner_loc = loc; - inner_loc.reset(first); - - auto quot = lex_quotation_mark::invoke(inner_loc); - if(!quot) - { - throw internal_error(format_underline("parse_basic_string: " - "invalid token", {{source_location(inner_loc), "should be \""}}), - source_location(inner_loc)); - } - - std::string retval; - retval.reserve(token.unwrap().size()); - - quot = none(); - while(!quot) - { - using lex_unescaped_seq = repeat; - if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) - { - retval += unescaped.unwrap().str(); - } - if(auto escaped = parse_escape_sequence(inner_loc)) - { - retval += escaped.unwrap(); - } - if(inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "parse_basic_string: unexpected end of region", - {{source_location(inner_loc), "not sufficient token"}}), - source_location(inner_loc)); - } - quot = lex_quotation_mark::invoke(inner_loc); - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval), token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_basic_string: " - "the next token is not a valid string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_ml_literal_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_ml_literal_string::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_ml_literal_string_open::invoke(inner_loc); - if(!open) - { - throw internal_error(format_underline( - "parse_ml_literal_string: invalid token", - {{source_location(inner_loc), "should be '''"}}), - source_location(inner_loc)); - } - // immediate newline is ignored (if exists) - /* discard return value */ lex_newline::invoke(inner_loc); - - const auto body = lex_ml_literal_body::invoke(inner_loc); - - const auto close = lex_ml_literal_string_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "parse_ml_literal_string: invalid token", - {{source_location(inner_loc), "should be '''"}}), - source_location(inner_loc)); - } - // `lex_ml_literal_string_close` allows 3 to 5 `'`s to allow 1 or 2 `'`s - // at just before the delimiter. Here, we need to attach `'`s at the - // end of the string body, if it exists. - // For detail, see the definition of `lex_ml_basic_string_close`. - - std::string retval = body.unwrap().str(); - assert(std::all_of(close.unwrap().first(), close.unwrap().last(), - [](const char c) noexcept {return c == '\'';})); - switch(close.unwrap().size()) - { - case 3: {break;} - case 4: {retval += "'"; break;} - case 5: {retval += "''"; break;} - default: - { - throw internal_error(format_underline( - "parse_ml_literal_string: closing delimiter has invalid length", - {{source_location(inner_loc), "end of this"}}), - source_location(inner_loc)); - } - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval, toml::string_t::literal), - token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_ml_literal_string: " - "the next token is not a valid multiline literal string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_literal_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_literal_string::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_apostrophe::invoke(inner_loc); - if(!open) - { - throw internal_error(format_underline( - "parse_literal_string: invalid token", - {{source_location(inner_loc), "should be '"}}), - source_location(inner_loc)); - } - - const auto body = repeat::invoke(inner_loc); - - const auto close = lex_apostrophe::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "parse_literal_string: invalid token", - {{source_location(inner_loc), "should be '"}}), - source_location(inner_loc)); - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair( - toml::string(body.unwrap().str(), toml::string_t::literal), - token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_literal_string: " - "the next token is not a valid literal string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_string(location& loc) -{ - if(loc.iter() != loc.end() && *(loc.iter()) == '"') - { - if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '"' && - loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '"') - { - return parse_ml_basic_string(loc); - } - else - { - return parse_basic_string(loc); - } - } - else if(loc.iter() != loc.end() && *(loc.iter()) == '\'') - { - if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '\'' && - loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '\'') - { - return parse_ml_literal_string(loc); - } - else - { - return parse_literal_string(loc); - } - } - return err(format_underline("toml::parse_string: ", - {{source_location(loc), "the next token is not a string"}})); -} - -inline result, std::string> -parse_local_date(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_date::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto y = lex_date_fullyear::invoke(inner_loc); - if(!y || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') - { - throw internal_error(format_underline( - "toml::parse_inner_local_date: invalid year format", - {{source_location(inner_loc), "should be `-`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto m = lex_date_month::invoke(inner_loc); - if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') - { - throw internal_error(format_underline( - "toml::parse_local_date: invalid month format", - {{source_location(inner_loc), "should be `-`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto d = lex_date_mday::invoke(inner_loc); - if(!d) - { - throw internal_error(format_underline( - "toml::parse_local_date: invalid day format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - - const auto year = static_cast(from_string(y.unwrap().str(), 0)); - const auto month = static_cast(from_string(m.unwrap().str(), 0)); - const auto day = static_cast(from_string(d.unwrap().str(), 0)); - - // We briefly check whether the input date is valid or not. But here, we - // only check if the RFC3339 compliance. - // Actually there are several special date that does not exist, - // because of historical reasons, such as 1582/10/5-1582/10/14 (only in - // several countries). But here, we do not care about such a complicated - // rule. It makes the code complicated and there is only low probability - // that such a specific date is needed in practice. If someone need to - // validate date accurately, that means that the one need a specialized - // library for their purpose in a different layer. - { - const bool is_leap = (year % 4 == 0) && ((year % 100 != 0) || (year % 400 == 0)); - const auto max_day = (month == 2) ? (is_leap ? 29 : 28) : - ((month == 4 || month == 6 || month == 9 || month == 11) ? 30 : 31); - - if((month < 1 || 12 < month) || (day < 1 || max_day < day)) - { - throw syntax_error(format_underline("toml::parse_date: " - "invalid date: it does not conform RFC3339.", {{ - source_location(loc), "month should be 01-12, day should be" - " 01-28,29,30,31, depending on month/year." - }}), source_location(inner_loc)); - } - } - return ok(std::make_pair(local_date(year, static_cast(month - 1), day), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_date: ", - {{source_location(loc), "the next token is not a local_date"}})); - } -} - -inline result, std::string> -parse_local_time(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto h = lex_time_hour::invoke(inner_loc); - if(!h || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid year format", - {{source_location(inner_loc), "should be `:`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto m = lex_time_minute::invoke(inner_loc); - if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid month format", - {{source_location(inner_loc), "should be `:`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto s = lex_time_second::invoke(inner_loc); - if(!s) - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid second format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - - const int hour = from_string(h.unwrap().str(), 0); - const int minute = from_string(m.unwrap().str(), 0); - const int second = from_string(s.unwrap().str(), 0); - - if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute) || - (second < 0 || 60 < second)) // it may be leap second - { - throw syntax_error(format_underline("toml::parse_time: " - "invalid time: it does not conform RFC3339.", {{ - source_location(loc), "hour should be 00-23, minute should be" - " 00-59, second should be 00-60 (depending on the leap" - " second rules.)"}}), source_location(inner_loc)); - } - - local_time time(hour, minute, second, 0, 0); - - const auto before_secfrac = inner_loc.iter(); - if(const auto secfrac = lex_time_secfrac::invoke(inner_loc)) - { - auto sf = secfrac.unwrap().str(); - sf.erase(sf.begin()); // sf.front() == '.' - switch(sf.size() % 3) - { - case 2: sf += '0'; break; - case 1: sf += "00"; break; - case 0: break; - default: break; - } - if(sf.size() >= 9) - { - time.millisecond = from_string(sf.substr(0, 3), 0u); - time.microsecond = from_string(sf.substr(3, 3), 0u); - time.nanosecond = from_string(sf.substr(6, 3), 0u); - } - else if(sf.size() >= 6) - { - time.millisecond = from_string(sf.substr(0, 3), 0u); - time.microsecond = from_string(sf.substr(3, 3), 0u); - } - else if(sf.size() >= 3) - { - time.millisecond = from_string(sf, 0u); - time.microsecond = 0u; - } - } - else - { - if(before_secfrac != inner_loc.iter()) - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid subsecond format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - return ok(std::make_pair(time, token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_time: ", - {{source_location(loc), "the next token is not a local_time"}})); - } -} - -inline result, std::string> -parse_local_datetime(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_date_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - const auto date = parse_local_date(inner_loc); - if(!date || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "date, not datetime"}}), - source_location(inner_loc)); - } - const char delim = *(inner_loc.iter()); - if(delim != 'T' && delim != 't' && delim != ' ') - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "should be `T` or ` ` (space)"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto time = parse_local_time(inner_loc); - if(!time) - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "invalid time format"}}), - source_location(inner_loc)); - } - return ok(std::make_pair( - local_datetime(date.unwrap().first, time.unwrap().first), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_datetime: ", - {{source_location(loc), "the next token is not a local_datetime"}})); - } -} - -inline result, std::string> -parse_offset_datetime(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_offset_date_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - const auto datetime = parse_local_datetime(inner_loc); - if(!datetime || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_offset_datetime: invalid datetime format", - {{source_location(inner_loc), "date, not datetime"}}), - source_location(inner_loc)); - } - time_offset offset(0, 0); - if(const auto ofs = lex_time_numoffset::invoke(inner_loc)) - { - const auto str = ofs.unwrap().str(); - - const auto hour = from_string(str.substr(1,2), 0); - const auto minute = from_string(str.substr(4,2), 0); - - if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute)) - { - throw syntax_error(format_underline("toml::parse_offset_datetime: " - "invalid offset: it does not conform RFC3339.", {{ - source_location(loc), "month should be 01-12, day should be" - " 01-28,29,30,31, depending on month/year." - }}), source_location(inner_loc)); - } - - if(str.front() == '+') - { - offset = time_offset(hour, minute); - } - else - { - offset = time_offset(-hour, -minute); - } - } - else if(*inner_loc.iter() != 'Z' && *inner_loc.iter() != 'z') - { - throw internal_error(format_underline( - "toml::parse_offset_datetime: invalid datetime format", - {{source_location(inner_loc), "should be `Z` or `+HH:MM`"}}), - source_location(inner_loc)); - } - return ok(std::make_pair(offset_datetime(datetime.unwrap().first, offset), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_offset_datetime: ", - {{source_location(loc), "the next token is not a offset_datetime"}})); - } -} - -inline result, std::string> -parse_simple_key(location& loc) -{ - if(const auto bstr = parse_basic_string(loc)) - { - return ok(std::make_pair(bstr.unwrap().first.str, bstr.unwrap().second)); - } - if(const auto lstr = parse_literal_string(loc)) - { - return ok(std::make_pair(lstr.unwrap().first.str, lstr.unwrap().second)); - } - if(const auto bare = lex_unquoted_key::invoke(loc)) - { - const auto reg = bare.unwrap(); - return ok(std::make_pair(reg.str(), reg)); - } - return err(format_underline("toml::parse_simple_key: ", - {{source_location(loc), "the next token is not a simple key"}})); -} - -// dotted key become vector of keys -inline result, region>, std::string> -parse_key(location& loc) -{ - const auto first = loc.iter(); - // dotted key -> `foo.bar.baz` where several single keys are chained by - // dots. Whitespaces between keys and dots are allowed. - if(const auto token = lex_dotted_key::invoke(loc)) - { - const auto reg = token.unwrap(); - location inner_loc(loc.name(), reg.str()); - std::vector keys; - - while(inner_loc.iter() != inner_loc.end()) - { - lex_ws::invoke(inner_loc); - if(const auto k = parse_simple_key(inner_loc)) - { - keys.push_back(k.unwrap().first); - } - else - { - throw internal_error(format_underline( - "toml::detail::parse_key: dotted key contains invalid key", - {{source_location(inner_loc), k.unwrap_err()}}), - source_location(inner_loc)); - } - - lex_ws::invoke(inner_loc); - if(inner_loc.iter() == inner_loc.end()) - { - break; - } - else if(*inner_loc.iter() == '.') - { - inner_loc.advance(); // to skip `.` - } - else - { - throw internal_error(format_underline("toml::parse_key: " - "dotted key contains invalid key ", - {{source_location(inner_loc), "should be `.`"}}), - source_location(inner_loc)); - } - } - return ok(std::make_pair(keys, reg)); - } - loc.reset(first); - - // simple_key: a single (basic_string|literal_string|bare key) - if(const auto smpl = parse_simple_key(loc)) - { - return ok(std::make_pair(std::vector(1, smpl.unwrap().first), - smpl.unwrap().second)); - } - return err(format_underline("toml::parse_key: an invalid key appeared.", - {{source_location(loc), "is not a valid key"}}, { - "bare keys : non-empty strings composed only of [A-Za-z0-9_-].", - "quoted keys: same as \"basic strings\" or 'literal strings'.", - "dotted keys: sequence of bare or quoted keys joined with a dot." - })); -} - -// forward-decl to implement parse_array and parse_table -template -result parse_value(location&); - -template -result, std::string> -parse_array(location& loc) -{ - using value_type = Value; - using array_type = typename value_type::array_type; - - const auto first = loc.iter(); - if(loc.iter() == loc.end()) - { - return err("toml::parse_array: input is empty"); - } - if(*loc.iter() != '[') - { - return err("toml::parse_array: token is not an array"); - } - loc.advance(); - - using lex_ws_comment_newline = repeat< - either, unlimited>; - - array_type retval; - while(loc.iter() != loc.end()) - { - lex_ws_comment_newline::invoke(loc); // skip - - if(loc.iter() != loc.end() && *loc.iter() == ']') - { - loc.advance(); // skip ']' - return ok(std::make_pair(retval, - region(loc, first, loc.iter()))); - } - - if(auto val = parse_value(loc)) - { - // After TOML v1.0.0-rc.1, array becomes to be able to have values - // with different types. So here we will omit this by default. - // - // But some of the test-suite checks if the parser accepts a hetero- - // geneous arrays, so we keep this for a while. -#ifdef TOML11_DISALLOW_HETEROGENEOUS_ARRAYS - if(!retval.empty() && retval.front().type() != val.as_ok().type()) - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array: " - "type of elements should be the same each other.", { - {source_location(array_start_loc), "array starts here"}, - { - retval.front().location(), - "value has type " + stringize(retval.front().type()) - }, - { - val.unwrap().location(), - "value has different type, " + stringize(val.unwrap().type()) - } - }), source_location(loc)); - } -#endif - retval.push_back(std::move(val.unwrap())); - } - else - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array: " - "value having invalid format appeared in an array", { - {source_location(array_start_loc), "array starts here"}, - {source_location(loc), "it is not a valid value."} - }), source_location(loc)); - } - - using lex_array_separator = sequence, character<','>>; - const auto sp = lex_array_separator::invoke(loc); - if(!sp) - { - lex_ws_comment_newline::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == ']') - { - loc.advance(); // skip ']' - return ok(std::make_pair(retval, - region(loc, first, loc.iter()))); - } - else - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array:" - " missing array separator `,` after a value", { - {source_location(array_start_loc), "array starts here"}, - {source_location(loc), "should be `,`"} - }), source_location(loc)); - } - } - } - loc.reset(first); - throw syntax_error(format_underline("toml::parse_array: " - "array did not closed by `]`", - {{source_location(loc), "should be closed"}}), - source_location(loc)); -} - -template -result, region>, Value>, std::string> -parse_key_value_pair(location& loc) -{ - using value_type = Value; - - const auto first = loc.iter(); - auto key_reg = parse_key(loc); - if(!key_reg) - { - std::string msg = std::move(key_reg.unwrap_err()); - // if the next token is keyvalue-separator, it means that there are no - // key. then we need to show error as "empty key is not allowed". - if(const auto keyval_sep = lex_keyval_sep::invoke(loc)) - { - loc.reset(first); - msg = format_underline("toml::parse_key_value_pair: " - "empty key is not allowed.", - {{source_location(loc), "key expected before '='"}}); - } - return err(std::move(msg)); - } - - const auto kvsp = lex_keyval_sep::invoke(loc); - if(!kvsp) - { - std::string msg; - // if the line contains '=' after the invalid sequence, possibly the - // error is in the key (like, invalid character in bare key). - const auto line_end = std::find(loc.iter(), loc.end(), '\n'); - if(std::find(loc.iter(), line_end, '=') != line_end) - { - msg = format_underline("toml::parse_key_value_pair: " - "invalid format for key", - {{source_location(loc), "invalid character in key"}}, - {"Did you forget '.' to separate dotted-key?", - "Allowed characters for bare key are [0-9a-zA-Z_-]."}); - } - else // if not, the error is lack of key-value separator. - { - msg = format_underline("toml::parse_key_value_pair: " - "missing key-value separator `=`", - {{source_location(loc), "should be `=`"}}); - } - loc.reset(first); - return err(std::move(msg)); - } - - const auto after_kvsp = loc.iter(); // err msg - auto val = parse_value(loc); - if(!val) - { - std::string msg; - loc.reset(after_kvsp); - // check there is something not a comment/whitespace after `=` - if(sequence, maybe, lex_newline>::invoke(loc)) - { - loc.reset(after_kvsp); - msg = format_underline("toml::parse_key_value_pair: " - "missing value after key-value separator '='", - {{source_location(loc), "expected value, but got nothing"}}); - } - else // there is something not a comment/whitespace, so invalid format. - { - msg = std::move(val.unwrap_err()); - } - loc.reset(first); - return err(msg); - } - return ok(std::make_pair(std::move(key_reg.unwrap()), - std::move(val.unwrap()))); -} - -// for error messages. -template -std::string format_dotted_keys(InputIterator first, const InputIterator last) -{ - static_assert(std::is_same::value_type>::value,""); - - std::string retval(*first++); - for(; first != last; ++first) - { - retval += '.'; - retval += *first; - } - return retval; -} - -// forward decl for is_valid_forward_table_definition -result, region>, std::string> -parse_table_key(location& loc); -template -result, std::string> -parse_inline_table(location& loc); - -// The following toml file is allowed. -// ```toml -// [a.b.c] # here, table `a` has element `b`. -// foo = "bar" -// [a] # merge a = {baz = "qux"} to a = {b = {...}} -// baz = "qux" -// ``` -// But the following is not allowed. -// ```toml -// [a] -// b.c.foo = "bar" -// [a] # error! the same table [a] defined! -// baz = "qux" -// ``` -// The following is neither allowed. -// ```toml -// a = { b.c.foo = "bar"} -// [a] # error! the same table [a] defined! -// baz = "qux" -// ``` -// Here, it parses region of `tab->at(k)` as a table key and check the depth -// of the key. If the key region points deeper node, it would be allowed. -// Otherwise, the key points the same node. It would be rejected. -template -bool is_valid_forward_table_definition(const Value& fwd, const Value& inserting, - Iterator key_first, Iterator key_curr, Iterator key_last) -{ - // ------------------------------------------------------------------------ - // check type of the value to be inserted/merged - - std::string inserting_reg = ""; - if(const auto ptr = detail::get_region(inserting)) - { - inserting_reg = ptr->str(); - } - location inserting_def("internal", std::move(inserting_reg)); - if(const auto inlinetable = parse_inline_table(inserting_def)) - { - // check if we are overwriting existing table. - // ```toml - // # NG - // a.b = 42 - // a = {d = 3.14} - // ``` - // Inserting an inline table to a existing super-table is not allowed in - // any case. If we found it, we can reject it without further checking. - return false; - } - - // ------------------------------------------------------------------------ - // check table defined before - - std::string internal = ""; - if(const auto ptr = detail::get_region(fwd)) - { - internal = ptr->str(); - } - location def("internal", std::move(internal)); - if(const auto tabkeys = parse_table_key(def)) // [table.key] - { - // table keys always contains all the nodes from the root. - const auto& tks = tabkeys.unwrap().first; - if(std::size_t(std::distance(key_first, key_last)) == tks.size() && - std::equal(tks.begin(), tks.end(), key_first)) - { - // the keys are equivalent. it is not allowed. - return false; - } - // the keys are not equivalent. it is allowed. - return true; - } - if(const auto dotkeys = parse_key(def)) - { - // consider the following case. - // [a] - // b.c = {d = 42} - // [a.b.c] - // e = 2.71 - // this defines the table [a.b.c] twice. no? - - // a dotted key starts from the node representing a table in which the - // dotted key belongs to. - const auto& dks = dotkeys.unwrap().first; - if(std::size_t(std::distance(key_curr, key_last)) == dks.size() && - std::equal(dks.begin(), dks.end(), key_curr)) - { - // the keys are equivalent. it is not allowed. - return false; - } - // the keys are not equivalent. it is allowed. - return true; - } - return false; -} - -template -result -insert_nested_key(typename Value::table_type& root, const Value& v, - InputIterator iter, const InputIterator last, - region key_reg, - const bool is_array_of_table = false) -{ - static_assert(std::is_same::value_type>::value,""); - - using value_type = Value; - using table_type = typename value_type::table_type; - using array_type = typename value_type::array_type; - - const auto first = iter; - assert(iter != last); - - table_type* tab = std::addressof(root); - for(; iter != last; ++iter) // search recursively - { - const key& k = *iter; - if(std::next(iter) == last) // k is the last key - { - // XXX if the value is array-of-tables, there can be several - // tables that are in the same array. in that case, we need to - // find the last element and insert it to there. - if(is_array_of_table) - { - if(tab->count(k) == 1) // there is already an array of table - { - if(tab->at(k).is_table()) - { - // show special err msg for conflicting table - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), - "\") cannot be defined"), { - {tab->at(k).location(), "table already defined"}, - {v.location(), "this conflicts with the previous table"} - }), v.location()); - } - else if(!(tab->at(k).is_array())) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides with" - " existing value"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value already exists")}, - {v.location(), - "while inserting this array-of-tables"} - }), v.location()); - } - // the above if-else-if checks tab->at(k) is an array - auto& a = tab->at(k).as_array(); - // If table element is defined as [[array_of_tables]], it - // cannot be an empty array. If an array of tables is - // defined as `aot = []`, it cannot be appended. - if(a.empty() || !(a.front().is_table())) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides with" - " existing value"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value already exists")}, - {v.location(), - "while inserting this array-of-tables"} - }), v.location()); - } - // avoid conflicting array of table like the following. - // ```toml - // a = [{b = 42}] # define a as an array of *inline* tables - // [[a]] # a is an array of *multi-line* tables - // b = 54 - // ``` - // Here, from the type information, these cannot be detected - // because inline table is also a table. - // But toml v0.5.0 explicitly says it is invalid. The above - // array-of-tables has a static size and appending to the - // array is invalid. - // In this library, multi-line table value has a region - // that points to the key of the table (e.g. [[a]]). By - // comparing the first two letters in key, we can detect - // the array-of-table is inline or multiline. - if(const auto ptr = detail::get_region(a.front())) - { - if(ptr->str().substr(0,2) != "[[") - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides " - "with existing array-of-tables"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value has static size")}, - {v.location(), - "appending it to the statically sized array"} - }), v.location()); - } - } - a.push_back(v); - return ok(true); - } - else // if not, we need to create the array of table - { - // XXX: Consider the following array of tables. - // ```toml - // # This is a comment. - // [[aot]] - // foo = "bar" - // ``` - // Here, the comment is for `aot`. But here, actually two - // values are defined. An array that contains tables, named - // `aot`, and the 0th element of the `aot`, `{foo = "bar"}`. - // Those two are different from each other. But both of them - // points to the same portion of the TOML file, `[[aot]]`, - // so `key_reg.comments()` returns `# This is a comment`. - // If it is assigned as a comment of `aot` defined here, the - // comment will be duplicated. Both the `aot` itself and - // the 0-th element will have the same comment. This causes - // "duplication of the same comments" bug when the data is - // serialized. - // Next, consider the following. - // ```toml - // # comment 1 - // aot = [ - // # comment 2 - // {foo = "bar"}, - // ] - // ``` - // In this case, we can distinguish those two comments. So - // here we need to add "comment 1" to the `aot` and - // "comment 2" to the 0th element of that. - // To distinguish those two, we check the key region. - std::vector comments{/* empty by default */}; - if(key_reg.str().substr(0, 2) != "[[") - { - comments = key_reg.comments(); - } - value_type aot(array_type(1, v), key_reg, std::move(comments)); - tab->insert(std::make_pair(k, aot)); - return ok(true); - } - } // end if(array of table) - - if(tab->count(k) == 1) - { - if(tab->at(k).is_table() && v.is_table()) - { - if(!is_valid_forward_table_definition( - tab->at(k), v, first, iter, last)) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: table (\"", - format_dotted_keys(first, last), - "\") already exists."), { - {tab->at(k).location(), "table already exists here"}, - {v.location(), "table defined twice"} - }), v.location()); - } - // to allow the following toml file. - // [a.b.c] - // d = 42 - // [a] - // e = 2.71 - auto& t = tab->at(k).as_table(); - for(const auto& kv : v.as_table()) - { - if(tab->at(k).contains(kv.first)) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: value (\"", - format_dotted_keys(first, last), - "\") already exists."), { - {t.at(kv.first).location(), "already exists here"}, - {v.location(), "this defined twice"} - }), v.location()); - } - t[kv.first] = kv.second; - } - detail::change_region(tab->at(k), key_reg); - return ok(true); - } - else if(v.is_table() && - tab->at(k).is_array() && - tab->at(k).as_array().size() > 0 && - tab->at(k).as_array().front().is_table()) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of tables (\"", - format_dotted_keys(first, last), "\") already exists."), { - {tab->at(k).location(), "array of tables defined here"}, - {v.location(), "table conflicts with the previous array of table"} - }), v.location()); - } - else - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: value (\"", - format_dotted_keys(first, last), "\") already exists."), { - {tab->at(k).location(), "value already exists here"}, - {v.location(), "value defined twice"} - }), v.location()); - } - } - tab->insert(std::make_pair(k, v)); - return ok(true); - } - else // k is not the last one, we should insert recursively - { - // if there is no corresponding value, insert it first. - // related: you don't need to write - // # [x] - // # [x.y] - // to write - // [x.y.z] - if(tab->count(k) == 0) - { - // a table that is defined implicitly doesn't have any comments. - (*tab)[k] = value_type(table_type{}, key_reg, {/*no comment*/}); - } - - // type checking... - if(tab->at(k).is_table()) - { - // According to toml-lang/toml:36d3091b3 "Clarify that inline - // tables are immutable", check if it adds key-value pair to an - // inline table. - if(const auto* ptr = get_region(tab->at(k))) - { - // here, if the value is a (multi-line) table, the region - // should be something like `[table-name]`. - if(ptr->front() == '{') - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: inserting to an inline table (", - format_dotted_keys(first, std::next(iter)), - ") but inline tables are immutable"), { - {tab->at(k).location(), "inline tables are immutable"}, - {v.location(), "inserting this"} - }), v.location()); - } - } - tab = std::addressof((*tab)[k].as_table()); - } - else if(tab->at(k).is_array()) // inserting to array-of-tables? - { - auto& a = (*tab)[k].as_array(); - if(!a.back().is_table()) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: target (", - format_dotted_keys(first, std::next(iter)), - ") is neither table nor an array of tables"), { - {a.back().location(), concat_to_string( - "actual type is ", a.back().type())}, - {v.location(), "inserting this"} - }), v.location()); - } - tab = std::addressof(a.back().as_table()); - } - else - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: target (", - format_dotted_keys(first, std::next(iter)), - ") is neither table nor an array of tables"), { - {tab->at(k).location(), concat_to_string( - "actual type is ", tab->at(k).type())}, - {v.location(), "inserting this"} - }), v.location()); - } - } - } - return err(std::string("toml::detail::insert_nested_key: never reach here")); -} - -template -result, std::string> -parse_inline_table(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - table_type retval; - if(!(loc.iter() != loc.end() && *loc.iter() == '{')) - { - return err(format_underline("toml::parse_inline_table: ", - {{source_location(loc), "the next token is not an inline table"}})); - } - loc.advance(); - - // check if the inline table is an empty table = { } - maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') - { - loc.advance(); // skip `}` - return ok(std::make_pair(retval, region(loc, first, loc.iter()))); - } - - // it starts from "{". it should be formatted as inline-table - while(loc.iter() != loc.end()) - { - const auto kv_r = parse_key_value_pair(loc); - if(!kv_r) - { - return err(kv_r.unwrap_err()); - } - - const auto& kvpair = kv_r.unwrap(); - const std::vector& keys = kvpair.first.first; - const auto& key_reg = kvpair.first.second; - const value_type& val = kvpair.second; - - const auto inserted = - insert_nested_key(retval, val, keys.begin(), keys.end(), key_reg); - if(!inserted) - { - throw internal_error("toml::parse_inline_table: " - "failed to insert value into table: " + inserted.unwrap_err(), - source_location(loc)); - } - - using lex_table_separator = sequence, character<','>>; - const auto sp = lex_table_separator::invoke(loc); - - if(!sp) - { - maybe::invoke(loc); - - if(loc.iter() == loc.end()) - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing table separator `}` ", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - else if(*loc.iter() == '}') - { - loc.advance(); // skip `}` - return ok(std::make_pair( - retval, region(loc, first, loc.iter()))); - } - else if(*loc.iter() == '#' || *loc.iter() == '\r' || *loc.iter() == '\n') - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing curly brace `}`", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - else - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing table separator `,` ", - {{source_location(loc), "should be `,`"}}), - source_location(loc)); - } - } - else // `,` is found - { - maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') - { - throw syntax_error(format_underline( - "toml::parse_inline_table: trailing comma is not allowed in" - " an inline table", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - } - } - loc.reset(first); - throw syntax_error(format_underline("toml::parse_inline_table: " - "inline table did not closed by `}`", - {{source_location(loc), "should be closed"}}), - source_location(loc)); -} - -inline result guess_number_type(const location& l) -{ - // This function tries to find some (common) mistakes by checking characters - // that follows the last character of a value. But it is often difficult - // because some non-newline characters can appear after a value. E.g. - // spaces, tabs, commas (in an array or inline table), closing brackets - // (of an array or inline table), comment-sign (#). Since this function - // does not parse further, those characters are always allowed to be there. - location loc = l; - - if(lex_offset_date_time::invoke(loc)) {return ok(value_t::offset_datetime);} - loc.reset(l.iter()); - - if(lex_local_date_time::invoke(loc)) - { - // bad offset may appear after this. - if(loc.iter() != loc.end() && (*loc.iter() == '+' || *loc.iter() == '-' - || *loc.iter() == 'Z' || *loc.iter() == 'z')) - { - return err(format_underline("bad offset: should be [+-]HH:MM or Z", - {{source_location(loc), "[+-]HH:MM or Z"}}, - {"pass: +09:00, -05:30", "fail: +9:00, -5:30"})); - } - return ok(value_t::local_datetime); - } - loc.reset(l.iter()); - - if(lex_local_date::invoke(loc)) - { - // bad time may appear after this. - // A space is allowed as a delimiter between local time. But there are - // both cases in which a space becomes valid or invalid. - // - invalid: 2019-06-16 7:00:00 - // - valid : 2019-06-16 07:00:00 - if(loc.iter() != loc.end()) - { - const auto c = *loc.iter(); - if(c == 'T' || c == 't') - { - return err(format_underline("bad time: should be HH:MM:SS.subsec", - {{source_location(loc), "HH:MM:SS.subsec"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 17:32"})); - } - if('0' <= c && c <= '9') - { - return err(format_underline("bad time: missing T", - {{source_location(loc), "T or space required here"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); - } - if(c == ' ' && std::next(loc.iter()) != loc.end() && - ('0' <= *std::next(loc.iter()) && *std::next(loc.iter())<= '9')) - { - loc.advance(); - return err(format_underline("bad time: should be HH:MM:SS.subsec", - {{source_location(loc), "HH:MM:SS.subsec"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); - } - } - return ok(value_t::local_date); - } - loc.reset(l.iter()); - - if(lex_local_time::invoke(loc)) {return ok(value_t::local_time);} - loc.reset(l.iter()); - - if(lex_float::invoke(loc)) - { - if(loc.iter() != loc.end() && *loc.iter() == '_') - { - return err(format_underline("bad float: `_` should be surrounded by digits", - {{source_location(loc), "here"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - return ok(value_t::floating); - } - loc.reset(l.iter()); - - if(lex_integer::invoke(loc)) - { - if(loc.iter() != loc.end()) - { - const auto c = *loc.iter(); - if(c == '_') - { - return err(format_underline("bad integer: `_` should be surrounded by digits", - {{source_location(loc), "here"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - if('0' <= c && c <= '9') - { - // leading zero. point '0' - loc.retrace(); - return err(format_underline("bad integer: leading zero", - {{source_location(loc), "here"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - if(c == ':' || c == '-') - { - return err(format_underline("bad datetime: invalid format", - {{source_location(loc), "here"}}, - {"pass: 1979-05-27T07:32:00-07:00, 1979-05-27 07:32:00.999999Z", - "fail: 1979-05-27T7:32:00-7:00, 1979-05-27 7:32-00:30"})); - } - if(c == '.' || c == 'e' || c == 'E') - { - return err(format_underline("bad float: invalid format", - {{source_location(loc), "here"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - } - return ok(value_t::integer); - } - if(loc.iter() != loc.end() && *loc.iter() == '.') - { - return err(format_underline("bad float: invalid format", - {{source_location(loc), "integer part required before this"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - if(loc.iter() != loc.end() && *loc.iter() == '_') - { - return err(format_underline("bad number: `_` should be surrounded by digits", - {{source_location(loc), "`_` is not surrounded by digits"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - return err(format_underline("bad format: unknown value appeared", - {{source_location(loc), "here"}})); -} - -inline result guess_value_type(const location& loc) -{ - switch(*loc.iter()) - { - case '"' : {return ok(value_t::string); } - case '\'': {return ok(value_t::string); } - case 't' : {return ok(value_t::boolean); } - case 'f' : {return ok(value_t::boolean); } - case '[' : {return ok(value_t::array); } - case '{' : {return ok(value_t::table); } - case 'i' : {return ok(value_t::floating);} // inf. - case 'n' : {return ok(value_t::floating);} // nan. - default : {return guess_number_type(loc);} - } -} - -template -result -parse_value_helper(result, std::string> rslt) -{ - if(rslt.is_ok()) - { - auto comments = rslt.as_ok().second.comments(); - return ok(Value(std::move(rslt.as_ok()), std::move(comments))); - } - else - { - return err(std::move(rslt.as_err())); - } -} - -template -result parse_value(location& loc) -{ - const auto first = loc.iter(); - if(first == loc.end()) - { - return err(format_underline("toml::parse_value: input is empty", - {{source_location(loc), ""}})); - } - - const auto type = guess_value_type(loc); - if(!type) - { - return err(type.unwrap_err()); - } - - switch(type.unwrap()) - { - case value_t::boolean : {return parse_value_helper(parse_boolean(loc) );} - case value_t::integer : {return parse_value_helper(parse_integer(loc) );} - case value_t::floating : {return parse_value_helper(parse_floating(loc) );} - case value_t::string : {return parse_value_helper(parse_string(loc) );} - case value_t::offset_datetime: {return parse_value_helper(parse_offset_datetime(loc) );} - case value_t::local_datetime : {return parse_value_helper(parse_local_datetime(loc) );} - case value_t::local_date : {return parse_value_helper(parse_local_date(loc) );} - case value_t::local_time : {return parse_value_helper(parse_local_time(loc) );} - case value_t::array : {return parse_value_helper(parse_array(loc) );} - case value_t::table : {return parse_value_helper(parse_inline_table(loc));} - default: - { - const auto msg = format_underline("toml::parse_value: " - "unknown token appeared", {{source_location(loc), "unknown"}}); - loc.reset(first); - return err(msg); - } - } -} - -inline result, region>, std::string> -parse_table_key(location& loc) -{ - if(auto token = lex_std_table::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_std_table_open::invoke(inner_loc); - if(!open || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `[`", - {{source_location(inner_loc), "should be `[`"}}), - source_location(inner_loc)); - } - // to skip [ a . b . c ] - // ^----------- this whitespace - lex_ws::invoke(inner_loc); - const auto keys = parse_key(inner_loc); - if(!keys) - { - throw internal_error(format_underline( - "toml::parse_table_key: invalid key", - {{source_location(inner_loc), "not key"}}), - source_location(inner_loc)); - } - // to skip [ a . b . c ] - // ^-- this whitespace - lex_ws::invoke(inner_loc); - const auto close = lex_std_table_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `]`", - {{source_location(inner_loc), "should be `]`"}}), - source_location(inner_loc)); - } - - // after [table.key], newline or EOF(empty table) required. - if(loc.iter() != loc.end()) - { - using lex_newline_after_table_key = - sequence, maybe, lex_newline>; - const auto nl = lex_newline_after_table_key::invoke(loc); - if(!nl) - { - throw syntax_error(format_underline( - "toml::parse_table_key: newline required after [table.key]", - {{source_location(loc), "expected newline"}}), - source_location(loc)); - } - } - return ok(std::make_pair(keys.unwrap().first, token.unwrap())); - } - else - { - return err(format_underline("toml::parse_table_key: " - "not a valid table key", {{source_location(loc), "here"}})); - } -} - -inline result, region>, std::string> -parse_array_table_key(location& loc) -{ - if(auto token = lex_array_table::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_array_table_open::invoke(inner_loc); - if(!open || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_array_table_key: no `[[`", - {{source_location(inner_loc), "should be `[[`"}}), - source_location(inner_loc)); - } - lex_ws::invoke(inner_loc); - const auto keys = parse_key(inner_loc); - if(!keys) - { - throw internal_error(format_underline( - "toml::parse_array_table_key: invalid key", - {{source_location(inner_loc), "not a key"}}), - source_location(inner_loc)); - } - lex_ws::invoke(inner_loc); - const auto close = lex_array_table_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `]]`", - {{source_location(inner_loc), "should be `]]`"}}), - source_location(inner_loc)); - } - - // after [[table.key]], newline or EOF(empty table) required. - if(loc.iter() != loc.end()) - { - using lex_newline_after_table_key = - sequence, maybe, lex_newline>; - const auto nl = lex_newline_after_table_key::invoke(loc); - if(!nl) - { - throw syntax_error(format_underline("toml::" - "parse_array_table_key: newline required after [[table.key]]", - {{source_location(loc), "expected newline"}}), - source_location(loc)); - } - } - return ok(std::make_pair(keys.unwrap().first, token.unwrap())); - } - else - { - return err(format_underline("toml::parse_array_table_key: " - "not a valid table key", {{source_location(loc), "here"}})); - } -} - -// parse table body (key-value pairs until the iter hits the next [tablekey]) -template -result -parse_ml_table(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - if(first == loc.end()) - { - return ok(table_type{}); - } - - // XXX at lest one newline is needed. - using skip_line = repeat< - sequence, maybe, lex_newline>, at_least<1>>; - skip_line::invoke(loc); - lex_ws::invoke(loc); - - table_type tab; - while(loc.iter() != loc.end()) - { - lex_ws::invoke(loc); - const auto before = loc.iter(); - if(const auto tmp = parse_array_table_key(loc)) // next table found - { - loc.reset(before); - return ok(tab); - } - if(const auto tmp = parse_table_key(loc)) // next table found - { - loc.reset(before); - return ok(tab); - } - - if(const auto kv = parse_key_value_pair(loc)) - { - const auto& kvpair = kv.unwrap(); - const std::vector& keys = kvpair.first.first; - const auto& key_reg = kvpair.first.second; - const value_type& val = kvpair.second; - const auto inserted = - insert_nested_key(tab, val, keys.begin(), keys.end(), key_reg); - if(!inserted) - { - return err(inserted.unwrap_err()); - } - } - else - { - return err(kv.unwrap_err()); - } - - // comment lines are skipped by the above function call. - // However, since the `skip_line` requires at least 1 newline, it fails - // if the file ends with ws and/or comment without newline. - // `skip_line` matches `ws? + comment? + newline`, not `ws` or `comment` - // itself. To skip the last ws and/or comment, call lexers. - // It does not matter if these fails, so the return value is discarded. - lex_ws::invoke(loc); - lex_comment::invoke(loc); - - // skip_line is (whitespace? comment? newline)_{1,}. multiple empty lines - // and comments after the last key-value pairs are allowed. - const auto newline = skip_line::invoke(loc); - if(!newline && loc.iter() != loc.end()) - { - const auto before2 = loc.iter(); - lex_ws::invoke(loc); // skip whitespace - const auto msg = format_underline("toml::parse_table: " - "invalid line format", {{source_location(loc), concat_to_string( - "expected newline, but got '", show_char(*loc.iter()), "'.")}}); - loc.reset(before2); - return err(msg); - } - - // the skip_lines only matches with lines that includes newline. - // to skip the last line that includes comment and/or whitespace - // but no newline, call them one more time. - lex_ws::invoke(loc); - lex_comment::invoke(loc); - } - return ok(tab); -} - -template -result parse_toml_file(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - if(first == loc.end()) - { - // For empty files, return an empty table with an empty region (zero-length). - // Without the region, error messages would miss the filename. - return ok(value_type(table_type{}, region(loc, first, first), {})); - } - - // put the first line as a region of a file - // Here first != loc.end(), so taking std::next is okay - const region file(loc, first, std::next(loc.iter())); - - // The first successive comments that are separated from the first value - // by an empty line are for a file itself. - // ```toml - // # this is a comment for a file. - // - // key = "the first value" - // ``` - // ```toml - // # this is a comment for "the first value". - // key = "the first value" - // ``` - std::vector comments; - using lex_first_comments = sequence< - repeat, lex_comment, lex_newline>, at_least<1>>, - sequence, lex_newline> - >; - if(const auto token = lex_first_comments::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - while(inner_loc.iter() != inner_loc.end()) - { - maybe::invoke(inner_loc); // remove ws if exists - if(lex_newline::invoke(inner_loc)) - { - assert(inner_loc.iter() == inner_loc.end()); - break; // empty line found. - } - auto com = lex_comment::invoke(inner_loc).unwrap().str(); - com.erase(com.begin()); // remove # sign - comments.push_back(std::move(com)); - lex_newline::invoke(inner_loc); - } - } - - table_type data; - // root object is also a table, but without [tablename] - if(const auto tab = parse_ml_table(loc)) - { - data = std::move(tab.unwrap()); - } - else // failed (empty table is regarded as success in parse_ml_table) - { - return err(tab.unwrap_err()); - } - while(loc.iter() != loc.end()) - { - // here, the region of [table] is regarded as the table-key because - // the table body is normally too big and it is not so informative - // if the first key-value pair of the table is shown in the error - // message. - if(const auto tabkey = parse_array_table_key(loc)) - { - const auto tab = parse_ml_table(loc); - if(!tab){return err(tab.unwrap_err());} - - const auto& tk = tabkey.unwrap(); - const auto& keys = tk.first; - const auto& reg = tk.second; - - const auto inserted = insert_nested_key(data, - value_type(tab.unwrap(), reg, reg.comments()), - keys.begin(), keys.end(), reg, - /*is_array_of_table=*/ true); - if(!inserted) {return err(inserted.unwrap_err());} - - continue; - } - if(const auto tabkey = parse_table_key(loc)) - { - const auto tab = parse_ml_table(loc); - if(!tab){return err(tab.unwrap_err());} - - const auto& tk = tabkey.unwrap(); - const auto& keys = tk.first; - const auto& reg = tk.second; - - const auto inserted = insert_nested_key(data, - value_type(tab.unwrap(), reg, reg.comments()), - keys.begin(), keys.end(), reg); - if(!inserted) {return err(inserted.unwrap_err());} - - continue; - } - return err(format_underline("toml::parse_toml_file: " - "unknown line appeared", {{source_location(loc), "unknown format"}})); - } - - return ok(Value(std::move(data), file, comments)); -} - -} // detail - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value -parse(std::istream& is, const std::string& fname = "unknown file") -{ - using value_type = basic_value; - - const auto beg = is.tellg(); - is.seekg(0, std::ios::end); - const auto end = is.tellg(); - const auto fsize = end - beg; - is.seekg(beg); - - // read whole file as a sequence of char - assert(fsize >= 0); - std::vector letters(static_cast(fsize)); - is.read(letters.data(), fsize); - - // append LF. - // Although TOML does not require LF at the EOF, to make parsing logic - // simpler, we "normalize" the content by adding LF if it does not exist. - // It also checks if the last char is CR, to avoid changing the meaning. - // This is not the *best* way to deal with the last character, but is a - // simple and quick fix. - if(!letters.empty() && letters.back() != '\n' && letters.back() != '\r') - { - letters.push_back('\n'); - } - - detail::location loc(std::move(fname), std::move(letters)); - - // skip BOM if exists. - // XXX component of BOM (like 0xEF) exceeds the representable range of - // signed char, so on some (actually, most) of the environment, these cannot - // be compared to char. However, since we are always out of luck, we need to - // check our chars are equivalent to BOM. To do this, first we need to - // convert char to unsigned char to guarantee the comparability. - if(loc.source()->size() >= 3) - { - std::array BOM; - std::memcpy(BOM.data(), loc.source()->data(), 3); - if(BOM[0] == 0xEF && BOM[1] == 0xBB && BOM[2] == 0xBF) - { - loc.advance(3); // BOM found. skip. - } - } - - const auto data = detail::parse_toml_file(loc); - if(!data) - { - throw syntax_error(data.unwrap_err(), source_location(loc)); - } - return data.unwrap(); -} - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const std::string& fname) -{ - std::ifstream ifs(fname.c_str(), std::ios_base::binary); - if(!ifs.good()) - { - throw std::runtime_error("toml::parse: file open error -> " + fname); - } - return parse(ifs, fname); -} - -#ifdef TOML11_HAS_STD_FILESYSTEM -// This function just forwards `parse("filename.toml")` to std::string version -// to avoid the ambiguity in overload resolution. -// -// Both std::string and std::filesystem::path are convertible from const char*. -// Without this, both parse(std::string) and parse(std::filesystem::path) -// matches to parse("filename.toml"). This breaks the existing code. -// -// This function exactly matches to the invocation with c-string. -// So this function is preferred than others and the ambiguity disappears. -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const char* fname) -{ - return parse(std::string(fname)); -} - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const std::filesystem::path& fpath) -{ - std::ifstream ifs(fpath, std::ios_base::binary); - if(!ifs.good()) - { - throw std::runtime_error("toml::parse: file open error -> " + - fpath.string()); - } - return parse(ifs, fpath.string()); -} -#endif // TOML11_HAS_STD_FILESYSTEM - -} // toml -#endif// TOML11_PARSER_HPP diff --git a/src/toml11/toml/region.hpp b/src/toml11/toml/region.hpp deleted file mode 100644 index 2e01e51d0..000000000 --- a/src/toml11/toml/region.hpp +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_REGION_HPP -#define TOML11_REGION_HPP -#include -#include -#include -#include -#include -#include -#include -#include "color.hpp" - -namespace toml -{ -namespace detail -{ - -// helper function to avoid std::string(0, 'c') or std::string(iter, iter) -template -std::string make_string(Iterator first, Iterator last) -{ - if(first == last) {return "";} - return std::string(first, last); -} -inline std::string make_string(std::size_t len, char c) -{ - if(len == 0) {return "";} - return std::string(len, c); -} - -// region_base is a base class of location and region that are defined below. -// it will be used to generate better error messages. -struct region_base -{ - region_base() = default; - virtual ~region_base() = default; - region_base(const region_base&) = default; - region_base(region_base&& ) = default; - region_base& operator=(const region_base&) = default; - region_base& operator=(region_base&& ) = default; - - virtual bool is_ok() const noexcept {return false;} - virtual char front() const noexcept {return '\0';} - - virtual std::string str() const {return std::string("unknown region");} - virtual std::string name() const {return std::string("unknown file");} - virtual std::string line() const {return std::string("unknown line");} - virtual std::string line_num() const {return std::string("?");} - - // length of the region - virtual std::size_t size() const noexcept {return 0;} - // number of characters in the line before the region - virtual std::size_t before() const noexcept {return 0;} - // number of characters in the line after the region - virtual std::size_t after() const noexcept {return 0;} - - virtual std::vector comments() const {return {};} - // ```toml - // # comment_before - // key = "value" # comment_inline - // ``` -}; - -// location represents a position in a container, which contains a file content. -// it can be considered as a region that contains only one character. -// -// it contains pointer to the file content and iterator that points the current -// location. -struct location final : public region_base -{ - using const_iterator = typename std::vector::const_iterator; - using difference_type = typename const_iterator::difference_type; - using source_ptr = std::shared_ptr>; - - location(std::string source_name, std::vector cont) - : source_(std::make_shared>(std::move(cont))), - line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) - {} - location(std::string source_name, const std::string& cont) - : source_(std::make_shared>(cont.begin(), cont.end())), - line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) - {} - - location(const location&) = default; - location(location&&) = default; - location& operator=(const location&) = default; - location& operator=(location&&) = default; - ~location() = default; - - bool is_ok() const noexcept override {return static_cast(source_);} - char front() const noexcept override {return *iter_;} - - // this const prohibits codes like `++(loc.iter())`. - const const_iterator iter() const noexcept {return iter_;} - - const_iterator begin() const noexcept {return source_->cbegin();} - const_iterator end() const noexcept {return source_->cend();} - - // XXX `location::line_num()` used to be implemented using `std::count` to - // count a number of '\n'. But with a long toml file (typically, 10k lines), - // it becomes intolerably slow because each time it generates error messages, - // it counts '\n' from thousands of characters. To workaround it, I decided - // to introduce `location::line_number_` member variable and synchronize it - // to the location changes the point to look. So an overload of `iter()` - // which returns mutable reference is removed and `advance()`, `retrace()` - // and `reset()` is added. - void advance(difference_type n = 1) noexcept - { - this->line_number_ += static_cast( - std::count(this->iter_, std::next(this->iter_, n), '\n')); - this->iter_ += n; - return; - } - void retrace(difference_type n = 1) noexcept - { - this->line_number_ -= static_cast( - std::count(std::prev(this->iter_, n), this->iter_, '\n')); - this->iter_ -= n; - return; - } - void reset(const_iterator rollback) noexcept - { - // since c++11, std::distance works in both ways for random-access - // iterators and returns a negative value if `first > last`. - if(0 <= std::distance(rollback, this->iter_)) // rollback < iter - { - this->line_number_ -= static_cast( - std::count(rollback, this->iter_, '\n')); - } - else // iter < rollback [[unlikely]] - { - this->line_number_ += static_cast( - std::count(this->iter_, rollback, '\n')); - } - this->iter_ = rollback; - return; - } - - std::string str() const override {return make_string(1, *this->iter());} - std::string name() const override {return source_name_;} - - std::string line_num() const override - { - return std::to_string(this->line_number_); - } - - std::string line() const override - { - return make_string(this->line_begin(), this->line_end()); - } - - const_iterator line_begin() const noexcept - { - using reverse_iterator = std::reverse_iterator; - return std::find(reverse_iterator(this->iter()), - reverse_iterator(this->begin()), '\n').base(); - } - const_iterator line_end() const noexcept - { - return std::find(this->iter(), this->end(), '\n'); - } - - // location is always points a character. so the size is 1. - std::size_t size() const noexcept override - { - return 1u; - } - std::size_t before() const noexcept override - { - const auto sz = std::distance(this->line_begin(), this->iter()); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t after() const noexcept override - { - const auto sz = std::distance(this->iter(), this->line_end()); - assert(sz >= 0); - return static_cast(sz); - } - - source_ptr const& source() const& noexcept {return source_;} - source_ptr&& source() && noexcept {return std::move(source_);} - - private: - - source_ptr source_; - std::size_t line_number_; - std::string source_name_; - const_iterator iter_; -}; - -// region represents a range in a container, which contains a file content. -// -// it contains pointer to the file content and iterator that points the first -// and last location. -struct region final : public region_base -{ - using const_iterator = typename std::vector::const_iterator; - using source_ptr = std::shared_ptr>; - - // delete default constructor. source_ never be null. - region() = delete; - - explicit region(const location& loc) - : source_(loc.source()), source_name_(loc.name()), - first_(loc.iter()), last_(loc.iter()) - {} - explicit region(location&& loc) - : source_(loc.source()), source_name_(loc.name()), - first_(loc.iter()), last_(loc.iter()) - {} - - region(const location& loc, const_iterator f, const_iterator l) - : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) - {} - region(location&& loc, const_iterator f, const_iterator l) - : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) - {} - - region(const region&) = default; - region(region&&) = default; - region& operator=(const region&) = default; - region& operator=(region&&) = default; - ~region() = default; - - region& operator+=(const region& other) - { - // different regions cannot be concatenated - assert(this->begin() == other.begin() && this->end() == other.end() && - this->last_ == other.first_); - - this->last_ = other.last_; - return *this; - } - - bool is_ok() const noexcept override {return static_cast(source_);} - char front() const noexcept override {return *first_;} - - std::string str() const override {return make_string(first_, last_);} - std::string line() const override - { - if(this->contain_newline()) - { - return make_string(this->line_begin(), - std::find(this->line_begin(), this->last(), '\n')); - } - return make_string(this->line_begin(), this->line_end()); - } - std::string line_num() const override - { - return std::to_string(1 + std::count(this->begin(), this->first(), '\n')); - } - - std::size_t size() const noexcept override - { - const auto sz = std::distance(first_, last_); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t before() const noexcept override - { - const auto sz = std::distance(this->line_begin(), this->first()); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t after() const noexcept override - { - const auto sz = std::distance(this->last(), this->line_end()); - assert(sz >= 0); - return static_cast(sz); - } - - bool contain_newline() const noexcept - { - return std::find(this->first(), this->last(), '\n') != this->last(); - } - - const_iterator line_begin() const noexcept - { - using reverse_iterator = std::reverse_iterator; - return std::find(reverse_iterator(this->first()), - reverse_iterator(this->begin()), '\n').base(); - } - const_iterator line_end() const noexcept - { - return std::find(this->last(), this->end(), '\n'); - } - - const_iterator begin() const noexcept {return source_->cbegin();} - const_iterator end() const noexcept {return source_->cend();} - const_iterator first() const noexcept {return first_;} - const_iterator last() const noexcept {return last_;} - - source_ptr const& source() const& noexcept {return source_;} - source_ptr&& source() && noexcept {return std::move(source_);} - - std::string name() const override {return source_name_;} - - std::vector comments() const override - { - // assuming the current region (`*this`) points a value. - // ```toml - // a = "value" - // ^^^^^^^- this region - // ``` - using rev_iter = std::reverse_iterator; - - std::vector com{}; - { - // find comments just before the current region. - // ```toml - // # this should be collected. - // # this also. - // a = value # not this. - // ``` - - // # this is a comment for `a`, not array elements. - // a = [1, 2, 3, 4, 5] - if(this->first() == std::find_if(this->line_begin(), this->first(), - [](const char c) noexcept -> bool {return c == '[' || c == '{';})) - { - auto iter = this->line_begin(); // points the first character - while(iter != this->begin()) - { - iter = std::prev(iter); - - // range [line_start, iter) represents the previous line - const auto line_start = std::find( - rev_iter(iter), rev_iter(this->begin()), '\n').base(); - const auto comment_found = std::find(line_start, iter, '#'); - if(comment_found == iter) - { - break; // comment not found. - } - - // exclude the following case. - // > a = "foo" # comment // <-- this is not a comment for b but a. - // > b = "current value" - if(std::all_of(line_start, comment_found, - [](const char c) noexcept -> bool { - return c == ' ' || c == '\t'; - })) - { - // unwrap the first '#' by std::next. - auto s = make_string(std::next(comment_found), iter); - if(!s.empty() && s.back() == '\r') {s.pop_back();} - com.push_back(std::move(s)); - } - else - { - break; - } - iter = line_start; - } - } - } - - if(com.size() > 1) - { - std::reverse(com.begin(), com.end()); - } - - { - // find comments just after the current region. - // ```toml - // # not this. - // a = value # this one. - // a = [ # not this (technically difficult) - // - // ] # and this. - // ``` - // The reason why it's difficult is that it requires parsing in the - // following case. - // ```toml - // a = [ 10 # this comment is for `10`. not for `a` but `a[0]`. - // # ... - // ] # this is apparently a comment for a. - // - // b = [ - // 3.14 ] # there is no way to add a comment to `3.14` currently. - // - // c = [ - // 3.14 # do this if you need a comment here. - // ] - // ``` - const auto comment_found = - std::find(this->last(), this->line_end(), '#'); - if(comment_found != this->line_end()) // '#' found - { - // table = {key = "value"} # what is this for? - // the above comment is not for "value", but {key="value"}. - if(comment_found == std::find_if(this->last(), comment_found, - [](const char c) noexcept -> bool { - return !(c == ' ' || c == '\t' || c == ','); - })) - { - // unwrap the first '#' by std::next. - auto s = make_string(std::next(comment_found), this->line_end()); - if(!s.empty() && s.back() == '\r') {s.pop_back();} - com.push_back(std::move(s)); - } - } - } - return com; - } - - private: - - source_ptr source_; - std::string source_name_; - const_iterator first_, last_; -}; - -} // detail -} // toml -#endif// TOML11_REGION_H diff --git a/src/toml11/toml/result.hpp b/src/toml11/toml/result.hpp deleted file mode 100644 index 77cd46c64..000000000 --- a/src/toml11/toml/result.hpp +++ /dev/null @@ -1,717 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_RESULT_HPP -#define TOML11_RESULT_HPP -#include "traits.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace toml -{ - -template -struct success -{ - using value_type = T; - value_type value; - - explicit success(const value_type& v) - noexcept(std::is_nothrow_copy_constructible::value) - : value(v) - {} - explicit success(value_type&& v) - noexcept(std::is_nothrow_move_constructible::value) - : value(std::move(v)) - {} - - template - explicit success(U&& v): value(std::forward(v)) {} - - template - explicit success(const success& v): value(v.value) {} - template - explicit success(success&& v): value(std::move(v.value)) {} - - ~success() = default; - success(const success&) = default; - success(success&&) = default; - success& operator=(const success&) = default; - success& operator=(success&&) = default; -}; - -template -struct failure -{ - using value_type = T; - value_type value; - - explicit failure(const value_type& v) - noexcept(std::is_nothrow_copy_constructible::value) - : value(v) - {} - explicit failure(value_type&& v) - noexcept(std::is_nothrow_move_constructible::value) - : value(std::move(v)) - {} - - template - explicit failure(U&& v): value(std::forward(v)) {} - - template - explicit failure(const failure& v): value(v.value) {} - template - explicit failure(failure&& v): value(std::move(v.value)) {} - - ~failure() = default; - failure(const failure&) = default; - failure(failure&&) = default; - failure& operator=(const failure&) = default; - failure& operator=(failure&&) = default; -}; - -template -success::type>::type> -ok(T&& v) -{ - return success< - typename std::remove_cv::type>::type - >(std::forward(v)); -} -template -failure::type>::type> -err(T&& v) -{ - return failure< - typename std::remove_cv::type>::type - >(std::forward(v)); -} - -inline success ok(const char* literal) -{ - return success(std::string(literal)); -} -inline failure err(const char* literal) -{ - return failure(std::string(literal)); -} - - -template -struct result -{ - using value_type = T; - using error_type = E; - using success_type = success; - using failure_type = failure; - - result(const success_type& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(s); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - result(const failure_type& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(f); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - result(success_type&& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - result(failure_type&& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - - template - result(const success& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - template - result(const failure& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - template - result(success&& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - template - result(failure&& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - - result& operator=(const success_type& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(s); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - result& operator=(const failure_type& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(f); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - result& operator=(success_type&& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - result& operator=(failure_type&& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - - template - result& operator=(const success& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - template - result& operator=(const failure& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - template - result& operator=(success&& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - template - result& operator=(failure&& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - - ~result() noexcept {this->cleanup();} - - result(const result& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - result(result&& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - - template - result(const result& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - template - result(result&& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - - result& operator=(const result& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - result& operator=(result&& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - - template - result& operator=(const result& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - template - result& operator=(result&& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - - bool is_ok() const noexcept {return is_ok_;} - bool is_err() const noexcept {return !is_ok_;} - - operator bool() const noexcept {return is_ok_;} - - value_type& unwrap() & - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return this->succ.value; - } - value_type const& unwrap() const& - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return this->succ.value; - } - value_type&& unwrap() && - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return std::move(this->succ.value); - } - - value_type& unwrap_or(value_type& opt) & - { - if(is_err()) {return opt;} - return this->succ.value; - } - value_type const& unwrap_or(value_type const& opt) const& - { - if(is_err()) {return opt;} - return this->succ.value; - } - value_type unwrap_or(value_type opt) && - { - if(is_err()) {return opt;} - return this->succ.value; - } - - error_type& unwrap_err() & - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return this->fail.value; - } - error_type const& unwrap_err() const& - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return this->fail.value; - } - error_type&& unwrap_err() && - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return std::move(this->fail.value); - } - - value_type& as_ok() & noexcept {return this->succ.value;} - value_type const& as_ok() const& noexcept {return this->succ.value;} - value_type&& as_ok() && noexcept {return std::move(this->succ.value);} - - error_type& as_err() & noexcept {return this->fail.value;} - error_type const& as_err() const& noexcept {return this->fail.value;} - error_type&& as_err() && noexcept {return std::move(this->fail.value);} - - - // prerequisities - // F: T -> U - // retval: result - template - result, error_type> - map(F&& f) & - { - if(this->is_ok()){return ok(f(this->as_ok()));} - return err(this->as_err()); - } - template - result, error_type> - map(F&& f) const& - { - if(this->is_ok()){return ok(f(this->as_ok()));} - return err(this->as_err()); - } - template - result, error_type> - map(F&& f) && - { - if(this->is_ok()){return ok(f(std::move(this->as_ok())));} - return err(std::move(this->as_err())); - } - - // prerequisities - // F: E -> F - // retval: result - template - result> - map_err(F&& f) & - { - if(this->is_err()){return err(f(this->as_err()));} - return ok(this->as_ok()); - } - template - result> - map_err(F&& f) const& - { - if(this->is_err()){return err(f(this->as_err()));} - return ok(this->as_ok()); - } - template - result> - map_err(F&& f) && - { - if(this->is_err()){return err(f(std::move(this->as_err())));} - return ok(std::move(this->as_ok())); - } - - // prerequisities - // F: T -> U - // retval: U - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) & - { - if(this->is_err()){return std::forward(opt);} - return f(this->as_ok()); - } - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) const& - { - if(this->is_err()){return std::forward(opt);} - return f(this->as_ok()); - } - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) && - { - if(this->is_err()){return std::forward(opt);} - return f(std::move(this->as_ok())); - } - - // prerequisities - // F: E -> U - // retval: U - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) & - { - if(this->is_ok()){return std::forward(opt);} - return f(this->as_err()); - } - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) const& - { - if(this->is_ok()){return std::forward(opt);} - return f(this->as_err()); - } - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) && - { - if(this->is_ok()){return std::forward(opt);} - return f(std::move(this->as_err())); - } - - // prerequisities: - // F: func T -> U - // toml::err(error_type) should be convertible to U. - // normally, type U is another result and E is convertible to F - template - detail::return_type_of_t - and_then(F&& f) & - { - if(this->is_ok()){return f(this->as_ok());} - return err(this->as_err()); - } - template - detail::return_type_of_t - and_then(F&& f) const& - { - if(this->is_ok()){return f(this->as_ok());} - return err(this->as_err()); - } - template - detail::return_type_of_t - and_then(F&& f) && - { - if(this->is_ok()){return f(std::move(this->as_ok()));} - return err(std::move(this->as_err())); - } - - // prerequisities: - // F: func E -> U - // toml::ok(value_type) should be convertible to U. - // normally, type U is another result and T is convertible to S - template - detail::return_type_of_t - or_else(F&& f) & - { - if(this->is_err()){return f(this->as_err());} - return ok(this->as_ok()); - } - template - detail::return_type_of_t - or_else(F&& f) const& - { - if(this->is_err()){return f(this->as_err());} - return ok(this->as_ok()); - } - template - detail::return_type_of_t - or_else(F&& f) && - { - if(this->is_err()){return f(std::move(this->as_err()));} - return ok(std::move(this->as_ok())); - } - - // if *this is error, returns *this. otherwise, returns other. - result and_other(const result& other) const& - { - return this->is_err() ? *this : other; - } - result and_other(result&& other) && - { - return this->is_err() ? std::move(*this) : std::move(other); - } - - // if *this is okay, returns *this. otherwise, returns other. - result or_other(const result& other) const& - { - return this->is_ok() ? *this : other; - } - result or_other(result&& other) && - { - return this->is_ok() ? std::move(*this) : std::move(other); - } - - void swap(result& other) - { - result tmp(std::move(*this)); - *this = std::move(other); - other = std::move(tmp); - return ; - } - - private: - - static std::string format_error(std::exception const& excpt) - { - return std::string(excpt.what()); - } - template::value, std::nullptr_t>::type = nullptr> - static std::string format_error(U const& others) - { - std::ostringstream oss; oss << others; - return oss.str(); - } - - void cleanup() noexcept - { - if(this->is_ok_) {this->succ.~success_type();} - else {this->fail.~failure_type();} - return; - } - - private: - - bool is_ok_; - union - { - success_type succ; - failure_type fail; - }; -}; - -template -void swap(result& lhs, result& rhs) -{ - lhs.swap(rhs); - return; -} - -// this might be confusing because it eagerly evaluated, while in the other -// cases operator && and || are short-circuited. -// -// template -// inline result -// operator&&(const result& lhs, const result& rhs) noexcept -// { -// return lhs.is_ok() ? rhs : lhs; -// } -// -// template -// inline result -// operator||(const result& lhs, const result& rhs) noexcept -// { -// return lhs.is_ok() ? lhs : rhs; -// } - -// ---------------------------------------------------------------------------- -// re-use result as a optional with none_t - -namespace detail -{ -struct none_t {}; -inline bool operator==(const none_t&, const none_t&) noexcept {return true;} -inline bool operator!=(const none_t&, const none_t&) noexcept {return false;} -inline bool operator< (const none_t&, const none_t&) noexcept {return false;} -inline bool operator<=(const none_t&, const none_t&) noexcept {return true;} -inline bool operator> (const none_t&, const none_t&) noexcept {return false;} -inline bool operator>=(const none_t&, const none_t&) noexcept {return true;} -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const none_t&) -{ - os << "none"; - return os; -} -inline failure none() noexcept {return failure{none_t{}};} -} // detail -} // toml11 -#endif// TOML11_RESULT_H diff --git a/src/toml11/toml/serializer.hpp b/src/toml11/toml/serializer.hpp deleted file mode 100644 index 88ae775a8..000000000 --- a/src/toml11/toml/serializer.hpp +++ /dev/null @@ -1,922 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_SERIALIZER_HPP -#define TOML11_SERIALIZER_HPP -#include -#include - -#include - -#include "lexer.hpp" -#include "value.hpp" - -namespace toml -{ - -// This function serialize a key. It checks a string is a bare key and -// escapes special characters if the string is not compatible to a bare key. -// ```cpp -// std::string k("non.bare.key"); // the key itself includes `.`s. -// std::string formatted = toml::format_key(k); -// assert(formatted == "\"non.bare.key\""); -// ``` -// -// This function is exposed to make it easy to write a user-defined serializer. -// Since toml restricts characters available in a bare key, generally a string -// should be escaped. But checking whether a string needs to be surrounded by -// a `"` and escaping some special character is boring. -template -std::basic_string -format_key(const std::basic_string& k) -{ - if(k.empty()) - { - return std::string("\"\""); - } - - // check the key can be a bare (unquoted) key - detail::location loc(k, std::vector(k.begin(), k.end())); - detail::lex_unquoted_key::invoke(loc); - if(loc.iter() == loc.end()) - { - return k; // all the tokens are consumed. the key is unquoted-key. - } - - //if it includes special characters, then format it in a "quoted" key. - std::basic_string serialized("\""); - for(const char c : k) - { - switch(c) - { - case '\\': {serialized += "\\\\"; break;} - case '\"': {serialized += "\\\""; break;} - case '\b': {serialized += "\\b"; break;} - case '\t': {serialized += "\\t"; break;} - case '\f': {serialized += "\\f"; break;} - case '\n': {serialized += "\\n"; break;} - case '\r': {serialized += "\\r"; break;} - default : {serialized += c; break;} - } - } - serialized += "\""; - return serialized; -} - -template -std::basic_string -format_keys(const std::vector>& keys) -{ - if(keys.empty()) - { - return std::string("\"\""); - } - - std::basic_string serialized; - for(const auto& ky : keys) - { - serialized += format_key(ky); - serialized += charT('.'); - } - serialized.pop_back(); // remove the last dot '.' - return serialized; -} - -template -struct serializer -{ - static_assert(detail::is_basic_value::value, - "toml::serializer is for toml::value and its variants, " - "toml::basic_value<...>."); - - using value_type = Value; - using key_type = typename value_type::key_type ; - using comment_type = typename value_type::comment_type ; - using boolean_type = typename value_type::boolean_type ; - using integer_type = typename value_type::integer_type ; - using floating_type = typename value_type::floating_type ; - using string_type = typename value_type::string_type ; - using local_time_type = typename value_type::local_time_type ; - using local_date_type = typename value_type::local_date_type ; - using local_datetime_type = typename value_type::local_datetime_type ; - using offset_datetime_type = typename value_type::offset_datetime_type; - using array_type = typename value_type::array_type ; - using table_type = typename value_type::table_type ; - - serializer(const std::size_t w = 80u, - const int float_prec = std::numeric_limits::max_digits10, - const bool can_be_inlined = false, - const bool no_comment = false, - std::vector ks = {}, - const bool value_has_comment = false) - : can_be_inlined_(can_be_inlined), no_comment_(no_comment), - value_has_comment_(value_has_comment && !no_comment), - float_prec_(float_prec), width_(w), keys_(std::move(ks)) - {} - ~serializer() = default; - - std::string operator()(const boolean_type& b) const - { - return b ? "true" : "false"; - } - std::string operator()(const integer_type i) const - { - return std::to_string(i); - } - std::string operator()(const floating_type f) const - { - if(std::isnan(f)) - { - if(std::signbit(f)) - { - return std::string("-nan"); - } - else - { - return std::string("nan"); - } - } - else if(!std::isfinite(f)) - { - if(std::signbit(f)) - { - return std::string("-inf"); - } - else - { - return std::string("inf"); - } - } - - const auto fmt = "%.*g"; - const auto bsz = std::snprintf(nullptr, 0, fmt, this->float_prec_, f); - // +1 for null character(\0) - std::vector buf(static_cast(bsz + 1), '\0'); - std::snprintf(buf.data(), buf.size(), fmt, this->float_prec_, f); - - std::string token(buf.begin(), std::prev(buf.end())); - if(!token.empty() && token.back() == '.') // 1. => 1.0 - { - token += '0'; - } - - const auto e = std::find_if( - token.cbegin(), token.cend(), [](const char c) noexcept -> bool { - return c == 'e' || c == 'E'; - }); - const auto has_exponent = (token.cend() != e); - const auto has_fraction = (token.cend() != std::find( - token.cbegin(), token.cend(), '.')); - - if(!has_exponent && !has_fraction) - { - // the resulting value does not have any float specific part! - token += ".0"; - } - return token; - } - std::string operator()(const string_type& s) const - { - if(s.kind == string_t::basic) - { - if((std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\"') != s.str.cend()) && - this->width_ != (std::numeric_limits::max)()) - { - // if linefeed or double-quote is contained, - // make it multiline basic string. - const auto escaped = this->escape_ml_basic_string(s.str); - std::string open("\"\"\""); - std::string close("\"\"\""); - if(escaped.find('\n') != std::string::npos || - this->width_ < escaped.size() + 6) - { - // if the string body contains newline or is enough long, - // add newlines after and before delimiters. - open += "\n"; - close = std::string("\\\n") + close; - } - return open + escaped + close; - } - - // no linefeed. try to make it oneline-string. - std::string oneline = this->escape_basic_string(s.str); - if(oneline.size() + 2 < width_ || width_ < 2) - { - const std::string quote("\""); - return quote + oneline + quote; - } - - // the line is too long compared to the specified width. - // split it into multiple lines. - std::string token("\"\"\"\n"); - while(!oneline.empty()) - { - if(oneline.size() < width_) - { - token += oneline; - oneline.clear(); - } - else if(oneline.at(width_-2) == '\\') - { - token += oneline.substr(0, width_-2); - token += "\\\n"; - oneline.erase(0, width_-2); - } - else - { - token += oneline.substr(0, width_-1); - token += "\\\n"; - oneline.erase(0, width_-1); - } - } - return token + std::string("\\\n\"\"\""); - } - else // the string `s` is literal-string. - { - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) - { - std::string open("'''"); - if(this->width_ + 6 < s.str.size()) - { - open += '\n'; // the first newline is ignored by TOML spec - } - const std::string close("'''"); - return open + s.str + close; - } - else - { - const std::string quote("'"); - return quote + s.str + quote; - } - } - } - - std::string operator()(const local_date_type& d) const - { - std::ostringstream oss; - oss << d; - return oss.str(); - } - std::string operator()(const local_time_type& t) const - { - std::ostringstream oss; - oss << t; - return oss.str(); - } - std::string operator()(const local_datetime_type& dt) const - { - std::ostringstream oss; - oss << dt; - return oss.str(); - } - std::string operator()(const offset_datetime_type& odt) const - { - std::ostringstream oss; - oss << odt; - return oss.str(); - } - - std::string operator()(const array_type& v) const - { - if(v.empty()) - { - return std::string("[]"); - } - if(this->is_array_of_tables(v)) - { - return make_array_of_tables(v); - } - - // not an array of tables. normal array. - // first, try to make it inline if none of the elements have a comment. - if( ! this->has_comment_inside(v)) - { - const auto inl = this->make_inline_array(v); - if(inl.size() < this->width_ && - std::find(inl.cbegin(), inl.cend(), '\n') == inl.cend()) - { - return inl; - } - } - - // if the length exceeds this->width_, print multiline array. - // key = [ - // # ... - // 42, - // ... - // ] - std::string token; - std::string current_line; - token += "[\n"; - for(const auto& item : v) - { - if( ! item.comments().empty() && !no_comment_) - { - // if comment exists, the element must be the only element in the line. - // e.g. the following is not allowed. - // ```toml - // array = [ - // # comment for what? - // 1, 2, 3, 4, 5 - // ] - // ``` - if(!current_line.empty()) - { - if(current_line.back() != '\n') - { - current_line += '\n'; - } - token += current_line; - current_line.clear(); - } - for(const auto& c : item.comments()) - { - token += '#'; - token += c; - token += '\n'; - } - token += toml::visit(*this, item); - if(!token.empty() && token.back() == '\n') {token.pop_back();} - token += ",\n"; - continue; - } - std::string next_elem; - if(item.is_table()) - { - serializer ser(*this); - ser.can_be_inlined_ = true; - ser.width_ = (std::numeric_limits::max)(); - next_elem += toml::visit(ser, item); - } - else - { - next_elem += toml::visit(*this, item); - } - - // comma before newline. - if(!next_elem.empty() && next_elem.back() == '\n') {next_elem.pop_back();} - - // if current line does not exceeds the width limit, continue. - if(current_line.size() + next_elem.size() + 1 < this->width_) - { - current_line += next_elem; - current_line += ','; - } - else if(current_line.empty()) - { - // if current line was empty, force put the next_elem because - // next_elem is not splittable - token += next_elem; - token += ",\n"; - // current_line is kept empty - } - else // reset current_line - { - assert(current_line.back() == ','); - token += current_line; - token += '\n'; - current_line = next_elem; - current_line += ','; - } - } - if(!current_line.empty()) - { - if(!current_line.empty() && current_line.back() != '\n') - { - current_line += '\n'; - } - token += current_line; - } - token += "]\n"; - return token; - } - - // templatize for any table-like container - std::string operator()(const table_type& v) const - { - // if an element has a comment, then it can't be inlined. - // table = {# how can we write a comment for this? key = "value"} - if(this->can_be_inlined_ && !(this->has_comment_inside(v))) - { - std::string token; - if(!this->keys_.empty()) - { - token += format_key(this->keys_.back()); - token += " = "; - } - token += this->make_inline_table(v); - if(token.size() < this->width_ && - token.end() == std::find(token.begin(), token.end(), '\n')) - { - return token; - } - } - - std::string token; - if(!keys_.empty()) - { - token += '['; - token += format_keys(keys_); - token += "]\n"; - } - token += this->make_multiline_table(v); - return token; - } - - private: - - std::string escape_basic_string(const std::string& s) const - { - //XXX assuming `s` is a valid utf-8 sequence. - std::string retval; - for(const char c : s) - { - switch(c) - { - case '\\': {retval += "\\\\"; break;} - case '\"': {retval += "\\\""; break;} - case '\b': {retval += "\\b"; break;} - case '\t': {retval += "\\t"; break;} - case '\f': {retval += "\\f"; break;} - case '\n': {retval += "\\n"; break;} - case '\r': {retval += "\\r"; break;} - default : - { - if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) - { - retval += "\\u00"; - retval += char(48 + (c / 16)); - retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); - } - else - { - retval += c; - } - } - } - } - return retval; - } - - std::string escape_ml_basic_string(const std::string& s) const - { - std::string retval; - for(auto i=s.cbegin(), e=s.cend(); i!=e; ++i) - { - switch(*i) - { - case '\\': {retval += "\\\\"; break;} - // One or two consecutive "s are allowed. - // Later we will check there are no three consecutive "s. - // case '\"': {retval += "\\\""; break;} - case '\b': {retval += "\\b"; break;} - case '\t': {retval += "\\t"; break;} - case '\f': {retval += "\\f"; break;} - case '\n': {retval += "\n"; break;} - case '\r': - { - if(std::next(i) != e && *std::next(i) == '\n') - { - retval += "\r\n"; - ++i; - } - else - { - retval += "\\r"; - } - break; - } - default : - { - const auto c = *i; - if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) - { - retval += "\\u00"; - retval += char(48 + (c / 16)); - retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); - } - else - { - retval += c; - } - } - - } - } - // Only 1 or 2 consecutive `"`s are allowed in multiline basic string. - // 3 consecutive `"`s are considered as a closing delimiter. - // We need to check if there are 3 or more consecutive `"`s and insert - // backslash to break them down into several short `"`s like the `str6` - // in the following example. - // ```toml - // str4 = """Here are two quotation marks: "". Simple enough.""" - // # str5 = """Here are three quotation marks: """.""" # INVALID - // str5 = """Here are three quotation marks: ""\".""" - // str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" - // ``` - auto found_3_quotes = retval.find("\"\"\""); - while(found_3_quotes != std::string::npos) - { - retval.replace(found_3_quotes, 3, "\"\"\\\""); - found_3_quotes = retval.find("\"\"\""); - } - return retval; - } - - // if an element of a table or an array has a comment, it cannot be inlined. - bool has_comment_inside(const array_type& a) const noexcept - { - // if no_comment is set, comments would not be written. - if(this->no_comment_) {return false;} - - for(const auto& v : a) - { - if(!v.comments().empty()) {return true;} - } - return false; - } - bool has_comment_inside(const table_type& t) const noexcept - { - // if no_comment is set, comments would not be written. - if(this->no_comment_) {return false;} - - for(const auto& kv : t) - { - if(!kv.second.comments().empty()) {return true;} - } - return false; - } - - std::string make_inline_array(const array_type& v) const - { - assert(!has_comment_inside(v)); - std::string token; - token += '['; - bool is_first = true; - for(const auto& item : v) - { - if(is_first) {is_first = false;} else {token += ',';} - token += visit(serializer( - (std::numeric_limits::max)(), this->float_prec_, - /* inlined */ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !item.comments().empty()), item); - } - token += ']'; - return token; - } - - std::string make_inline_table(const table_type& v) const - { - assert(!has_comment_inside(v)); - assert(this->can_be_inlined_); - std::string token; - token += '{'; - bool is_first = true; - for(const auto& kv : v) - { - // in inline tables, trailing comma is not allowed (toml-lang #569). - if(is_first) {is_first = false;} else {token += ',';} - token += format_key(kv.first); - token += '='; - token += visit(serializer( - (std::numeric_limits::max)(), this->float_prec_, - /* inlined */ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - } - token += '}'; - return token; - } - - std::string make_multiline_table(const table_type& v) const - { - std::string token; - - // print non-table elements first. - // ```toml - // [foo] # a table we're writing now here - // key = "value" # <- non-table element, "key" - // # ... - // [foo.bar] # <- table element, "bar" - // ``` - // because after printing [foo.bar], the remaining non-table values will - // be assigned into [foo.bar], not [foo]. Those values should be printed - // earlier. - for(const auto& kv : v) - { - if(kv.second.is_table() || is_array_of_tables(kv.second)) - { - continue; - } - - token += write_comments(kv.second); - - const auto key_and_sep = format_key(kv.first) + " = "; - const auto residual_width = (this->width_ > key_and_sep.size()) ? - this->width_ - key_and_sep.size() : 0; - token += key_and_sep; - token += visit(serializer(residual_width, this->float_prec_, - /*can be inlined*/ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - - if(token.back() != '\n') - { - token += '\n'; - } - } - - // normal tables / array of tables - - // after multiline table appeared, the other tables cannot be inline - // because the table would be assigned into the table. - // [foo] - // ... - // bar = {...} # <- bar will be a member of [foo]. - bool multiline_table_printed = false; - for(const auto& kv : v) - { - if(!kv.second.is_table() && !is_array_of_tables(kv.second)) - { - continue; // other stuff are already serialized. skip them. - } - - std::vector ks(this->keys_); - ks.push_back(kv.first); - - auto tmp = visit(serializer(this->width_, this->float_prec_, - !multiline_table_printed, this->no_comment_, ks, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - - // If it is the first time to print a multi-line table, it would be - // helpful to separate normal key-value pair and subtables by a - // newline. - // (this checks if the current key-value pair contains newlines. - // but it is not perfect because multi-line string can also contain - // a newline. in such a case, an empty line will be written) TODO - if((!multiline_table_printed) && - std::find(tmp.cbegin(), tmp.cend(), '\n') != tmp.cend()) - { - multiline_table_printed = true; - token += '\n'; // separate key-value pairs and subtables - - token += write_comments(kv.second); - token += tmp; - - // care about recursive tables (all tables in each level prints - // newline and there will be a full of newlines) - if(tmp.substr(tmp.size() - 2, 2) != "\n\n" && - tmp.substr(tmp.size() - 4, 4) != "\r\n\r\n" ) - { - token += '\n'; - } - } - else - { - token += write_comments(kv.second); - token += tmp; - token += '\n'; - } - } - return token; - } - - std::string make_array_of_tables(const array_type& v) const - { - // if it's not inlined, we need to add `[[table.key]]`. - // but if it can be inlined, we can format it as the following. - // ``` - // table.key = [ - // {...}, - // # comment - // {...}, - // ] - // ``` - // This function checks if inlinization is possible or not, and then - // format the array-of-tables in a proper way. - // - // Note about comments: - // - // If the array itself has a comment (value_has_comment_ == true), we - // should try to make it inline. - // ```toml - // # comment about array - // array = [ - // # comment about table element - // {of = "table"} - // ] - // ``` - // If it is formatted as a multiline table, the two comments becomes - // indistinguishable. - // ```toml - // # comment about array - // # comment about table element - // [[array]] - // of = "table" - // ``` - // So we need to try to make it inline, and it force-inlines regardless - // of the line width limit. - // It may fail if the element of a table has comment. In that case, - // the array-of-tables will be formatted as a multiline table. - if(this->can_be_inlined_ || this->value_has_comment_) - { - std::string token; - if(!keys_.empty()) - { - token += format_key(keys_.back()); - token += " = "; - } - - bool failed = false; - token += "[\n"; - for(const auto& item : v) - { - // if an element of the table has a comment, the table - // cannot be inlined. - if(this->has_comment_inside(item.as_table())) - { - failed = true; - break; - } - // write comments for the table itself - token += write_comments(item); - - const auto t = this->make_inline_table(item.as_table()); - - if(t.size() + 1 > width_ || // +1 for the last comma {...}, - std::find(t.cbegin(), t.cend(), '\n') != t.cend()) - { - // if the value itself has a comment, ignore the line width limit - if( ! this->value_has_comment_) - { - failed = true; - break; - } - } - token += t; - token += ",\n"; - } - - if( ! failed) - { - token += "]\n"; - return token; - } - // if failed, serialize them as [[array.of.tables]]. - } - - std::string token; - for(const auto& item : v) - { - token += write_comments(item); - token += "[["; - token += format_keys(keys_); - token += "]]\n"; - token += this->make_multiline_table(item.as_table()); - } - return token; - } - - std::string write_comments(const value_type& v) const - { - std::string retval; - if(this->no_comment_) {return retval;} - - for(const auto& c : v.comments()) - { - retval += '#'; - retval += c; - retval += '\n'; - } - return retval; - } - - bool is_array_of_tables(const value_type& v) const - { - if(!v.is_array() || v.as_array().empty()) {return false;} - return is_array_of_tables(v.as_array()); - } - bool is_array_of_tables(const array_type& v) const - { - // Since TOML v0.5.0, heterogeneous arrays are allowed. So we need to - // check all the element in an array to check if the array is an array - // of tables. - return std::all_of(v.begin(), v.end(), [](const value_type& elem) { - return elem.is_table(); - }); - } - - private: - - bool can_be_inlined_; - bool no_comment_; - bool value_has_comment_; - int float_prec_; - std::size_t width_; - std::vector keys_; -}; - -template class M, template class V> -std::string -format(const basic_value& v, std::size_t w = 80u, - int fprec = std::numeric_limits::max_digits10, - bool no_comment = false, bool force_inline = false) -{ - using value_type = basic_value; - // if value is a table, it is considered to be a root object. - // the root object can't be an inline table. - if(v.is_table()) - { - std::ostringstream oss; - if(!v.comments().empty()) - { - oss << v.comments(); - oss << '\n'; // to split the file comment from the first element - } - const auto serialized = visit(serializer(w, fprec, false, no_comment), v); - oss << serialized; - return oss.str(); - } - return visit(serializer(w, fprec, force_inline), v); -} - -namespace detail -{ -template -int comment_index(std::basic_ostream&) -{ - static const int index = std::ios_base::xalloc(); - return index; -} -} // detail - -template -std::basic_ostream& -nocomment(std::basic_ostream& os) -{ - // by default, it is zero. and by default, it shows comments. - os.iword(detail::comment_index(os)) = 1; - return os; -} - -template -std::basic_ostream& -showcomment(std::basic_ostream& os) -{ - // by default, it is zero. and by default, it shows comments. - os.iword(detail::comment_index(os)) = 0; - return os; -} - -template class M, template class V> -std::basic_ostream& -operator<<(std::basic_ostream& os, const basic_value& v) -{ - using value_type = basic_value; - - // get status of std::setw(). - const auto w = static_cast(os.width()); - const int fprec = static_cast(os.precision()); - os.width(0); - - // by default, iword is initialized by 0. And by default, toml11 outputs - // comments. So `0` means showcomment. 1 means nocommnet. - const bool no_comment = (1 == os.iword(detail::comment_index(os))); - - if(!no_comment && v.is_table() && !v.comments().empty()) - { - os << v.comments(); - os << '\n'; // to split the file comment from the first element - } - // the root object can't be an inline table. so pass `false`. - const auto serialized = visit(serializer(w, fprec, no_comment, false), v); - os << serialized; - - // if v is a non-table value, and has only one comment, then - // put a comment just after a value. in the following way. - // - // ```toml - // key = "value" # comment. - // ``` - // - // Since the top-level toml object is a table, one who want to put a - // non-table toml value must use this in a following way. - // - // ```cpp - // toml::value v; - // std::cout << "user-defined-key = " << v << std::endl; - // ``` - // - // In this case, it is impossible to put comments before key-value pair. - // The only way to preserve comments is to put all of them after a value. - if(!no_comment && !v.is_table() && !v.comments().empty()) - { - os << " #"; - for(const auto& c : v.comments()) {os << c;} - } - return os; -} - -} // toml -#endif// TOML11_SERIALIZER_HPP diff --git a/src/toml11/toml/source_location.hpp b/src/toml11/toml/source_location.hpp deleted file mode 100644 index fa175b5b4..000000000 --- a/src/toml11/toml/source_location.hpp +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_SOURCE_LOCATION_HPP -#define TOML11_SOURCE_LOCATION_HPP -#include -#include - -#include "region.hpp" - -namespace toml -{ - -// A struct to contain location in a toml file. -// The interface imitates std::experimental::source_location, -// but not completely the same. -// -// It would be constructed by toml::value. It can be used to generate -// user-defined error messages. -// -// - std::uint_least32_t line() const noexcept -// - returns the line number where the region is on. -// - std::uint_least32_t column() const noexcept -// - returns the column number where the region starts. -// - std::uint_least32_t region() const noexcept -// - returns the size of the region. -// -// +-- line() +-- region of interest (region() == 9) -// v .---+---. -// 12 | value = "foo bar" -// ^ -// +-- column() -// -// - std::string const& file_name() const noexcept; -// - name of the file. -// - std::string const& line_str() const noexcept; -// - the whole line that contains the region of interest. -// -struct source_location -{ - public: - - source_location() - : line_num_(1), column_num_(1), region_size_(1), - file_name_("unknown file"), line_str_("") - {} - - explicit source_location(const detail::region_base* reg) - : line_num_(1), column_num_(1), region_size_(1), - file_name_("unknown file"), line_str_("") - { - if(reg) - { - if(reg->line_num() != detail::region_base().line_num()) - { - line_num_ = static_cast( - std::stoul(reg->line_num())); - } - column_num_ = static_cast(reg->before() + 1); - region_size_ = static_cast(reg->size()); - file_name_ = reg->name(); - line_str_ = reg->line(); - } - } - - explicit source_location(const detail::region& reg) - : line_num_(static_cast(std::stoul(reg.line_num()))), - column_num_(static_cast(reg.before() + 1)), - region_size_(static_cast(reg.size())), - file_name_(reg.name()), - line_str_ (reg.line()) - {} - explicit source_location(const detail::location& loc) - : line_num_(static_cast(std::stoul(loc.line_num()))), - column_num_(static_cast(loc.before() + 1)), - region_size_(static_cast(loc.size())), - file_name_(loc.name()), - line_str_ (loc.line()) - {} - - ~source_location() = default; - source_location(source_location const&) = default; - source_location(source_location &&) = default; - source_location& operator=(source_location const&) = default; - source_location& operator=(source_location &&) = default; - - std::uint_least32_t line() const noexcept {return line_num_;} - std::uint_least32_t column() const noexcept {return column_num_;} - std::uint_least32_t region() const noexcept {return region_size_;} - - std::string const& file_name() const noexcept {return file_name_;} - std::string const& line_str() const noexcept {return line_str_;} - - private: - - std::uint_least32_t line_num_; - std::uint_least32_t column_num_; - std::uint_least32_t region_size_; - std::string file_name_; - std::string line_str_; -}; - -namespace detail -{ - -// internal error message generation. -inline std::string format_underline(const std::string& message, - const std::vector>& loc_com, - const std::vector& helps = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - std::size_t line_num_width = 0; - for(const auto& lc : loc_com) - { - std::uint_least32_t line = lc.first.line(); - std::size_t digit = 0; - while(line != 0) - { - line /= 10; - digit += 1; - } - line_num_width = (std::max)(line_num_width, digit); - } - // 1 is the minimum width - line_num_width = std::max(line_num_width, 1); - - std::ostringstream retval; - - if(colorize) - { - retval << color::colorize; // turn on ANSI color - } - - // XXX - // Here, before `colorize` support, it does not output `[error]` prefix - // automatically. So some user may output it manually and this change may - // duplicate the prefix. To avoid it, check the first 7 characters and - // if it is "[error]", it removes that part from the message shown. - if(message.size() > 7 && message.substr(0, 7) == "[error]") - { - retval << color::bold << color::red << "[error]" << color::reset - << color::bold << message.substr(7) << color::reset << '\n'; - } - else - { - retval << color::bold << color::red << "[error] " << color::reset - << color::bold << message << color::reset << '\n'; - } - - const auto format_one_location = [line_num_width] - (std::ostringstream& oss, - const source_location& loc, const std::string& comment) -> void - { - oss << ' ' << color::bold << color::blue - << std::setw(static_cast(line_num_width)) - << std::right << loc.line() << " | " << color::reset - << loc.line_str() << '\n'; - - oss << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " | " << color::reset - << make_string(loc.column()-1 /*1-origin*/, ' '); - - if(loc.region() == 1) - { - // invalid - // ^------ - oss << color::bold << color::red << "^---" << color::reset; - } - else - { - // invalid - // ~~~~~~~ - const auto underline_len = (std::min)( - static_cast(loc.region()), loc.line_str().size()); - oss << color::bold << color::red - << make_string(underline_len, '~') << color::reset; - } - oss << ' '; - oss << comment; - return; - }; - - assert(!loc_com.empty()); - - // --> example.toml - // | - retval << color::bold << color::blue << " --> " << color::reset - << loc_com.front().first.file_name() << '\n'; - retval << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " |\n" << color::reset; - // 1 | key value - // | ^--- missing = - format_one_location(retval, loc_com.front().first, loc_com.front().second); - - // process the rest of the locations - for(std::size_t i=1; i filename.toml" again - { - retval << color::bold << color::blue << " --> " << color::reset - << curr.first.file_name() << '\n'; - retval << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " |\n" << color::reset; - } - - format_one_location(retval, curr.first, curr.second); - } - - if(!helps.empty()) - { - retval << '\n'; - retval << make_string(line_num_width + 1, ' '); - retval << color::bold << color::blue << " |" << color::reset; - for(const auto& help : helps) - { - retval << color::bold << "\nHint: " << color::reset; - retval << help; - } - } - return retval.str(); -} - -} // detail -} // toml -#endif// TOML11_SOURCE_LOCATION_HPP diff --git a/src/toml11/toml/storage.hpp b/src/toml11/toml/storage.hpp deleted file mode 100644 index 202f9035f..000000000 --- a/src/toml11/toml/storage.hpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_STORAGE_HPP -#define TOML11_STORAGE_HPP -#include "utility.hpp" - -namespace toml -{ -namespace detail -{ - -// this contains pointer and deep-copy the content if copied. -// to avoid recursive pointer. -template -struct storage -{ - using value_type = T; - - explicit storage(value_type const& v): ptr(toml::make_unique(v)) {} - explicit storage(value_type&& v): ptr(toml::make_unique(std::move(v))) {} - ~storage() = default; - storage(const storage& rhs): ptr(toml::make_unique(*rhs.ptr)) {} - storage& operator=(const storage& rhs) - { - this->ptr = toml::make_unique(*rhs.ptr); - return *this; - } - storage(storage&&) = default; - storage& operator=(storage&&) = default; - - bool is_ok() const noexcept {return static_cast(ptr);} - - value_type& value() & noexcept {return *ptr;} - value_type const& value() const& noexcept {return *ptr;} - value_type&& value() && noexcept {return std::move(*ptr);} - - private: - std::unique_ptr ptr; -}; - -} // detail -} // toml -#endif// TOML11_STORAGE_HPP diff --git a/src/toml11/toml/string.hpp b/src/toml11/toml/string.hpp deleted file mode 100644 index 5136d8c56..000000000 --- a/src/toml11/toml/string.hpp +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_STRING_HPP -#define TOML11_STRING_HPP -#include - -#include -#include - -#if __cplusplus >= 201703L -#if __has_include() -#define TOML11_USING_STRING_VIEW 1 -#include -#endif -#endif - -namespace toml -{ - -enum class string_t : std::uint8_t -{ - basic = 0, - literal = 1, -}; - -struct string -{ - string() = default; - ~string() = default; - string(const string& s) = default; - string(string&& s) = default; - string& operator=(const string& s) = default; - string& operator=(string&& s) = default; - - string(const std::string& s): kind(string_t::basic), str(s){} - string(const std::string& s, string_t k): kind(k), str(s){} - string(const char* s): kind(string_t::basic), str(s){} - string(const char* s, string_t k): kind(k), str(s){} - - string(std::string&& s): kind(string_t::basic), str(std::move(s)){} - string(std::string&& s, string_t k): kind(k), str(std::move(s)){} - - string& operator=(const std::string& s) - {kind = string_t::basic; str = s; return *this;} - string& operator=(std::string&& s) - {kind = string_t::basic; str = std::move(s); return *this;} - - operator std::string& () & noexcept {return str;} - operator std::string const& () const& noexcept {return str;} - operator std::string&& () && noexcept {return std::move(str);} - - string& operator+=(const char* rhs) {str += rhs; return *this;} - string& operator+=(const char rhs) {str += rhs; return *this;} - string& operator+=(const std::string& rhs) {str += rhs; return *this;} - string& operator+=(const string& rhs) {str += rhs.str; return *this;} - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 - explicit string(std::string_view s): kind(string_t::basic), str(s){} - string(std::string_view s, string_t k): kind(k), str(s){} - - string& operator=(std::string_view s) - {kind = string_t::basic; str = s; return *this;} - - explicit operator std::string_view() const noexcept - {return std::string_view(str);} - - string& operator+=(const std::string_view& rhs) {str += rhs; return *this;} -#endif - - string_t kind; - std::string str; -}; - -inline bool operator==(const string& lhs, const string& rhs) -{ - return lhs.kind == rhs.kind && lhs.str == rhs.str; -} -inline bool operator!=(const string& lhs, const string& rhs) -{ - return !(lhs == rhs); -} -inline bool operator<(const string& lhs, const string& rhs) -{ - return (lhs.kind == rhs.kind) ? (lhs.str < rhs.str) : (lhs.kind < rhs.kind); -} -inline bool operator>(const string& lhs, const string& rhs) -{ - return rhs < lhs; -} -inline bool operator<=(const string& lhs, const string& rhs) -{ - return !(rhs < lhs); -} -inline bool operator>=(const string& lhs, const string& rhs) -{ - return !(lhs < rhs); -} - -inline bool -operator==(const string& lhs, const std::string& rhs) {return lhs.str == rhs;} -inline bool -operator!=(const string& lhs, const std::string& rhs) {return lhs.str != rhs;} -inline bool -operator< (const string& lhs, const std::string& rhs) {return lhs.str < rhs;} -inline bool -operator> (const string& lhs, const std::string& rhs) {return lhs.str > rhs;} -inline bool -operator<=(const string& lhs, const std::string& rhs) {return lhs.str <= rhs;} -inline bool -operator>=(const string& lhs, const std::string& rhs) {return lhs.str >= rhs;} - -inline bool -operator==(const std::string& lhs, const string& rhs) {return lhs == rhs.str;} -inline bool -operator!=(const std::string& lhs, const string& rhs) {return lhs != rhs.str;} -inline bool -operator< (const std::string& lhs, const string& rhs) {return lhs < rhs.str;} -inline bool -operator> (const std::string& lhs, const string& rhs) {return lhs > rhs.str;} -inline bool -operator<=(const std::string& lhs, const string& rhs) {return lhs <= rhs.str;} -inline bool -operator>=(const std::string& lhs, const string& rhs) {return lhs >= rhs.str;} - -inline bool -operator==(const string& lhs, const char* rhs) {return lhs.str == std::string(rhs);} -inline bool -operator!=(const string& lhs, const char* rhs) {return lhs.str != std::string(rhs);} -inline bool -operator< (const string& lhs, const char* rhs) {return lhs.str < std::string(rhs);} -inline bool -operator> (const string& lhs, const char* rhs) {return lhs.str > std::string(rhs);} -inline bool -operator<=(const string& lhs, const char* rhs) {return lhs.str <= std::string(rhs);} -inline bool -operator>=(const string& lhs, const char* rhs) {return lhs.str >= std::string(rhs);} - -inline bool -operator==(const char* lhs, const string& rhs) {return std::string(lhs) == rhs.str;} -inline bool -operator!=(const char* lhs, const string& rhs) {return std::string(lhs) != rhs.str;} -inline bool -operator< (const char* lhs, const string& rhs) {return std::string(lhs) < rhs.str;} -inline bool -operator> (const char* lhs, const string& rhs) {return std::string(lhs) > rhs.str;} -inline bool -operator<=(const char* lhs, const string& rhs) {return std::string(lhs) <= rhs.str;} -inline bool -operator>=(const char* lhs, const string& rhs) {return std::string(lhs) >= rhs.str;} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const string& s) -{ - if(s.kind == string_t::basic) - { - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend()) - { - // it contains newline. make it multiline string. - os << "\"\"\"\n"; - for(auto i=s.str.cbegin(), e=s.str.cend(); i!=e; ++i) - { - switch(*i) - { - case '\\': {os << "\\\\"; break;} - case '\"': {os << "\\\""; break;} - case '\b': {os << "\\b"; break;} - case '\t': {os << "\\t"; break;} - case '\f': {os << "\\f"; break;} - case '\n': {os << '\n'; break;} - case '\r': - { - // since it is a multiline string, - // CRLF is not needed to be escaped. - if(std::next(i) != e && *std::next(i) == '\n') - { - os << "\r\n"; - ++i; - } - else - { - os << "\\r"; - } - break; - } - default: {os << *i; break;} - } - } - os << "\\\n\"\"\""; - return os; - } - // no newline. make it inline. - os << "\""; - for(const auto c : s.str) - { - switch(c) - { - case '\\': {os << "\\\\"; break;} - case '\"': {os << "\\\""; break;} - case '\b': {os << "\\b"; break;} - case '\t': {os << "\\t"; break;} - case '\f': {os << "\\f"; break;} - case '\n': {os << "\\n"; break;} - case '\r': {os << "\\r"; break;} - default : {os << c; break;} - } - } - os << "\""; - return os; - } - // the string `s` is literal-string. - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) - { - // contains newline or single quote. make it multiline. - os << "'''\n" << s.str << "'''"; - return os; - } - // normal literal string - os << '\'' << s.str << '\''; - return os; -} - -} // toml -#endif// TOML11_STRING_H diff --git a/src/toml11/toml/traits.hpp b/src/toml11/toml/traits.hpp deleted file mode 100644 index 5495c93b2..000000000 --- a/src/toml11/toml/traits.hpp +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_TRAITS_HPP -#define TOML11_TRAITS_HPP - -#include "from.hpp" -#include "into.hpp" - -#include -#include -#include -#include -#include -#include - -#if __cplusplus >= 201703L -#if __has_include() -#include -#endif // has_include() -#endif // cplusplus >= C++17 - -namespace toml -{ -template class T, template class A> -class basic_value; - -namespace detail -{ -// --------------------------------------------------------------------------- -// check whether type T is a kind of container/map class - -struct has_iterator_impl -{ - template static std::true_type check(typename T::iterator*); - template static std::false_type check(...); -}; -struct has_value_type_impl -{ - template static std::true_type check(typename T::value_type*); - template static std::false_type check(...); -}; -struct has_key_type_impl -{ - template static std::true_type check(typename T::key_type*); - template static std::false_type check(...); -}; -struct has_mapped_type_impl -{ - template static std::true_type check(typename T::mapped_type*); - template static std::false_type check(...); -}; -struct has_reserve_method_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval().reserve(std::declval()))*); -}; -struct has_push_back_method_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval().push_back(std::declval()))*); -}; -struct is_comparable_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval() < std::declval())*); -}; - -struct has_from_toml_method_impl -{ - template class Tb, template class A> - static std::true_type check( - decltype(std::declval().from_toml( - std::declval<::toml::basic_value>()))*); - - template class Tb, template class A> - static std::false_type check(...); -}; -struct has_into_toml_method_impl -{ - template - static std::true_type check(decltype(std::declval().into_toml())*); - template - static std::false_type check(...); -}; - -struct has_specialized_from_impl -{ - template - static std::false_type check(...); - template)> - static std::true_type check(::toml::from*); -}; -struct has_specialized_into_impl -{ - template - static std::false_type check(...); - template)> - static std::true_type check(::toml::from*); -}; - - -/// Intel C++ compiler can not use decltype in parent class declaration, here -/// is a hack to work around it. https://stackoverflow.com/a/23953090/4692076 -#ifdef __INTEL_COMPILER -#define decltype(...) std::enable_if::type -#endif - -template -struct has_iterator : decltype(has_iterator_impl::check(nullptr)){}; -template -struct has_value_type : decltype(has_value_type_impl::check(nullptr)){}; -template -struct has_key_type : decltype(has_key_type_impl::check(nullptr)){}; -template -struct has_mapped_type : decltype(has_mapped_type_impl::check(nullptr)){}; -template -struct has_reserve_method : decltype(has_reserve_method_impl::check(nullptr)){}; -template -struct has_push_back_method : decltype(has_push_back_method_impl::check(nullptr)){}; -template -struct is_comparable : decltype(is_comparable_impl::check(nullptr)){}; - -template class Tb, template class A> -struct has_from_toml_method -: decltype(has_from_toml_method_impl::check(nullptr)){}; - -template -struct has_into_toml_method -: decltype(has_into_toml_method_impl::check(nullptr)){}; - -template -struct has_specialized_from : decltype(has_specialized_from_impl::check(nullptr)){}; -template -struct has_specialized_into : decltype(has_specialized_into_impl::check(nullptr)){}; - -#ifdef __INTEL_COMPILER -#undef decltype -#endif - -// --------------------------------------------------------------------------- -// C++17 and/or/not - -#if __cplusplus >= 201703L - -using std::conjunction; -using std::disjunction; -using std::negation; - -#else - -template struct conjunction : std::true_type{}; -template struct conjunction : T{}; -template -struct conjunction : - std::conditional(T::value), conjunction, T>::type -{}; - -template struct disjunction : std::false_type{}; -template struct disjunction : T {}; -template -struct disjunction : - std::conditional(T::value), T, disjunction>::type -{}; - -template -struct negation : std::integral_constant(T::value)>{}; - -#endif - -// --------------------------------------------------------------------------- -// type checkers - -template struct is_std_pair : std::false_type{}; -template -struct is_std_pair> : std::true_type{}; - -template struct is_std_tuple : std::false_type{}; -template -struct is_std_tuple> : std::true_type{}; - -template struct is_std_forward_list : std::false_type{}; -template -struct is_std_forward_list> : std::true_type{}; - -template struct is_chrono_duration: std::false_type{}; -template -struct is_chrono_duration>: std::true_type{}; - -template -struct is_map : conjunction< // map satisfies all the following conditions - has_iterator, // has T::iterator - has_value_type, // has T::value_type - has_key_type, // has T::key_type - has_mapped_type // has T::mapped_type - >{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; - -template -struct is_container : conjunction< - negation>, // not a map - negation>, // not a std::string -#if __cplusplus >= 201703L -#if __has_include() - negation>, // not a std::string_view -#endif // has_include() -#endif - has_iterator, // has T::iterator - has_value_type // has T::value_type - >{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; - -template -struct is_basic_value: std::false_type{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template class M, template class V> -struct is_basic_value<::toml::basic_value>: std::true_type{}; - -// --------------------------------------------------------------------------- -// C++14 index_sequence - -#if __cplusplus >= 201402L - -using std::index_sequence; -using std::make_index_sequence; - -#else - -template struct index_sequence{}; - -template struct push_back_index_sequence{}; -template -struct push_back_index_sequence, N> -{ - typedef index_sequence type; -}; - -template -struct index_sequence_maker -{ - typedef typename push_back_index_sequence< - typename index_sequence_maker::type, N>::type type; -}; -template<> -struct index_sequence_maker<0> -{ - typedef index_sequence<0> type; -}; -template -using make_index_sequence = typename index_sequence_maker::type; - -#endif // __cplusplus >= 2014 - -// --------------------------------------------------------------------------- -// C++14 enable_if_t - -#if __cplusplus >= 201402L - -using std::enable_if_t; - -#else - -template -using enable_if_t = typename std::enable_if::type; - -#endif // __cplusplus >= 2014 - -// --------------------------------------------------------------------------- -// return_type_of_t - -#if __cplusplus >= 201703L && defined(__cpp_lib_is_invocable) && __cpp_lib_is_invocable>=201703 - -template -using return_type_of_t = std::invoke_result_t; - -#else -// result_of is deprecated after C++17 -template -using return_type_of_t = typename std::result_of::type; - -#endif - -// --------------------------------------------------------------------------- -// is_string_literal -// -// to use this, pass `typename remove_reference::type` to T. - -template -struct is_string_literal: -disjunction< - std::is_same, - conjunction< - std::is_array, - std::is_same::type> - > - >{}; - -// --------------------------------------------------------------------------- -// C++20 remove_cvref_t - -template -struct remove_cvref -{ - using type = typename std::remove_cv< - typename std::remove_reference::type>::type; -}; - -template -using remove_cvref_t = typename remove_cvref::type; - -}// detail -}//toml -#endif // TOML_TRAITS diff --git a/src/toml11/toml/types.hpp b/src/toml11/toml/types.hpp deleted file mode 100644 index 1e420e7fd..000000000 --- a/src/toml11/toml/types.hpp +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_TYPES_HPP -#define TOML11_TYPES_HPP -#include -#include - -#include "comments.hpp" -#include "datetime.hpp" -#include "string.hpp" -#include "traits.hpp" - -namespace toml -{ - -template class Table, // map-like class - template class Array> // vector-like class -class basic_value; - -using character = char; -using key = std::string; - -#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ <= 4 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wshadow" -#endif - -using boolean = bool; -using integer = std::int64_t; -using floating = double; // "float" is a keyword, cannot use it here. -// the following stuffs are structs defined here, so aliases are not needed. -// - string -// - offset_datetime -// - offset_datetime -// - local_datetime -// - local_date -// - local_time - -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif - -// default toml::value and default array/table. these are defined after defining -// basic_value itself. -// using value = basic_value; -// using array = typename value::array_type; -// using table = typename value::table_type; - -// to avoid warnings about `value_t::integer` is "shadowing" toml::integer in -// GCC -Wshadow=global. -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic push -# if 7 <= __GNUC__ -# pragma GCC diagnostic ignored "-Wshadow=global" -# else // gcc-6 or older -# pragma GCC diagnostic ignored "-Wshadow" -# endif -#endif -enum class value_t : std::uint8_t -{ - empty = 0, - boolean = 1, - integer = 2, - floating = 3, - string = 4, - offset_datetime = 5, - local_datetime = 6, - local_date = 7, - local_time = 8, - array = 9, - table = 10, -}; -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif - -template -inline std::basic_ostream& -operator<<(std::basic_ostream& os, value_t t) -{ - switch(t) - { - case value_t::boolean : os << "boolean"; return os; - case value_t::integer : os << "integer"; return os; - case value_t::floating : os << "floating"; return os; - case value_t::string : os << "string"; return os; - case value_t::offset_datetime : os << "offset_datetime"; return os; - case value_t::local_datetime : os << "local_datetime"; return os; - case value_t::local_date : os << "local_date"; return os; - case value_t::local_time : os << "local_time"; return os; - case value_t::array : os << "array"; return os; - case value_t::table : os << "table"; return os; - case value_t::empty : os << "empty"; return os; - default : os << "unknown"; return os; - } -} - -template, - typename alloc = std::allocator> -inline std::basic_string stringize(value_t t) -{ - std::basic_ostringstream oss; - oss << t; - return oss.str(); -} - -namespace detail -{ - -// helper to define a type that represents a value_t value. -template -using value_t_constant = std::integral_constant; - -// meta-function that convertes from value_t to the exact toml type that corresponds to. -// It takes toml::basic_value type because array and table types depend on it. -template struct enum_to_type {using type = void ;}; -template struct enum_to_type{using type = void ;}; -template struct enum_to_type{using type = boolean ;}; -template struct enum_to_type{using type = integer ;}; -template struct enum_to_type{using type = floating ;}; -template struct enum_to_type{using type = string ;}; -template struct enum_to_type{using type = offset_datetime ;}; -template struct enum_to_type{using type = local_datetime ;}; -template struct enum_to_type{using type = local_date ;}; -template struct enum_to_type{using type = local_time ;}; -template struct enum_to_type{using type = typename Value::array_type;}; -template struct enum_to_type{using type = typename Value::table_type;}; - -// meta-function that converts from an exact toml type to the enum that corresponds to. -template -struct type_to_enum : std::conditional< - std::is_same::value, // if T == array_type, - value_t_constant, // then value_t::array - typename std::conditional< // else... - std::is_same::value, // if T == table_type - value_t_constant, // then value_t::table - value_t_constant // else value_t::empty - >::type - >::type {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; - -// meta-function that checks the type T is the same as one of the toml::* types. -template -struct is_exact_toml_type : disjunction< - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same - >{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type: is_exact_toml_type{}; - -} // detail -} // toml - -#endif// TOML11_TYPES_H diff --git a/src/toml11/toml/utility.hpp b/src/toml11/toml/utility.hpp deleted file mode 100644 index 4a6b4309d..000000000 --- a/src/toml11/toml/utility.hpp +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_UTILITY_HPP -#define TOML11_UTILITY_HPP -#include -#include -#include - -#include "traits.hpp" - -#if __cplusplus >= 201402L -# define TOML11_MARK_AS_DEPRECATED(msg) [[deprecated(msg)]] -#elif defined(__GNUC__) -# define TOML11_MARK_AS_DEPRECATED(msg) __attribute__((deprecated(msg))) -#elif defined(_MSC_VER) -# define TOML11_MARK_AS_DEPRECATED(msg) __declspec(deprecated(msg)) -#else -# define TOML11_MARK_AS_DEPRECATED -#endif - -namespace toml -{ - -#if __cplusplus >= 201402L - -using std::make_unique; - -#else - -template -inline std::unique_ptr make_unique(Ts&& ... args) -{ - return std::unique_ptr(new T(std::forward(args)...)); -} - -#endif // __cplusplus >= 2014 - -namespace detail -{ -template -void try_reserve_impl(Container& container, std::size_t N, std::true_type) -{ - container.reserve(N); - return; -} -template -void try_reserve_impl(Container&, std::size_t, std::false_type) noexcept -{ - return; -} -} // detail - -template -void try_reserve(Container& container, std::size_t N) -{ - if(N <= container.size()) {return;} - detail::try_reserve_impl(container, N, detail::has_reserve_method{}); - return; -} - -namespace detail -{ -inline std::string concat_to_string_impl(std::ostringstream& oss) -{ - return oss.str(); -} -template -std::string concat_to_string_impl(std::ostringstream& oss, T&& head, Ts&& ... tail) -{ - oss << std::forward(head); - return concat_to_string_impl(oss, std::forward(tail) ... ); -} -} // detail - -template -std::string concat_to_string(Ts&& ... args) -{ - std::ostringstream oss; - oss << std::boolalpha << std::fixed; - return detail::concat_to_string_impl(oss, std::forward(args) ...); -} - -template -T from_string(const std::string& str, T opt) -{ - T v(opt); - std::istringstream iss(str); - iss >> v; - return v; -} - -namespace detail -{ -#if __cplusplus >= 201402L -template -decltype(auto) last_one(T&& tail) noexcept -{ - return std::forward(tail); -} - -template -decltype(auto) last_one(T&& /*head*/, Ts&& ... tail) noexcept -{ - return last_one(std::forward(tail)...); -} -#else // C++11 -// The following code -// ```cpp -// 1 | template -// 2 | auto last_one(T&& /*head*/, Ts&& ... tail) -// 3 | -> decltype(last_one(std::forward(tail)...)) -// 4 | { -// 5 | return last_one(std::forward(tail)...); -// 6 | } -// ``` -// does not work because the function `last_one(...)` is not yet defined at -// line #3, so `decltype()` cannot deduce the type returned from `last_one`. -// So we need to determine return type in a different way, like a meta func. - -template -struct last_one_in_pack -{ - using type = typename last_one_in_pack::type; -}; -template -struct last_one_in_pack -{ - using type = T; -}; -template -using last_one_in_pack_t = typename last_one_in_pack::type; - -template -T&& last_one(T&& tail) noexcept -{ - return std::forward(tail); -} -template -enable_if_t<(sizeof...(Ts) > 0), last_one_in_pack_t> -last_one(T&& /*head*/, Ts&& ... tail) -{ - return last_one(std::forward(tail)...); -} - -#endif -} // detail - -}// toml -#endif // TOML11_UTILITY diff --git a/src/toml11/toml/value.hpp b/src/toml11/toml/value.hpp deleted file mode 100644 index 1b43db8d4..000000000 --- a/src/toml11/toml/value.hpp +++ /dev/null @@ -1,2035 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_VALUE_HPP -#define TOML11_VALUE_HPP -#include - -#include "comments.hpp" -#include "exception.hpp" -#include "into.hpp" -#include "region.hpp" -#include "source_location.hpp" -#include "storage.hpp" -#include "traits.hpp" -#include "types.hpp" -#include "utility.hpp" - -namespace toml -{ - -namespace detail -{ - -// to show error messages. not recommended for users. -template -inline region_base const* get_region(const Value& v) -{ - return v.region_info_.get(); -} - -template -void change_region(Value& v, region reg) -{ - v.region_info_ = std::make_shared(std::move(reg)); - return; -} - -template -[[noreturn]] inline void -throw_bad_cast(const std::string& funcname, value_t actual, const Value& v) -{ - throw type_error(detail::format_underline( - concat_to_string(funcname, "bad_cast to ", Expected), { - {v.location(), concat_to_string("the actual type is ", actual)} - }), v.location()); -} - -// Throw `out_of_range` from `toml::value::at()` and `toml::find()` -// after generating an error message. -// -// The implementation is a bit complicated and there are many edge-cases. -// If you are not interested in the error message generation, just skip this. -template -[[noreturn]] void -throw_key_not_found_error(const Value& v, const key& ky) -{ - // The top-level table has its region at the first character of the file. - // That means that, in the case when a key is not found in the top-level - // table, the error message points to the first character. If the file has - // its first table at the first line, the error message would be like this. - // ```console - // [error] key "a" not found - // --> example.toml - // | - // 1 | [table] - // | ^------ in this table - // ``` - // It actually points to the top-level table at the first character, - // not `[table]`. But it is too confusing. To avoid the confusion, the error - // message should explicitly say "key not found in the top-level table", - // or "the parsed file is empty" if there is no content at all (0 bytes in file). - const auto loc = v.location(); - if(loc.line() == 1 && loc.region() == 0) - { - // First line with a zero-length region means "empty file". - // The region will be generated at `parse_toml_file` function - // if the file contains no bytes. - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found in the top-level table"), { - {loc, "the parsed file is empty"} - })); - } - else if(loc.line() == 1 && loc.region() == 1) - { - // Here it assumes that top-level table starts at the first character. - // The region corresponds to the top-level table will be generated at - // `parse_toml_file` function. - // It also assumes that the top-level table size is just one and - // the line number is `1`. It is always satisfied. And those conditions - // are satisfied only if the table is the top-level table. - // - // 1. one-character dot-key at the first line - // ```toml - // a.b = "c" - // ``` - // toml11 counts whole key as the table key. Here, `a.b` is the region - // of the table "a". It could be counter intuitive, but it works. - // The size of the region is 3, not 1. The above example is the shortest - // dot-key example. The size cannot be 1. - // - // 2. one-character inline-table at the first line - // ```toml - // a = {b = "c"} - // ``` - // toml11 considers the inline table body as the table region. Here, - // `{b = "c"}` is the region of the table "a". The size of the region - // is 9, not 1. The shotest inline table still has two characters, `{` - // and `}`. The size cannot be 1. - // - // 3. one-character table declaration at the first line - // ```toml - // [a] - // ``` - // toml11 considers the whole table key as the table region. Here, - // `[a]` is the table region. The size is 3, not 1. - // - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found in the top-level table"), { - {loc, "the top-level table starts here"} - })); - } - else - { - // normal table. - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found"), { {loc, "in this table"} })); - } -} - -// switch by `value_t` at the compile time. -template -struct switch_cast {}; -#define TOML11_GENERATE_SWITCH_CASTER(TYPE) \ - template<> \ - struct switch_cast \ - { \ - template \ - static typename Value::TYPE##_type& invoke(Value& v) \ - { \ - return v.as_##TYPE(); \ - } \ - template \ - static typename Value::TYPE##_type const& invoke(const Value& v) \ - { \ - return v.as_##TYPE(); \ - } \ - template \ - static typename Value::TYPE##_type&& invoke(Value&& v) \ - { \ - return std::move(v).as_##TYPE(); \ - } \ - }; \ - /**/ -TOML11_GENERATE_SWITCH_CASTER(boolean) -TOML11_GENERATE_SWITCH_CASTER(integer) -TOML11_GENERATE_SWITCH_CASTER(floating) -TOML11_GENERATE_SWITCH_CASTER(string) -TOML11_GENERATE_SWITCH_CASTER(offset_datetime) -TOML11_GENERATE_SWITCH_CASTER(local_datetime) -TOML11_GENERATE_SWITCH_CASTER(local_date) -TOML11_GENERATE_SWITCH_CASTER(local_time) -TOML11_GENERATE_SWITCH_CASTER(array) -TOML11_GENERATE_SWITCH_CASTER(table) - -#undef TOML11_GENERATE_SWITCH_CASTER - -}// detail - -template class Table = std::unordered_map, - template class Array = std::vector> -class basic_value -{ - template - static void assigner(T& dst, U&& v) - { - const auto tmp = ::new(std::addressof(dst)) T(std::forward(v)); - assert(tmp == std::addressof(dst)); - (void)tmp; - } - - using region_base = detail::region_base; - - template class T, - template class A> - friend class basic_value; - - public: - - using comment_type = Comment; - using key_type = ::toml::key; - using value_type = basic_value; - using boolean_type = ::toml::boolean; - using integer_type = ::toml::integer; - using floating_type = ::toml::floating; - using string_type = ::toml::string; - using local_time_type = ::toml::local_time; - using local_date_type = ::toml::local_date; - using local_datetime_type = ::toml::local_datetime; - using offset_datetime_type = ::toml::offset_datetime; - using array_type = Array; - using table_type = Table; - - public: - - basic_value() noexcept - : type_(value_t::empty), - region_info_(std::make_shared(region_base{})) - {} - ~basic_value() noexcept {this->cleanup();} - - basic_value(const basic_value& v) - : type_(v.type()), region_info_(v.region_info_), comments_(v.comments_) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - } - basic_value(basic_value&& v) - : type_(v.type()), region_info_(std::move(v.region_info_)), - comments_(std::move(v.comments_)) - { - switch(this->type_) // here this->type_ is already initialized - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - } - basic_value& operator=(const basic_value& v) - { - this->cleanup(); - this->region_info_ = v.region_info_; - this->comments_ = v.comments_; - this->type_ = v.type(); - switch(this->type_) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - return *this; - } - basic_value& operator=(basic_value&& v) - { - this->cleanup(); - this->region_info_ = std::move(v.region_info_); - this->comments_ = std::move(v.comments_); - this->type_ = v.type(); - switch(this->type_) - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - return *this; - } - - // overwrite comments ---------------------------------------------------- - - basic_value(const basic_value& v, std::vector com) - : type_(v.type()), region_info_(v.region_info_), - comments_(std::move(com)) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - } - - basic_value(basic_value&& v, std::vector com) - : type_(v.type()), region_info_(std::move(v.region_info_)), - comments_(std::move(com)) - { - switch(this->type_) // here this->type_ is already initialized - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - } - - // ----------------------------------------------------------------------- - // conversion between different basic_values. - template class T, - template class A> - basic_value(const basic_value& v) - : type_(v.type()), region_info_(v.region_info_), comments_(v.comments()) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - } - template class T, - template class A> - basic_value(const basic_value& v, std::vector com) - : type_(v.type()), region_info_(v.region_info_), - comments_(std::move(com)) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - } - template class T, - template class A> - basic_value& operator=(const basic_value& v) - { - this->region_info_ = v.region_info_; - this->comments_ = comment_type(v.comments()); - this->type_ = v.type(); - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - return *this; - } - - // boolean ============================================================== - - basic_value(boolean b) - : type_(value_t::boolean), - region_info_(std::make_shared(region_base{})) - { - assigner(this->boolean_, b); - } - basic_value& operator=(boolean b) - { - this->cleanup(); - this->type_ = value_t::boolean; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->boolean_, b); - return *this; - } - basic_value(boolean b, std::vector com) - : type_(value_t::boolean), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->boolean_, b); - } - - // integer ============================================================== - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value(T i) - : type_(value_t::integer), - region_info_(std::make_shared(region_base{})) - { - assigner(this->integer_, static_cast(i)); - } - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value& operator=(T i) - { - this->cleanup(); - this->type_ = value_t::integer; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->integer_, static_cast(i)); - return *this; - } - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value(T i, std::vector com) - : type_(value_t::integer), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->integer_, static_cast(i)); - } - - // floating ============================================================= - - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f) - : type_(value_t::floating), - region_info_(std::make_shared(region_base{})) - { - assigner(this->floating_, static_cast(f)); - } - - - template::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(T f) - { - this->cleanup(); - this->type_ = value_t::floating; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->floating_, static_cast(f)); - return *this; - } - - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f, std::vector com) - : type_(value_t::floating), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->floating_, f); - } - - // string =============================================================== - - basic_value(toml::string s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, std::move(s)); - } - basic_value& operator=(toml::string s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, s); - return *this; - } - basic_value(toml::string s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, std::move(s)); - } - - basic_value(std::string s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::move(s))); - } - basic_value& operator=(std::string s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(std::move(s))); - return *this; - } - basic_value(std::string s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::move(s), kind)); - } - basic_value(std::string s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::move(s))); - } - basic_value(std::string s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::move(s), kind)); - } - - basic_value(const char* s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::string(s))); - } - basic_value& operator=(const char* s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(std::string(s))); - return *this; - } - basic_value(const char* s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::string(s), kind)); - } - basic_value(const char* s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::string(s))); - } - basic_value(const char* s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::string(s), kind)); - } - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 - basic_value(std::string_view s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(s)); - } - basic_value& operator=(std::string_view s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(s)); - return *this; - } - basic_value(std::string_view s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(s)); - } - basic_value(std::string_view s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(s, kind)); - } - basic_value(std::string_view s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(s, kind)); - } -#endif - - // local date =========================================================== - - basic_value(const local_date& ld) - : type_(value_t::local_date), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_date_, ld); - } - basic_value& operator=(const local_date& ld) - { - this->cleanup(); - this->type_ = value_t::local_date; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_date_, ld); - return *this; - } - basic_value(const local_date& ld, std::vector com) - : type_(value_t::local_date), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_date_, ld); - } - - // local time =========================================================== - - basic_value(const local_time& lt) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_time_, lt); - } - basic_value(const local_time& lt, std::vector com) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_time_, lt); - } - basic_value& operator=(const local_time& lt) - { - this->cleanup(); - this->type_ = value_t::local_time; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_time_, lt); - return *this; - } - - template - basic_value(const std::chrono::duration& dur) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_time_, local_time(dur)); - } - template - basic_value(const std::chrono::duration& dur, - std::vector com) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_time_, local_time(dur)); - } - template - basic_value& operator=(const std::chrono::duration& dur) - { - this->cleanup(); - this->type_ = value_t::local_time; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_time_, local_time(dur)); - return *this; - } - - // local datetime ======================================================= - - basic_value(const local_datetime& ldt) - : type_(value_t::local_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_datetime_, ldt); - } - basic_value(const local_datetime& ldt, std::vector com) - : type_(value_t::local_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_datetime_, ldt); - } - basic_value& operator=(const local_datetime& ldt) - { - this->cleanup(); - this->type_ = value_t::local_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_datetime_, ldt); - return *this; - } - - // offset datetime ====================================================== - - basic_value(const offset_datetime& odt) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->offset_datetime_, odt); - } - basic_value(const offset_datetime& odt, std::vector com) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->offset_datetime_, odt); - } - basic_value& operator=(const offset_datetime& odt) - { - this->cleanup(); - this->type_ = value_t::offset_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->offset_datetime_, odt); - return *this; - } - basic_value(const std::chrono::system_clock::time_point& tp) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->offset_datetime_, offset_datetime(tp)); - } - basic_value(const std::chrono::system_clock::time_point& tp, - std::vector com) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->offset_datetime_, offset_datetime(tp)); - } - basic_value& operator=(const std::chrono::system_clock::time_point& tp) - { - this->cleanup(); - this->type_ = value_t::offset_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->offset_datetime_, offset_datetime(tp)); - return *this; - } - - // array ================================================================ - - basic_value(const array_type& ary) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - assigner(this->array_, ary); - } - basic_value(const array_type& ary, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->array_, ary); - } - basic_value& operator=(const array_type& ary) - { - this->cleanup(); - this->type_ = value_t::array ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->array_, ary); - return *this; - } - - // array (initializer_list) ---------------------------------------------- - - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::initializer_list list) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - } - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::initializer_list list, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - } - template::value, - std::nullptr_t>::type = nullptr> - basic_value& operator=(std::initializer_list list) - { - this->cleanup(); - this->type_ = value_t::array; - this->region_info_ = std::make_shared(region_base{}); - - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - return *this; - } - - // array (STL Containers) ------------------------------------------------ - - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value(const T& list) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - } - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value(const T& list, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - } - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const T& list) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - this->cleanup(); - this->type_ = value_t::array; - this->region_info_ = std::make_shared(region_base{}); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - return *this; - } - - // table ================================================================ - - basic_value(const table_type& tab) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - assigner(this->table_, tab); - } - basic_value(const table_type& tab, std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->table_, tab); - } - basic_value& operator=(const table_type& tab) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->table_, tab); - return *this; - } - - // initializer-list ------------------------------------------------------ - - basic_value(std::initializer_list> list) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - - basic_value(std::initializer_list> list, - std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - basic_value& operator=(std::initializer_list> list) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - return *this; - } - - // other table-like ----------------------------------------------------- - - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value(const Map& mp) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value(const Map& mp, std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const Map& mp) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - return *this; - } - - // user-defined ========================================================= - - // convert using into_toml() method ------------------------------------- - - template::value, std::nullptr_t>::type = nullptr> - basic_value(const T& ud): basic_value(ud.into_toml()) {} - - template::value, std::nullptr_t>::type = nullptr> - basic_value(const T& ud, std::vector com) - : basic_value(ud.into_toml(), std::move(com)) - {} - template::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const T& ud) - { - *this = ud.into_toml(); - return *this; - } - - // convert using into struct ----------------------------------------- - - template)> - basic_value(const T& ud): basic_value(::toml::into::into_toml(ud)) {} - template)> - basic_value(const T& ud, std::vector com) - : basic_value(::toml::into::into_toml(ud), std::move(com)) - {} - template)> - basic_value& operator=(const T& ud) - { - *this = ::toml::into::into_toml(ud); - return *this; - } - - // for internal use ------------------------------------------------------ - // - // Those constructors take detail::region that contains parse result. - - basic_value(boolean b, detail::region reg, std::vector cm) - : type_(value_t::boolean), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->boolean_, b); - } - template, detail::negation> - >::value, std::nullptr_t>::type = nullptr> - basic_value(T i, detail::region reg, std::vector cm) - : type_(value_t::integer), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->integer_, static_cast(i)); - } - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f, detail::region reg, std::vector cm) - : type_(value_t::floating), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->floating_, static_cast(f)); - } - basic_value(toml::string s, detail::region reg, - std::vector cm) - : type_(value_t::string), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->string_, std::move(s)); - } - basic_value(const local_date& ld, detail::region reg, - std::vector cm) - : type_(value_t::local_date), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_date_, ld); - } - basic_value(const local_time& lt, detail::region reg, - std::vector cm) - : type_(value_t::local_time), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_time_, lt); - } - basic_value(const local_datetime& ldt, detail::region reg, - std::vector cm) - : type_(value_t::local_datetime), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_datetime_, ldt); - } - basic_value(const offset_datetime& odt, detail::region reg, - std::vector cm) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->offset_datetime_, odt); - } - basic_value(const array_type& ary, detail::region reg, - std::vector cm) - : type_(value_t::array), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->array_, ary); - } - basic_value(const table_type& tab, detail::region reg, - std::vector cm) - : type_(value_t::table), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->table_, tab); - } - - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::pair parse_result, std::vector com) - : basic_value(std::move(parse_result.first), - std::move(parse_result.second), - std::move(com)) - {} - - // type checking and casting ============================================ - - template::value, - std::nullptr_t>::type = nullptr> - bool is() const noexcept - { - return detail::type_to_enum::value == this->type_; - } - bool is(value_t t) const noexcept {return t == this->type_;} - - bool is_uninitialized() const noexcept {return this->is(value_t::empty );} - bool is_boolean() const noexcept {return this->is(value_t::boolean );} - bool is_integer() const noexcept {return this->is(value_t::integer );} - bool is_floating() const noexcept {return this->is(value_t::floating );} - bool is_string() const noexcept {return this->is(value_t::string );} - bool is_offset_datetime() const noexcept {return this->is(value_t::offset_datetime);} - bool is_local_datetime() const noexcept {return this->is(value_t::local_datetime );} - bool is_local_date() const noexcept {return this->is(value_t::local_date );} - bool is_local_time() const noexcept {return this->is(value_t::local_time );} - bool is_array() const noexcept {return this->is(value_t::array );} - bool is_table() const noexcept {return this->is(value_t::table );} - - value_t type() const noexcept {return type_;} - - template - typename detail::enum_to_type::type& cast() & - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(*this); - } - template - typename detail::enum_to_type::type const& cast() const& - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(*this); - } - template - typename detail::enum_to_type::type&& cast() && - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(std::move(*this)); - } - - // ------------------------------------------------------------------------ - // nothrow version - - boolean const& as_boolean (const std::nothrow_t&) const& noexcept {return this->boolean_;} - integer const& as_integer (const std::nothrow_t&) const& noexcept {return this->integer_;} - floating const& as_floating (const std::nothrow_t&) const& noexcept {return this->floating_;} - string const& as_string (const std::nothrow_t&) const& noexcept {return this->string_;} - offset_datetime const& as_offset_datetime(const std::nothrow_t&) const& noexcept {return this->offset_datetime_;} - local_datetime const& as_local_datetime (const std::nothrow_t&) const& noexcept {return this->local_datetime_;} - local_date const& as_local_date (const std::nothrow_t&) const& noexcept {return this->local_date_;} - local_time const& as_local_time (const std::nothrow_t&) const& noexcept {return this->local_time_;} - array_type const& as_array (const std::nothrow_t&) const& noexcept {return this->array_.value();} - table_type const& as_table (const std::nothrow_t&) const& noexcept {return this->table_.value();} - - boolean & as_boolean (const std::nothrow_t&) & noexcept {return this->boolean_;} - integer & as_integer (const std::nothrow_t&) & noexcept {return this->integer_;} - floating & as_floating (const std::nothrow_t&) & noexcept {return this->floating_;} - string & as_string (const std::nothrow_t&) & noexcept {return this->string_;} - offset_datetime& as_offset_datetime(const std::nothrow_t&) & noexcept {return this->offset_datetime_;} - local_datetime & as_local_datetime (const std::nothrow_t&) & noexcept {return this->local_datetime_;} - local_date & as_local_date (const std::nothrow_t&) & noexcept {return this->local_date_;} - local_time & as_local_time (const std::nothrow_t&) & noexcept {return this->local_time_;} - array_type & as_array (const std::nothrow_t&) & noexcept {return this->array_.value();} - table_type & as_table (const std::nothrow_t&) & noexcept {return this->table_.value();} - - boolean && as_boolean (const std::nothrow_t&) && noexcept {return std::move(this->boolean_);} - integer && as_integer (const std::nothrow_t&) && noexcept {return std::move(this->integer_);} - floating && as_floating (const std::nothrow_t&) && noexcept {return std::move(this->floating_);} - string && as_string (const std::nothrow_t&) && noexcept {return std::move(this->string_);} - offset_datetime&& as_offset_datetime(const std::nothrow_t&) && noexcept {return std::move(this->offset_datetime_);} - local_datetime && as_local_datetime (const std::nothrow_t&) && noexcept {return std::move(this->local_datetime_);} - local_date && as_local_date (const std::nothrow_t&) && noexcept {return std::move(this->local_date_);} - local_time && as_local_time (const std::nothrow_t&) && noexcept {return std::move(this->local_time_);} - array_type && as_array (const std::nothrow_t&) && noexcept {return std::move(this->array_.value());} - table_type && as_table (const std::nothrow_t&) && noexcept {return std::move(this->table_.value());} - - // ======================================================================== - // throw version - // ------------------------------------------------------------------------ - // const reference {{{ - - boolean const& as_boolean() const& - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return this->boolean_; - } - integer const& as_integer() const& - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return this->integer_; - } - floating const& as_floating() const& - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return this->floating_; - } - string const& as_string() const& - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return this->string_; - } - offset_datetime const& as_offset_datetime() const& - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return this->offset_datetime_; - } - local_datetime const& as_local_datetime() const& - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return this->local_datetime_; - } - local_date const& as_local_date() const& - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return this->local_date_; - } - local_time const& as_local_time() const& - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return this->local_time_; - } - array_type const& as_array() const& - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return this->array_.value(); - } - table_type const& as_table() const& - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return this->table_.value(); - } - // }}} - // ------------------------------------------------------------------------ - // nonconst reference {{{ - - boolean & as_boolean() & - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return this->boolean_; - } - integer & as_integer() & - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return this->integer_; - } - floating & as_floating() & - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return this->floating_; - } - string & as_string() & - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return this->string_; - } - offset_datetime & as_offset_datetime() & - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return this->offset_datetime_; - } - local_datetime & as_local_datetime() & - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return this->local_datetime_; - } - local_date & as_local_date() & - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return this->local_date_; - } - local_time & as_local_time() & - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return this->local_time_; - } - array_type & as_array() & - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return this->array_.value(); - } - table_type & as_table() & - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return this->table_.value(); - } - - // }}} - // ------------------------------------------------------------------------ - // rvalue reference {{{ - - boolean && as_boolean() && - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return std::move(this->boolean_); - } - integer && as_integer() && - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return std::move(this->integer_); - } - floating && as_floating() && - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return std::move(this->floating_); - } - string && as_string() && - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return std::move(this->string_); - } - offset_datetime && as_offset_datetime() && - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return std::move(this->offset_datetime_); - } - local_datetime && as_local_datetime() && - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return std::move(this->local_datetime_); - } - local_date && as_local_date() && - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return std::move(this->local_date_); - } - local_time && as_local_time() && - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return std::move(this->local_time_); - } - array_type && as_array() && - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return std::move(this->array_.value()); - } - table_type && as_table() && - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return std::move(this->table_.value()); - } - // }}} - - // accessors ============================================================= - // - // may throw type_error or out_of_range - // - value_type& at(const key& k) - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::at(key): ", this->type_, *this); - } - if(this->as_table(std::nothrow).count(k) == 0) - { - detail::throw_key_not_found_error(*this, k); - } - return this->as_table(std::nothrow).at(k); - } - value_type const& at(const key& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::at(key): ", this->type_, *this); - } - if(this->as_table(std::nothrow).count(k) == 0) - { - detail::throw_key_not_found_error(*this, k); - } - return this->as_table(std::nothrow).at(k); - } - value_type& operator[](const key& k) - { - if(this->is_uninitialized()) - { - *this = table_type{}; - } - else if(!this->is_table()) // initialized, but not a table - { - detail::throw_bad_cast( - "toml::value::operator[](key): ", this->type_, *this); - } - return this->as_table(std::nothrow)[k]; - } - - value_type& at(const std::size_t idx) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::at(idx): ", this->type_, *this); - } - if(this->as_array(std::nothrow).size() <= idx) - { - throw std::out_of_range(detail::format_underline( - "toml::value::at(idx): no element corresponding to the index", { - {this->location(), concat_to_string("the length is ", - this->as_array(std::nothrow).size(), - ", and the specified index is ", idx)} - })); - } - return this->as_array().at(idx); - } - value_type const& at(const std::size_t idx) const - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::at(idx): ", this->type_, *this); - } - if(this->as_array(std::nothrow).size() <= idx) - { - throw std::out_of_range(detail::format_underline( - "toml::value::at(idx): no element corresponding to the index", { - {this->location(), concat_to_string("the length is ", - this->as_array(std::nothrow).size(), - ", and the specified index is ", idx)} - })); - } - return this->as_array(std::nothrow).at(idx); - } - - value_type& operator[](const std::size_t idx) noexcept - { - // no check... - return this->as_array(std::nothrow)[idx]; - } - value_type const& operator[](const std::size_t idx) const noexcept - { - // no check... - return this->as_array(std::nothrow)[idx]; - } - - void push_back(const value_type& x) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::push_back(value): ", this->type_, *this); - } - this->as_array(std::nothrow).push_back(x); - return; - } - void push_back(value_type&& x) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::push_back(value): ", this->type_, *this); - } - this->as_array(std::nothrow).push_back(std::move(x)); - return; - } - - template - value_type& emplace_back(Ts&& ... args) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::emplace_back(...): ", this->type_, *this); - } - this->as_array(std::nothrow).emplace_back(std::forward(args) ...); - return this->as_array(std::nothrow).back(); - } - - std::size_t size() const - { - switch(this->type_) - { - case value_t::array: - { - return this->as_array(std::nothrow).size(); - } - case value_t::table: - { - return this->as_table(std::nothrow).size(); - } - case value_t::string: - { - return this->as_string(std::nothrow).str.size(); - } - default: - { - throw type_error(detail::format_underline( - "toml::value::size(): bad_cast to container types", { - {this->location(), - concat_to_string("the actual type is ", this->type_)} - }), this->location()); - } - } - } - - std::size_t count(const key_type& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::count(key): ", this->type_, *this); - } - return this->as_table(std::nothrow).count(k); - } - - bool contains(const key_type& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::contains(key): ", this->type_, *this); - } - return (this->as_table(std::nothrow).count(k) != 0); - } - - source_location location() const - { - return source_location(this->region_info_.get()); - } - - comment_type const& comments() const noexcept {return this->comments_;} - comment_type& comments() noexcept {return this->comments_;} - - private: - - void cleanup() noexcept - { - switch(this->type_) - { - case value_t::string : {string_.~string(); return;} - case value_t::array : {array_.~array_storage(); return;} - case value_t::table : {table_.~table_storage(); return;} - default : return; - } - } - - // for error messages - template - friend region_base const* detail::get_region(const Value& v); - - template - friend void detail::change_region(Value& v, detail::region reg); - - private: - - using array_storage = detail::storage; - using table_storage = detail::storage; - - value_t type_; - union - { - boolean boolean_; - integer integer_; - floating floating_; - string string_; - offset_datetime offset_datetime_; - local_datetime local_datetime_; - local_date local_date_; - local_time local_time_; - array_storage array_; - table_storage table_; - }; - std::shared_ptr region_info_; - comment_type comments_; -}; - -// default toml::value and default array/table. -// TOML11_DEFAULT_COMMENT_STRATEGY is defined in comments.hpp -using value = basic_value; -using array = typename value::array_type; -using table = typename value::table_type; - -template class T, template class A> -inline bool -operator==(const basic_value& lhs, const basic_value& rhs) -{ - if(lhs.type() != rhs.type()) {return false;} - if(lhs.comments() != rhs.comments()) {return false;} - - switch(lhs.type()) - { - case value_t::boolean : - { - return lhs.as_boolean() == rhs.as_boolean(); - } - case value_t::integer : - { - return lhs.as_integer() == rhs.as_integer(); - } - case value_t::floating : - { - return lhs.as_floating() == rhs.as_floating(); - } - case value_t::string : - { - return lhs.as_string() == rhs.as_string(); - } - case value_t::offset_datetime: - { - return lhs.as_offset_datetime() == rhs.as_offset_datetime(); - } - case value_t::local_datetime: - { - return lhs.as_local_datetime() == rhs.as_local_datetime(); - } - case value_t::local_date: - { - return lhs.as_local_date() == rhs.as_local_date(); - } - case value_t::local_time: - { - return lhs.as_local_time() == rhs.as_local_time(); - } - case value_t::array : - { - return lhs.as_array() == rhs.as_array(); - } - case value_t::table : - { - return lhs.as_table() == rhs.as_table(); - } - case value_t::empty : {return true; } - default: {return false;} - } -} - -template class T, template class A> -inline bool operator!=(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs == rhs); -} - -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator<(const basic_value& lhs, const basic_value& rhs) -{ - if(lhs.type() != rhs.type()){return (lhs.type() < rhs.type());} - switch(lhs.type()) - { - case value_t::boolean : - { - return lhs.as_boolean() < rhs.as_boolean() || - (lhs.as_boolean() == rhs.as_boolean() && - lhs.comments() < rhs.comments()); - } - case value_t::integer : - { - return lhs.as_integer() < rhs.as_integer() || - (lhs.as_integer() == rhs.as_integer() && - lhs.comments() < rhs.comments()); - } - case value_t::floating : - { - return lhs.as_floating() < rhs.as_floating() || - (lhs.as_floating() == rhs.as_floating() && - lhs.comments() < rhs.comments()); - } - case value_t::string : - { - return lhs.as_string() < rhs.as_string() || - (lhs.as_string() == rhs.as_string() && - lhs.comments() < rhs.comments()); - } - case value_t::offset_datetime: - { - return lhs.as_offset_datetime() < rhs.as_offset_datetime() || - (lhs.as_offset_datetime() == rhs.as_offset_datetime() && - lhs.comments() < rhs.comments()); - } - case value_t::local_datetime: - { - return lhs.as_local_datetime() < rhs.as_local_datetime() || - (lhs.as_local_datetime() == rhs.as_local_datetime() && - lhs.comments() < rhs.comments()); - } - case value_t::local_date: - { - return lhs.as_local_date() < rhs.as_local_date() || - (lhs.as_local_date() == rhs.as_local_date() && - lhs.comments() < rhs.comments()); - } - case value_t::local_time: - { - return lhs.as_local_time() < rhs.as_local_time() || - (lhs.as_local_time() == rhs.as_local_time() && - lhs.comments() < rhs.comments()); - } - case value_t::array : - { - return lhs.as_array() < rhs.as_array() || - (lhs.as_array() == rhs.as_array() && - lhs.comments() < rhs.comments()); - } - case value_t::table : - { - return lhs.as_table() < rhs.as_table() || - (lhs.as_table() == rhs.as_table() && - lhs.comments() < rhs.comments()); - } - case value_t::empty : - { - return lhs.comments() < rhs.comments(); - } - default: - { - return lhs.comments() < rhs.comments(); - } - } -} - -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator<=(const basic_value& lhs, const basic_value& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator>(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs <= rhs); -} -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator>=(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs < rhs); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const basic_value& v, const std::string& comment, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, {{v.location(), comment}}, - std::move(hints), colorize); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const toml::basic_value& v1, const std::string& comment1, - const toml::basic_value& v2, const std::string& comment2, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, { - {v1.location(), comment1}, {v2.location(), comment2} - }, std::move(hints), colorize); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const toml::basic_value& v1, const std::string& comment1, - const toml::basic_value& v2, const std::string& comment2, - const toml::basic_value& v3, const std::string& comment3, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, {{v1.location(), comment1}, - {v2.location(), comment2}, {v3.location(), comment3} - }, std::move(hints), colorize); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, const toml::basic_value& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(v.as_boolean ());} - case value_t::integer : {return visitor(v.as_integer ());} - case value_t::floating : {return visitor(v.as_floating ());} - case value_t::string : {return visitor(v.as_string ());} - case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} - case value_t::local_datetime : {return visitor(v.as_local_datetime ());} - case value_t::local_date : {return visitor(v.as_local_date ());} - case value_t::local_time : {return visitor(v.as_local_time ());} - case value_t::array : {return visitor(v.as_array ());} - case value_t::table : {return visitor(v.as_table ());} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, toml::basic_value& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(v.as_boolean ());} - case value_t::integer : {return visitor(v.as_integer ());} - case value_t::floating : {return visitor(v.as_floating ());} - case value_t::string : {return visitor(v.as_string ());} - case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} - case value_t::local_datetime : {return visitor(v.as_local_datetime ());} - case value_t::local_date : {return visitor(v.as_local_date ());} - case value_t::local_time : {return visitor(v.as_local_time ());} - case value_t::array : {return visitor(v.as_array ());} - case value_t::table : {return visitor(v.as_table ());} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, toml::basic_value&& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(std::move(v.as_boolean ()));} - case value_t::integer : {return visitor(std::move(v.as_integer ()));} - case value_t::floating : {return visitor(std::move(v.as_floating ()));} - case value_t::string : {return visitor(std::move(v.as_string ()));} - case value_t::offset_datetime: {return visitor(std::move(v.as_offset_datetime()));} - case value_t::local_datetime : {return visitor(std::move(v.as_local_datetime ()));} - case value_t::local_date : {return visitor(std::move(v.as_local_date ()));} - case value_t::local_time : {return visitor(std::move(v.as_local_time ()));} - case value_t::array : {return visitor(std::move(v.as_array ()));} - case value_t::table : {return visitor(std::move(v.as_table ()));} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -}// toml -#endif// TOML11_VALUE diff --git a/tests/functional/add.sh b/tests/functional/add.sh index a6cf88e1a..3b37ee7d4 100755 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -31,7 +31,7 @@ test "$hash1" = "sha256:$hash2" #### New style commands -clearStore +clearStoreIfPossible ( path1=$(nix store add ./dummy) diff --git a/tests/functional/bash-profile.sh b/tests/functional/bash-profile.sh index 6cfa5bd9c..4228d4a20 100755 --- a/tests/functional/bash-profile.sh +++ b/tests/functional/bash-profile.sh @@ -2,10 +2,10 @@ source common.sh -sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > "$TEST_ROOT"/nix-profile.sh user=$(whoami) -rm -rf $TEST_HOME $TEST_ROOT/profile-var -mkdir -p $TEST_HOME +rm -rf "$TEST_HOME" "$TEST_ROOT/profile-var" +mkdir -p "$TEST_HOME" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency diff --git a/tests/functional/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh index 0303e9410..5046d0064 100755 --- a/tests/functional/binary-cache-build-remote.sh +++ b/tests/functional/binary-cache-build-remote.sh @@ -2,7 +2,9 @@ source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible clearCacheCache # Fails without remote builders @@ -12,7 +14,7 @@ clearCacheCache outPath=$(nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix) # Test that the path exactly exists in the destination store. -nix path-info --store "file://$cacheDir" $outPath +nix path-info --store "file://$cacheDir" "$outPath" # Succeeds without any build capability because no-op nix-build --store "file://$cacheDir" -j0 dependencies.nix diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 54a3687ca..6a177b657 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + needLocalStore "'--no-require-sigs' can’t be used with the daemon" # We can produce drvs directly into the binary cache @@ -14,9 +16,9 @@ clearStore clearCache outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --to file://$cacheDir $outPath +nix copy --to "file://$cacheDir" "$outPath" -readarray -t paths < <(nix path-info --all --json --store file://$cacheDir | jq 'keys|sort|.[]' -r) +readarray -t paths < <(nix path-info --all --json --store "file://$cacheDir" | jq 'keys|sort|.[]' -r) [[ "${#paths[@]}" -eq 3 ]] for path in "${paths[@]}"; do [[ "$path" =~ -dependencies-input-0$ ]] \ @@ -25,16 +27,16 @@ for path in "${paths[@]}"; do done # Test copying build logs to the binary cache. -expect 1 nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available' -nix store copy-log --to file://$cacheDir $outPath -nix log --store file://$cacheDir $outPath | grep FOO -rm -rf $TEST_ROOT/var/log/nix -expect 1 nix log $outPath 2>&1 | grep 'is not available' -nix log --substituters file://$cacheDir $outPath | grep FOO +expect 1 nix log --store "file://$cacheDir" "$outPath" 2>&1 | grep 'is not available' +nix store copy-log --to "file://$cacheDir" "$outPath" +nix log --store "file://$cacheDir" "$outPath" | grep FOO +rm -rf "$TEST_ROOT/var/log/nix" +expect 1 nix log "$outPath" 2>&1 | grep 'is not available' +nix log --substituters "file://$cacheDir" "$outPath" | grep FOO # Test copying build logs from the binary cache. -nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath)^'*' -nix log $outPath | grep FOO +nix store copy-log --from "file://$cacheDir" "$(nix-store -qd "$outPath")"^'*' +nix log "$outPath" | grep FOO basicDownloadTests() { # No uploading tests bcause upload with force HTTP doesn't work. @@ -46,15 +48,15 @@ basicDownloadTests() { nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" - nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath" - [ -x $outPath/program ] + [ -x "$outPath/program" ] # But with the right configuration, "nix-env -qas" should also work. clearStore clearCacheCache - echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info + echo "WantMassQuery: 1" >> "$cacheDir/nix-cache-info" nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" @@ -62,12 +64,12 @@ basicDownloadTests() { x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only) [ -z "$x" ] - nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath" - nix-store --check-validity $outPath - nix-store -qR $outPath | grep input-2 + nix-store --check-validity "$outPath" + nix-store -qR "$outPath" | grep input-2 - echo "WantMassQuery: 0" >> $cacheDir/nix-cache-info + echo "WantMassQuery: 0" >> "$cacheDir/nix-cache-info" } @@ -83,22 +85,22 @@ basicDownloadTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) -mv $nar $nar.good -mkdir -p $TEST_ROOT/empty -nix-store --dump $TEST_ROOT/empty | xz > $nar +nar=$(find "$cacheDir/nar/" -type f -name "*.nar.xz" | head -n1) +mv "$nar" "$nar".good +mkdir -p "$TEST_ROOT/empty" +nix-store --dump "$TEST_ROOT/empty" | xz > "$nar" -expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grepQuiet "hash mismatch" $TEST_ROOT/log +expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "hash mismatch" "$TEST_ROOT/log" -mv $nar.good $nar +mv "$nar".good "$nar" # Test whether this unsigned cache is rejected if the user requires signed caches. clearStore clearCacheCache -if nix-store --substituters "file://$cacheDir" -r $outPath; then +if nix-store --substituters "file://$cacheDir" -r "$outPath"; then echo "unsigned binary cache incorrectly accepted" exit 1 fi @@ -107,131 +109,134 @@ fi # Test whether fallback works if a NAR has disappeared. This does not require --fallback. clearStore -mv $cacheDir/nar $cacheDir/nar2 +mv "$cacheDir/nar" "$cacheDir/nar2" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" -mv $cacheDir/nar2 $cacheDir/nar +mv "$cacheDir/nar2" "$cacheDir/nar" # Test whether fallback works if a NAR is corrupted. This does require --fallback. clearStore -mv $cacheDir/nar $cacheDir/nar2 -mkdir $cacheDir/nar -for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done +mv "$cacheDir/nar" "$cacheDir/nar2" +mkdir "$cacheDir/nar" +for i in $(cd "$cacheDir/nar2" && echo *); do touch "$cacheDir"/nar/"$i"; done -(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) +(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result") -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" --fallback -rm -rf $cacheDir/nar -mv $cacheDir/nar2 $cacheDir/nar +rm -rf "$cacheDir/nar" +mv "$cacheDir/nar2" "$cacheDir/nar" # Test whether building works if the binary cache contains an # incomplete closure. clearStore -rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) +rm -v "$(grep -l "StorePath:.*dependencies-input-2" "$cacheDir"/*.narinfo)" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grepQuiet "copying path.*input-0" $TEST_ROOT/log -grepQuiet "copying path.*input-2" $TEST_ROOT/log -grepQuiet "copying path.*top" $TEST_ROOT/log +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "copying path.*input-0" "$TEST_ROOT/log" +grepQuiet "copying path.*input-2" "$TEST_ROOT/log" +grepQuiet "copying path.*top" "$TEST_ROOT/log" # Idem, but without cached .narinfo. clearStore clearCacheCache -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grepQuiet "don't know how to build" $TEST_ROOT/log -grepQuiet "building.*input-1" $TEST_ROOT/log -grepQuiet "building.*input-2" $TEST_ROOT/log -grepQuiet "copying path.*input-0" $TEST_ROOT/log -grepQuiet "copying path.*top" $TEST_ROOT/log +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "don't know how to build" "$TEST_ROOT/log" +grepQuiet "building.*input-1" "$TEST_ROOT/log" +grepQuiet "building.*input-2" "$TEST_ROOT/log" +grepQuiet "copying path.*input-0" "$TEST_ROOT/log" +grepQuiet "copying path.*top" "$TEST_ROOT/log" # Create a signed binary cache. clearCache clearCacheCache -nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk1 -publicKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk1) +nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk1" +publicKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk1") -nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk2 -badKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk2) +nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk2" +badKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk2") -nix key generate-secret --key-name foo.nixos.org-1 > $TEST_ROOT/sk3 -otherKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk3) +nix key generate-secret --key-name foo.nixos.org-1 > "$TEST_ROOT/sk3" +otherKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk3") -_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP='' nix copy --to "file://$cacheDir"?secret-key="$TEST_ROOT"/sk1 "$outPath" # Downloading should fail if we don't provide a key. clearStore clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir") +(! nix-store -r "$outPath" --substituters "file://$cacheDir") # And it should fail if we provide an incorrect key. clearStore clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$badKey") +(! nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$badKey") # It should succeed if we provide the correct key. -nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" +nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" # It should fail if we corrupt the .narinfo. clearStore cacheDir2=$TEST_ROOT/binary-cache-2 -rm -rf $cacheDir2 -cp -r $cacheDir $cacheDir2 +rm -rf "$cacheDir2" +cp -r "$cacheDir" "$cacheDir2" -for i in $cacheDir2/*.narinfo; do - grep -v References $i > $i.tmp - mv $i.tmp $i +for i in "$cacheDir2"/*.narinfo; do + grep -v References "$i" > "$i".tmp + mv "$i".tmp "$i" done clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") +(! nix-store -r "$outPath" --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") # If we provide a bad and a good binary cache, it should succeed. -nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" +nix-store -r "$outPath" --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" unset _NIX_FORCE_HTTP # Test 'nix verify --all' on a binary cache. -nix store verify -vvvvv --all --store file://$cacheDir --no-trust +nix store verify -vvvvv --all --store "file://$cacheDir" --no-trust # Test local NAR caching. narCache=$TEST_ROOT/nar-cache -rm -rf $narCache -mkdir $narCache +rm -rf "$narCache" +mkdir "$narCache" -[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] rm -rfv "$cacheDir/nar" -[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] -(! nix store cat --store file://$cacheDir $outPath/foobar) +(! nix store cat --store "file://$cacheDir" "$outPath/foobar") # Test NAR listing generation. clearCache + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' with import ./config.nix; mkDerivation { @@ -240,16 +245,18 @@ outPath=$(nix-build --no-out-link -E ' } ') -nix copy --to file://$cacheDir?write-nar-listing=1 $outPath +nix copy --to "file://$cacheDir"?write-nar-listing=1 "$outPath" diff -u \ - <(jq -S < $cacheDir/$(basename $outPath | cut -c1-32).ls) \ + <(jq -S < "$cacheDir/$(basename "$outPath" | cut -c1-32).ls") \ <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S) # Test debug info index generation. clearCache +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' with import ./config.nix; mkDerivation { @@ -258,14 +265,16 @@ outPath=$(nix-build --no-out-link -E ' } ') -nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath +nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" "$outPath" diff -u \ - <(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \ + <(jq -S < "$cacheDir"/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) \ <(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S) # Test against issue https://github.com/NixOS/nix/issues/3964 -# + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 expr=' with import ./config.nix; mkDerivation { @@ -275,22 +284,22 @@ expr=' } ' outPath=$(nix-build --no-out-link -E "$expr") -docPath=$(nix-store -q --references $outPath) +docPath=$(nix-store -q --references "$outPath") # $ nix-store -q --tree $outPath # ...-multi-output # +---...-multi-output-doc -nix copy --to "file://$cacheDir" $outPath +nix copy --to "file://$cacheDir" "$outPath" hashpart() { basename "$1" | cut -c1-32 } # break the closure of out by removing doc -rm $cacheDir/$(hashpart $docPath).narinfo +rm "$cacheDir/$(hashpart "$docPath")".narinfo -nix-store --delete $outPath $docPath +nix-store --delete "$outPath" "$docPath" # -vvv is the level that logs during the loop timeout 60 nix-build --no-out-link -E "$expr" --option substituters "file://$cacheDir" \ --option trusted-binary-caches "file://$cacheDir" --no-require-sigs diff --git a/tests/functional/brotli.sh b/tests/functional/brotli.sh index 02a2a0875..327eab4a5 100755 --- a/tests/functional/brotli.sh +++ b/tests/functional/brotli.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache @@ -9,15 +11,15 @@ cacheURI="file://$cacheDir?compression=br" outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -HASH=$(nix hash path $outPath) +HASH=$(nix hash path "$outPath") clearStore clearCacheCache -nix copy --from $cacheURI $outPath --no-check-sigs +nix copy --from "$cacheURI" "$outPath" --no-check-sigs -HASH2=$(nix hash path $outPath) +HASH2=$(nix hash path "$outPath") -[[ $HASH = $HASH2 ]] +[[ $HASH == "$HASH2" ]] diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh index 2ef3008f6..18841509d 100755 --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -2,29 +2,29 @@ source common.sh -clearStore +clearStoreIfPossible # https://github.com/NixOS/nix/issues/6572 issue_6572_independent_outputs() { - nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json + nix build -f multiple-outputs.nix --json independent --no-link > "$TEST_ROOT"/independent.json # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) nix-store --delete "$p" # Clean up for next test # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. - nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)" + nix-store --delete "$(jq -r <"$TEST_ROOT"/independent.json .[0].outputs.first)" p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) - cmp $p < $TEST_ROOT/a.json + nix build -f multiple-outputs.nix --json a --no-link > "$TEST_ROOT"/a.json # # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) nix-store --delete "$p" # Clean up for next test # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. - nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)" + nix-store --delete "$(jq -r <"$TEST_ROOT"/a.json .[0].outputs.second)" p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) - cmp $p <> "$NIX_CONF_DIR/nix.conf" + echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> "$test_nix_conf" } registerBuildHook diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index 269f7f112..4eccb73e0 100755 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -4,6 +4,7 @@ source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS restartDaemon requireSandboxSupport diff --git a/tests/functional/build-remote-trustless-should-pass-2.sh b/tests/functional/build-remote-trustless-should-pass-2.sh index ba5d1ff7a..34ce7fbe4 100755 --- a/tests/functional/build-remote-trustless-should-pass-2.sh +++ b/tests/functional/build-remote-trustless-should-pass-2.sh @@ -4,6 +4,8 @@ source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS + restartDaemon # Remote doesn't trust us diff --git a/tests/functional/build-remote-trustless-should-pass-3.sh b/tests/functional/build-remote-trustless-should-pass-3.sh index 187b89948..d01d79191 100755 --- a/tests/functional/build-remote-trustless-should-pass-3.sh +++ b/tests/functional/build-remote-trustless-should-pass-3.sh @@ -4,6 +4,7 @@ source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS restartDaemon # Remote doesn't trusts us, but this is fine because we are only diff --git a/tests/functional/build.sh b/tests/functional/build.sh index a14e6d672..9de953d8c 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible # Make sure that 'nix build' returns all outputs by default. nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' diff --git a/tests/functional/ca/build-cache.sh b/tests/functional/ca/build-cache.sh index 6a4080fec..5cc71823e 100644 --- a/tests/functional/ca/build-cache.sh +++ b/tests/functional/ca/build-cache.sh @@ -26,7 +26,8 @@ copyAttr () { # Note: to copy CA derivations, we need to copy the realisations, which # currently requires naming the installables, not just the derivation output # path. - nix copy --to file://$cacheDir "${args[@]}" + + nix copy --to "file://$cacheDir" "${args[@]}" } testRemoteCacheFor () { @@ -35,7 +36,7 @@ testRemoteCacheFor () { copyAttr "$derivationPath" 1 clearStore # Check nothing gets built. - buildAttr "$derivationPath" 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse " will be built:" + buildAttr "$derivationPath" 1 --option substituters "file://$cacheDir" --no-require-sigs |& grepQuietInverse " will be built:" } testRemoteCache () { @@ -48,4 +49,4 @@ testRemoteCache () { } clearStore -testRemoteCache \ No newline at end of file +testRemoteCache diff --git a/tests/functional/ca/build.sh b/tests/functional/ca/build.sh index e1a8a7625..e5ad9d2a0 100644 --- a/tests/functional/ca/build.sh +++ b/tests/functional/ca/build.sh @@ -20,11 +20,11 @@ testDeterministicCA () { testCutoffFor () { local out1 out2 - out1=$(buildAttr $1 1) + out1=$(buildAttr "$1" 1) # The seed only changes the root derivation, and not it's output, so the # dependent derivations should only need to be built once. buildAttr rootCA 2 - out2=$(buildAttr $1 2 -j0) + out2=$(buildAttr "$1" 2 -j0) test "$out1" == "$out2" } @@ -41,7 +41,7 @@ testGC () { nix-instantiate ./content-addressed.nix -A rootCA --arg seed 5 nix-collect-garbage --option keep-derivations true clearStore - buildAttr rootCA 1 --out-link $TEST_ROOT/rootCA + buildAttr rootCA 1 --out-link "$TEST_ROOT"/rootCA nix-collect-garbage buildAttr rootCA 1 -j0 } @@ -55,7 +55,7 @@ testNixCommand () { testNormalization () { clearStore outPath=$(buildAttr rootCA 1) - test "$(stat -c %Y $outPath)" -eq 1 + test "$(stat -c %Y "$outPath")" -eq 1 } clearStore diff --git a/tests/functional/ca/common.sh b/tests/functional/ca/common.sh index b104b5a78..48f1ac46b 100644 --- a/tests/functional/ca/common.sh +++ b/tests/functional/ca/common.sh @@ -2,4 +2,6 @@ source ../common.sh enableFeatures "ca-derivations" +TODO_NixOS + restartDaemon diff --git a/tests/functional/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh index c1480fd17..1e2a8fe35 100644 --- a/tests/functional/ca/derivation-json.sh +++ b/tests/functional/ca/derivation-json.sh @@ -1,29 +1,31 @@ +#!/usr/bin/env bash +# source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 drvPath=$(nix-instantiate ../simple.nix) -nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json +nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/simple.json -drvPath2=$(nix derivation add < $TEST_HOME/simple.json) +drvPath2=$(nix derivation add < "$TEST_HOME"/simple.json) [[ "$drvPath" = "$drvPath2" ]] # Content-addressed derivations can be renamed. -jq '.name = "foo"' < $TEST_HOME/simple.json > $TEST_HOME/foo.json -drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) +jq '.name = "foo"' < "$TEST_HOME"/simple.json > "$TEST_HOME"/foo.json +drvPath3=$(nix derivation add --dry-run < "$TEST_HOME"/foo.json) # With --dry-run nothing is actually written [[ ! -e "$drvPath3" ]] # But the JSON is rejected without the experimental feature -expectStderr 1 nix derivation add < $TEST_HOME/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" +expectStderr 1 nix derivation add < "$TEST_HOME"/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" # Without --dry-run it is actually written -drvPath4=$(nix derivation add < $TEST_HOME/foo.json) +drvPath4=$(nix derivation add < "$TEST_HOME"/foo.json) [[ "$drvPath4" = "$drvPath3" ]] [[ -e "$drvPath3" ]] # The modified derivation read back as JSON matches -nix derivation show $drvPath3 | jq .[] > $TEST_HOME/foo-read.json -diff $TEST_HOME/foo.json $TEST_HOME/foo-read.json +nix derivation show "$drvPath3" | jq .[] > "$TEST_HOME"/foo-read.json +diff "$TEST_HOME"/foo.json "$TEST_HOME"/foo-read.json diff --git a/tests/functional/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh index da9cd8fb4..0baf15cc2 100644 --- a/tests/functional/ca/duplicate-realisation-in-closure.sh +++ b/tests/functional/ca/duplicate-realisation-in-closure.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source ./common.sh requireDaemonNewerThan "2.4pre20210625" @@ -5,7 +7,7 @@ requireDaemonNewerThan "2.4pre20210625" export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" -rm -rf $REMOTE_STORE_DIR +rm -rf "$REMOTE_STORE_DIR" clearStore # Build dep1 and push that to the binary cache. diff --git a/tests/functional/ca/nix-copy.sh b/tests/functional/ca/nix-copy.sh index 7a8307a4e..f77b00030 100755 --- a/tests/functional/ca/nix-copy.sh +++ b/tests/functional/ca/nix-copy.sh @@ -15,13 +15,13 @@ testOneCopy () { rm -rf "$REMOTE_STORE_DIR" attrPath="$1" - nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix + nix copy --to "$REMOTE_STORE" "$attrPath" --file ./content-addressed.nix ensureCorrectlyCopied "$attrPath" # Ensure that we can copy back what we put in the store clearStore - nix copy --from $REMOTE_STORE \ + nix copy --from "$REMOTE_STORE" \ --file ./content-addressed.nix "$attrPath" \ --no-check-sigs } diff --git a/tests/functional/ca/nix-run.sh b/tests/functional/ca/nix-run.sh index 5f46518e8..920950c11 100755 --- a/tests/functional/ca/nix-run.sh +++ b/tests/functional/ca/nix-run.sh @@ -4,4 +4,4 @@ source common.sh FLAKE_PATH=path:$PWD -nix run --no-write-lock-file $FLAKE_PATH#runnable +nix run --no-write-lock-file "$FLAKE_PATH#runnable" diff --git a/tests/functional/ca/signatures.sh b/tests/functional/ca/signatures.sh index eb18a4130..f69a205d2 100644 --- a/tests/functional/ca/signatures.sh +++ b/tests/functional/ca/signatures.sh @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + source common.sh clearStore clearCache -nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 -pk1=$(cat $TEST_ROOT/pk1) +nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT/sk1" "$TEST_ROOT/pk1" +pk1=$(cat "$TEST_ROOT/pk1") export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" @@ -19,16 +21,16 @@ testOneCopy () { rm -rf "$REMOTE_STORE_DIR" attrPath="$1" - nix copy -vvvv --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix \ + nix copy -vvvv --to "$REMOTE_STORE" "$attrPath" --file ./content-addressed.nix \ --secret-key-files "$TEST_ROOT/sk1" --show-trace ensureCorrectlyCopied "$attrPath" # Ensure that we can copy back what we put in the store clearStore - nix copy --from $REMOTE_STORE \ + nix copy --from "$REMOTE_STORE" \ --file ./content-addressed.nix "$attrPath" \ - --trusted-public-keys $pk1 + --trusted-public-keys "$pk1" } for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index ea981adc4..9728470f0 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -4,9 +4,10 @@ source common.sh +# shellcheck disable=SC1111 needLocalStore "“--no-require-sigs” can’t be used with the daemon" -rm -rf $TEST_ROOT/binary_cache +rm -rf "$TEST_ROOT/binary_cache" export REMOTE_STORE_DIR=$TEST_ROOT/binary_cache export REMOTE_STORE=file://$REMOTE_STORE_DIR @@ -17,11 +18,11 @@ buildDrvs () { # Populate the remote cache clearStore -nix copy --to $REMOTE_STORE --file ./content-addressed.nix +nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix # Restart the build on an empty store, ensuring that we don't build clearStore -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA # Check that the thing we’ve just substituted has its realisation stored nix realisation info --file ./content-addressed.nix transitivelyDependentCA # Check that its dependencies have it too @@ -63,9 +64,9 @@ clearStore # Add the realisations of rootCA to the cachecache clearCacheCache export _NIX_FORCE_HTTP=1 -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm $REMOTE_STORE_DIR/realisations/* -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +rm "$REMOTE_STORE_DIR"/realisations/* +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 diff --git a/tests/functional/case-hack.sh b/tests/functional/case-hack.sh index fbc8242ff..feddc6583 100755 --- a/tests/functional/case-hack.sh +++ b/tests/functional/case-hack.sh @@ -2,20 +2,23 @@ source common.sh +TODO_NixOS + clearStore -rm -rf $TEST_ROOT/case +rm -rf "$TEST_ROOT/case" -opts="--option use-case-hack true" +opts=("--option" "use-case-hack" "true") # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. -nix-store $opts --restore $TEST_ROOT/case < case.nar -nix-store $opts --dump $TEST_ROOT/case > $TEST_ROOT/case.nar -cmp case.nar $TEST_ROOT/case.nar -[ "$(nix-hash $opts --type sha256 $TEST_ROOT/case)" = "$(nix-hash --flat --type sha256 case.nar)" ] + +nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case.nar +nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > "$TEST_ROOT/case.nar" +cmp case.nar "$TEST_ROOT/case.nar" +[ "$(nix-hash "${opts[@]}" --type sha256 "$TEST_ROOT/case")" = "$(nix-hash --flat --type sha256 case.nar)" ] # Check whether we detect true collisions (e.g. those remaining after # removal of the suffix). touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" -(! nix-store $opts --dump $TEST_ROOT/case > /dev/null) +(! nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null) diff --git a/tests/functional/lang-test-infra.sh b/tests/functional/characterisation-test-infra.sh similarity index 98% rename from tests/functional/lang-test-infra.sh rename to tests/functional/characterisation-test-infra.sh index f32ccef05..279454550 100755 --- a/tests/functional/lang-test-infra.sh +++ b/tests/functional/characterisation-test-infra.sh @@ -3,7 +3,7 @@ # Test the function for lang.sh source common.sh -source lang/framework.sh +source characterisation/framework.sh # We are testing this, so don't want outside world to affect us. unset _NIX_TEST_ACCEPT diff --git a/tests/functional/lang/empty.exp b/tests/functional/characterisation/empty similarity index 100% rename from tests/functional/lang/empty.exp rename to tests/functional/characterisation/empty diff --git a/tests/functional/characterisation/framework.sh b/tests/functional/characterisation/framework.sh new file mode 100644 index 000000000..913fdd967 --- /dev/null +++ b/tests/functional/characterisation/framework.sh @@ -0,0 +1,77 @@ +# shellcheck shell=bash + +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=characterisation/empty + fi + + # Diff so we get a nice message + if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then + echo "FAIL: evaluation result of $testName not as expected" + # shellcheck disable=SC2034 + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} + +function characterisationTestExit() { + # Make sure shellcheck knows all these will be defined by the caller + : "${badDiff?} ${badExitCode?}" + + if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + set +x + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + set -x + else + set +x + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + set -x + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi + else + if (( "$badDiff" )); then + set +x + echo '' + echo 'You can rerun this test with:' + echo '' + echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.test" + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + set -x + fi + exit $(( "$badExitCode" + "$badDiff" )) + fi +} diff --git a/tests/functional/check-refs.sh b/tests/functional/check-refs.sh index 2cebdd84d..5c3ac915e 100755 --- a/tests/functional/check-refs.sh +++ b/tests/functional/check-refs.sh @@ -2,54 +2,61 @@ source common.sh +TODO_NixOS + clearStore RESULT=$TEST_ROOT/result -dep=$(nix-build -o $RESULT check-refs.nix -A dep) +dep=$(nix-build -o "$RESULT" check-refs.nix -A dep) # test1 references dep, not itself. -test1=$(nix-build -o $RESULT check-refs.nix -A test1) -nix-store -q --references $test1 | grepQuietInverse $test1 -nix-store -q --references $test1 | grepQuiet $dep +test1=$(nix-build -o "$RESULT" check-refs.nix -A test1) +nix-store -q --references "$test1" | grepQuietInverse "$test1" +nix-store -q --references "$test1" | grepQuiet "$dep" # test2 references src, not itself nor dep. -test2=$(nix-build -o $RESULT check-refs.nix -A test2) -nix-store -q --references $test2 | grepQuietInverse $test2 -nix-store -q --references $test2 | grepQuietInverse $dep -nix-store -q --references $test2 | grepQuiet aux-ref +test2=$(nix-build -o "$RESULT" check-refs.nix -A test2) +nix-store -q --references "$test2" | grepQuietInverse "$test2" +nix-store -q --references "$test2" | grepQuietInverse "$dep" +nix-store -q --references "$test2" | grepQuiet aux-ref # test3 should fail (unallowed ref). -(! nix-build -o $RESULT check-refs.nix -A test3) +(! nix-build -o "$RESULT" check-refs.nix -A test3) # test4 should succeed. -nix-build -o $RESULT check-refs.nix -A test4 +nix-build -o "$RESULT" check-refs.nix -A test4 # test5 should succeed. -nix-build -o $RESULT check-refs.nix -A test5 +nix-build -o "$RESULT" check-refs.nix -A test5 # test6 should fail (unallowed self-ref). -(! nix-build -o $RESULT check-refs.nix -A test6) +(! nix-build -o "$RESULT" check-refs.nix -A test6) # test7 should succeed (allowed self-ref). -nix-build -o $RESULT check-refs.nix -A test7 +nix-build -o "$RESULT" check-refs.nix -A test7 # test8 should fail (toFile depending on derivation output). -(! nix-build -o $RESULT check-refs.nix -A test8) +(! nix-build -o "$RESULT" check-refs.nix -A test8) # test9 should fail (disallowed reference). -(! nix-build -o $RESULT check-refs.nix -A test9) +(! nix-build -o "$RESULT" check-refs.nix -A test9) # test10 should succeed (no disallowed references). -nix-build -o $RESULT check-refs.nix -A test10 +nix-build -o "$RESULT" check-refs.nix -A test10 -if isDaemonNewer 2.12pre20230103; then - if ! isDaemonNewer 2.16.0; then - enableFeatures discard-references - restartDaemon +if ! isTestOnNixOS; then + # If we have full control over our store, we can test some more things. + + if isDaemonNewer 2.12pre20230103; then + if ! isDaemonNewer 2.16.0; then + enableFeatures discard-references + restartDaemon + fi + + # test11 should succeed. + test11=$(nix-build -o "$RESULT" check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] fi - # test11 should succeed. - test11=$(nix-build -o $RESULT check-refs.nix -A test11) - [[ -z $(nix-store -q --references "$test11") ]] fi diff --git a/tests/functional/check-reqs.sh b/tests/functional/check-reqs.sh index 2bcd558fd..34eb133db 100755 --- a/tests/functional/check-reqs.sh +++ b/tests/functional/check-reqs.sh @@ -2,17 +2,17 @@ source common.sh -clearStore +clearStoreIfPossible RESULT=$TEST_ROOT/result -nix-build -o $RESULT check-reqs.nix -A test1 +nix-build -o "$RESULT" check-reqs.nix -A test1 -(! nix-build -o $RESULT check-reqs.nix -A test2) -(! nix-build -o $RESULT check-reqs.nix -A test3) -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep1' -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep2' -(! nix-build -o $RESULT check-reqs.nix -A test5) -(! nix-build -o $RESULT check-reqs.nix -A test6) +(! nix-build -o "$RESULT" check-reqs.nix -A test2) +(! nix-build -o "$RESULT" check-reqs.nix -A test3) +(! nix-build -o "$RESULT" check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep1' +(! nix-build -o "$RESULT" check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep2' +(! nix-build -o "$RESULT" check-reqs.nix -A test5) +(! nix-build -o "$RESULT" check-reqs.nix -A test6) -nix-build -o $RESULT check-reqs.nix -A test7 +nix-build -o "$RESULT" check-reqs.nix -A test7 diff --git a/tests/functional/check.sh b/tests/functional/check.sh index efb93eeb0..9b15dccb6 100755 --- a/tests/functional/check.sh +++ b/tests/functional/check.sh @@ -15,6 +15,8 @@ checkBuildTempDirRemoved () # written to build temp directories to verify created by this instance checkBuildId=$(date +%s%N) +TODO_NixOS + clearStore nix-build dependencies.nix --no-out-link @@ -46,7 +48,10 @@ test_custom_build_dir() { --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> $TEST_ROOT/log || status=$? [ "$status" = "100" ] [[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]] - local buildDir="$customBuildDir/nix-build-"* + local buildDir="$customBuildDir/nix-build-"*"" + if [[ -e $buildDir/build ]]; then + buildDir=$buildDir/build + fi grep $checkBuildId $buildDir/checkBuildId } test_custom_build_dir @@ -76,6 +81,8 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi +TODO_NixOS + clearStore path=$(nix-build check.nix -A fetchurl --no-out-link) diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 60b9c50a7..03803a2b9 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -2,34 +2,34 @@ source common.sh -echo example > $TEST_ROOT/example.txt -mkdir -p $TEST_ROOT/x +echo example > "$TEST_ROOT"/example.txt +mkdir -p "$TEST_ROOT/x" export NIX_STORE_DIR=/nix2/store -CORRECT_PATH=$(cd $TEST_ROOT && nix-store --store ./x --add example.txt) +CORRECT_PATH=$(cd "$TEST_ROOT" && nix-store --store ./x --add example.txt) [[ $CORRECT_PATH =~ ^/nix2/store/.*-example.txt$ ]] -PATH1=$(cd $TEST_ROOT && nix path-info --store ./x $CORRECT_PATH) -[ $CORRECT_PATH == $PATH1 ] +PATH1=$(cd "$TEST_ROOT" && nix path-info --store ./x "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH1" ] -PATH2=$(nix path-info --store "$TEST_ROOT/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH2 ] +PATH2=$(nix path-info --store "$TEST_ROOT/x" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH2" ] -PATH3=$(nix path-info --store "local?root=$TEST_ROOT/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH3 ] +PATH3=$(nix path-info --store "local?root=$TEST_ROOT/x" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH3" ] # Ensure store info trusted works with local store -nix --store $TEST_ROOT/x store info --json | jq -e '.trusted' +nix --store "$TEST_ROOT/x" store info --json | jq -e '.trusted' # Test building in a chroot store. if canUseSandbox; then flakeDir=$TEST_ROOT/flake - mkdir -p $flakeDir + mkdir -p "$flakeDir" - cat > $flakeDir/flake.nix < "$flakeDir"/flake.nix < "$test_nix_conf_dir/nix.conf" <&2 + exit 1 +} + set +x commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" source "$commonDir/subst-vars.sh" # Make sure shellcheck knows all these will be defined by the above generated snippet -: "${PATH?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" +: "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" + +source "$commonDir/paths.sh" +source "$commonDir/test-root.sh" + +test_nix_conf_dir=$TEST_ROOT/etc +test_nix_conf=$test_nix_conf_dir/nix.conf -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} -export NIX_STORE_DIR -if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then - # Maybe the build directory is symlinked. - export NIX_IGNORE_SYMLINK_STORE=1 - NIX_STORE_DIR=$TEST_ROOT/store -fi -export NIX_LOCALSTATE_DIR=$TEST_ROOT/var -export NIX_LOG_DIR=$TEST_ROOT/var/log/nix -export NIX_STATE_DIR=$TEST_ROOT/var/nix -export NIX_CONF_DIR=$TEST_ROOT/etc -export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket -unset NIX_USER_CONF_FILES -export _NIX_TEST_SHARED=$TEST_ROOT/shared -if [[ -n $NIX_STORE ]]; then - export _NIX_TEST_NO_SANDBOX=1 -fi -export _NIX_IN_TEST=$TEST_ROOT/shared -export _NIX_TEST_NO_LSOF=1 -export NIX_REMOTE=${NIX_REMOTE_-} -unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home + +if ! isTestOnNixOS; then + export NIX_STORE_DIR + if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store + fi + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_CONF_DIR=$test_nix_conf_dir + export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket + unset NIX_USER_CONF_FILES + export _NIX_TEST_SHARED=$TEST_ROOT/shared + if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 + fi + export _NIX_IN_TEST=$TEST_ROOT/shared + export _NIX_TEST_NO_LSOF=1 + export NIX_REMOTE=${NIX_REMOTE_-} + +fi # ! isTestOnNixOS + +unset NIX_PATH export HOME=$TEST_HOME unset XDG_STATE_HOME unset XDG_DATA_HOME @@ -43,14 +62,6 @@ unset XDG_CONFIG_HOME unset XDG_CONFIG_DIRS unset XDG_CACHE_HOME -if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then - export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH -fi -DAEMON_PATH="$PATH" -if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" -fi - export IMPURE_VAR1=foo export IMPURE_VAR2=bar @@ -65,7 +76,25 @@ clearProfiles() { rm -rf "$profiles" } +# Clear the store, but do not fail if we're in an environment where we can't. +# This allows the test to run in a NixOS test environment, where we use the system store. +# See doc/manual/src/contributing/testing.md / Running functional tests on NixOS. +clearStoreIfPossible() { + if isTestOnNixOS; then + echo "clearStoreIfPossible: Not clearing store, because we're on NixOS. Moving on." + else + doClearStore + fi +} + clearStore() { + if isTestOnNixOS; then + die "clearStore: not supported when testing on NixOS. If not essential, call clearStoreIfPossible. If really needed, add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + doClearStore +} + +doClearStore() { echo "clearing store..." chmod -R +w "$NIX_STORE_DIR" rm -rf "$NIX_STORE_DIR" @@ -84,6 +113,10 @@ clearCacheCache() { } startDaemon() { + if isTestOnNixOS; then + die "startDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + # Don’t start the daemon twice, as this would just make it loop indefinitely if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then return @@ -110,6 +143,10 @@ startDaemon() { } killDaemon() { + if isTestOnNixOS; then + die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + # Don’t fail trying to stop a non-existant daemon twice if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return @@ -130,6 +167,10 @@ killDaemon() { } restartDaemon() { + if isTestOnNixOS; then + die "restartDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + [[ -z "${_NIX_TEST_DAEMON_PID:-}" ]] && return 0 killDaemon @@ -152,6 +193,12 @@ skipTest () { exit 99 } +TODO_NixOS() { + if isTestOnNixOS; then + skipTest "This test has not been adapted for NixOS yet" + fi +} + requireDaemonNewerThan () { isDaemonNewer "$1" || skipTest "Daemon is too old" } @@ -169,7 +216,7 @@ requireGit() { } fail() { - echo "$1" >&2 + echo "test failed: $*" >&2 exit 1 } @@ -234,7 +281,7 @@ buggyNeedLocalStore() { enableFeatures() { local features="$1" - sed -i 's/experimental-features .*/& '"$features"'/' "$NIX_CONF_DIR"/nix.conf + sed -i 's/experimental-features .*/& '"$features"'/' "$test_nix_conf_dir"/nix.conf } set -x diff --git a/tests/functional/compression-levels.sh b/tests/functional/compression-levels.sh index 34f66c531..399265f9c 100755 --- a/tests/functional/compression-levels.sh +++ b/tests/functional/compression-levels.sh @@ -2,23 +2,23 @@ source common.sh -clearStore +clearStoreIfPossible clearCache outPath=$(nix-build dependencies.nix --no-out-link) cacheURI="file://$cacheDir?compression=xz&compression-level=0" -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -FILESIZES=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') +FILESIZES=$(cat "${cacheDir}"/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') clearCache cacheURI="file://$cacheDir?compression=xz&compression-level=5" -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -FILESIZES2=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') +FILESIZES2=$(cat "${cacheDir}"/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') [[ $FILESIZES -gt $FILESIZES2 ]] diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 1811b755c..50858eaa4 100755 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -28,6 +28,8 @@ nix registry remove userhome-with-xdg # Assert the .config folder hasn't been created. [ ! -e "$HOME/.config" ] +TODO_NixOS # Very specific test setup not compatible with the NixOS test environment? + # Test that files are loaded from XDG by default export XDG_CONFIG_HOME=$TEST_ROOT/confighome export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2 diff --git a/tests/functional/db-migration.sh b/tests/functional/db-migration.sh index a6a5c7744..6feabb90d 100755 --- a/tests/functional/db-migration.sh +++ b/tests/functional/db-migration.sh @@ -10,6 +10,8 @@ if [[ -z "${NIX_DAEMON_PACKAGE-}" ]]; then skipTest "not using the Nix daemon" fi +TODO_NixOS + killDaemon # Fill the db using the older Nix diff --git a/tests/functional/debugger.sh b/tests/functional/debugger.sh index 47e644bb7..b96b7e5d3 100755 --- a/tests/functional/debugger.sh +++ b/tests/functional/debugger.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible # regression #9932 echo ":env" | expect 1 nix eval --debugger --expr '(_: throw "oh snap") 42' diff --git a/tests/functional/dependencies.sh b/tests/functional/dependencies.sh index 5922a1f98..972bc5a9b 100755 --- a/tests/functional/dependencies.sh +++ b/tests/functional/dependencies.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible drvPath=$(nix-instantiate dependencies.nix) @@ -33,7 +33,7 @@ nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' echo "output path is $outPath" -text=$(cat "$outPath"/foobar) +text=$(cat "$outPath/foobar") if test "$text" != "FOOBAR"; then exit 1; fi deps=$(nix-store -quR "$drvPath") @@ -65,6 +65,8 @@ drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay) # now --valid-derivers returns both test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")" +TODO_NixOS # The following --delete fails, because it seems to be still alive. This might be caused by a different test using the same path. We should try make the derivations unique, e.g. naming after tests, and adding a timestamp that's constant for that test script run. + # check that nix-store --valid-derivers only returns existing drv nix-store --delete "$drvPath" test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2" diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh new file mode 100755 index 000000000..6c0c76b4c --- /dev/null +++ b/tests/functional/derivation-advanced-attributes.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source common/test-root.sh +source common/paths.sh + +set -o pipefail + +source characterisation/framework.sh + +badDiff=0 +badExitCode=0 + +store="$TEST_ROOT/store" + +for nixFile in derivation/*.nix; do + drvPath=$(nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + testName=$(basename "$nixFile" .nix) + got="${store}${drvPath}" + expected="derivation/$testName.drv" + diffAndAcceptInner "$testName" "$got" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/derivation-json.sh b/tests/functional/derivation-json.sh index 59c77e6c5..06f934cfe 100755 --- a/tests/functional/derivation-json.sh +++ b/tests/functional/derivation-json.sh @@ -4,11 +4,11 @@ source common.sh drvPath=$(nix-instantiate simple.nix) -nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json +nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/simple.json -drvPath2=$(nix derivation add < $TEST_HOME/simple.json) +drvPath2=$(nix derivation add < "$TEST_HOME"/simple.json) [[ "$drvPath" = "$drvPath2" ]] # Input addressed derivations cannot be renamed. -jq '.name = "foo"' < $TEST_HOME/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" +jq '.name = "foo"' < "$TEST_HOME"/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" diff --git a/tests/functional/derivation/advanced-attributes-defaults.drv b/tests/functional/derivation/advanced-attributes-defaults.drv new file mode 100644 index 000000000..391c6ab80 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("builder","/bin/bash"),("name","advanced-attributes-defaults"),("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-defaults.nix b/tests/functional/derivation/advanced-attributes-defaults.nix new file mode 100644 index 000000000..51a8d0e7e --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.nix @@ -0,0 +1,6 @@ +derivation { + name = "advanced-attributes-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 100644 index 000000000..9dd402057 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +Derive([("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev","",""),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}"),("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev"),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix new file mode 100644 index 000000000..0c13a7691 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix @@ -0,0 +1,8 @@ +derivation { + name = "advanced-attributes-structured-attrs-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + outputs = [ "out" "dev" ]; + __structuredAttrs = true; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/advanced-attributes-structured-attrs.drv new file mode 100644 index 000000000..e47a41ad5 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +Derive([("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev"),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix new file mode 100644 index 000000000..0044b65fd --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -0,0 +1,45 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes-structured-attrs"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + outputs = [ "out" "bin" "dev" ]; + __structuredAttrs = true; + outputChecks = { + out = { + allowedReferences = [foo]; + allowedRequisites = [foo]; + }; + bin = { + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + }; + dev = { + maxSize = 789; + maxClosureSize = 5909; + }; + }; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/functional/derivation/advanced-attributes.drv b/tests/functional/derivation/advanced-attributes.drv new file mode 100644 index 000000000..ec3112ab2 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("allowedRequisites","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("builder","/bin/bash"),("disallowedReferences","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("disallowedRequisites","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix new file mode 100644 index 000000000..ff680c567 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.nix @@ -0,0 +1,33 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + allowedReferences = [foo]; + allowedRequisites = [foo]; + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/functional/dump-db.sh b/tests/functional/dump-db.sh index 2d0460275..14181b4b6 100755 --- a/tests/functional/dump-db.sh +++ b/tests/functional/dump-db.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + needLocalStore "--dump-db requires a local store" clearStore diff --git a/tests/functional/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh index c786f6925..0d95881b6 100644 --- a/tests/functional/dyn-drv/common.sh +++ b/tests/functional/dyn-drv/common.sh @@ -5,4 +5,6 @@ requireDaemonNewerThan "2.16.0pre20230419" enableFeatures "ca-derivations dynamic-derivations" +TODO_NixOS + restartDaemon diff --git a/tests/functional/dyn-drv/text-hashed-output.sh b/tests/functional/dyn-drv/text-hashed-output.sh index f3e5aa93b..2cc877219 100644 --- a/tests/functional/dyn-drv/text-hashed-output.sh +++ b/tests/functional/dyn-drv/text-hashed-output.sh @@ -20,7 +20,7 @@ nix show-derivation "$drvProducingDrv" out1=$(nix-build ./text-hashed-output.nix -A producingDrv --no-out-link) -nix path-info $drv --derivation --json | jq -nix path-info $out1 --derivation --json | jq +nix path-info "$drv" --derivation --json | jq +nix path-info "$out1" --derivation --json | jq -test $out1 == $drv +test "$out1" == "$drv" diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh index 0ab608acc..202e7b004 100755 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + # Using `--eval-store` with the daemon will eventually copy everything # to the build store, invalidating most of the tests here needLocalStore "“--eval-store” doesn't achieve much with the daemon" diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index acd6e2915..27cdce478 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible testStdinHeredoc=$(nix eval -f - <$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -[[ $(cat $TEST_ROOT/stdout) = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr -grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr +' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +[[ $(cat "$TEST_ROOT/stdout") = '' ]] +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" # 'flakes' experimental-feature is disabled after, ignore and warn NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command -' expect 1 nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -[[ $(cat $TEST_ROOT/stdout) = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr -grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr +' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +[[ $(cat "$TEST_ROOT/stdout") = '' ]] +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" # 'flakes' experimental-feature is enabled before, process NIX_CONFIG=' experimental-features = nix-command flakes accept-flake-config = true -' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -grepQuiet "true" $TEST_ROOT/stdout -grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr +' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +grepQuiet "true" "$TEST_ROOT/stdout" +grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" # 'flakes' experimental-feature is enabled after, process NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command flakes -' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -grepQuiet "true" $TEST_ROOT/stdout -grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr +' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +grepQuiet "true" "$TEST_ROOT/stdout" +grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" function exit_code_both_ways { expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null diff --git a/tests/functional/export-graph.sh b/tests/functional/export-graph.sh index 281b5b05b..b507b6d3a 100755 --- a/tests/functional/export-graph.sh +++ b/tests/functional/export-graph.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearProfiles diff --git a/tests/functional/export.sh b/tests/functional/export.sh index fce2e333a..3e895a540 100755 --- a/tests/functional/export.sh +++ b/tests/functional/export.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore outPath=$(nix-build dependencies.nix --no-out-link) diff --git a/tests/functional/fetchClosure.sh b/tests/functional/fetchClosure.sh index 4026b0790..7ef635d36 100755 --- a/tests/functional/fetchClosure.sh +++ b/tests/functional/fetchClosure.sh @@ -4,6 +4,8 @@ source common.sh enableFeatures "fetch-closure" +TODO_NixOS + clearStore clearCacheCache diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index d89d59a81..78925b5cd 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -4,7 +4,7 @@ source common.sh requireGit -clearStore +clearStoreIfPossible # Intentionally not in a canonical form # See https://github.com/NixOS/nix/issues/6195 diff --git a/tests/functional/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh index b17cc2090..ee054fabc 100755 --- a/tests/functional/fetchGitRefs.sh +++ b/tests/functional/fetchGitRefs.sh @@ -4,7 +4,7 @@ source common.sh requireGit -clearStore +clearStoreIfPossible repo="$TEST_ROOT/git" @@ -14,7 +14,7 @@ git init "$repo" git -C "$repo" config user.email "foobar@example.com" git -C "$repo" config user.name "Foobar" -echo utrecht > "$repo"/hello +echo utrecht > "$repo/hello" git -C "$repo" add hello git -C "$repo" commit -m 'Bla1' diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index 7c5f7b0d6..4a3e4c347 100755 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -6,7 +6,7 @@ set -u requireGit -clearStore +clearStoreIfPossible rootRepo=$TEST_ROOT/gitSubmodulesRoot subRepo=$TEST_ROOT/gitSubmodulesSub diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh index 27f9a8cf5..4012d8229 100755 --- a/tests/functional/fetchGitVerification.sh +++ b/tests/functional/fetchGitVerification.sh @@ -7,7 +7,7 @@ requireGit enableFeatures "verified-fetches" -clearStore +clearStoreIfPossible repo="$TEST_ROOT/git" diff --git a/tests/functional/fetchMercurial.sh b/tests/functional/fetchMercurial.sh index f3774fc74..6de192865 100755 --- a/tests/functional/fetchMercurial.sh +++ b/tests/functional/fetchMercurial.sh @@ -4,6 +4,8 @@ source common.sh [[ $(type -p hg) ]] || skipTest "Mercurial not installed" +TODO_NixOS + clearStore # Intentionally not in a canonical form diff --git a/tests/functional/fetchPath.sh b/tests/functional/fetchPath.sh index e466e4494..560a270c1 100755 --- a/tests/functional/fetchPath.sh +++ b/tests/functional/fetchPath.sh @@ -2,7 +2,7 @@ source common.sh -touch $TEST_ROOT/foo -t 202211111111 +touch "$TEST_ROOT/foo" -t 202211111111 # We only check whether 2022-11-1* **:**:** is the last modified date since # `lastModified` is transformed into UTC in `builtins.fetchTarball`. [[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]] diff --git a/tests/functional/fetchTree-file.sh b/tests/functional/fetchTree-file.sh index 9c9532876..66be928c7 100755 --- a/tests/functional/fetchTree-file.sh +++ b/tests/functional/fetchTree-file.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore cd "$TEST_ROOT" @@ -90,7 +92,7 @@ EOF EOF # Test tarball URLs on the command line. - [[ $(nix flake metadata --json file://$PWD/test_input_no_ext | jq -r .resolved.type) = tarball ]] + [[ $(nix flake metadata --json "file://$PWD/test_input_no_ext" | jq -r .resolved.type) = tarball ]] popd diff --git a/tests/functional/fetchurl.sh b/tests/functional/fetchurl.sh index 2255dbbdc..5af44fcf2 100755 --- a/tests/functional/fetchurl.sh +++ b/tests/functional/fetchurl.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore # Test fetching a flat file. diff --git a/tests/functional/filter-source.sh b/tests/functional/filter-source.sh index c5e10be93..b32f5b59d 100755 --- a/tests/functional/filter-source.sh +++ b/tests/functional/filter-source.sh @@ -2,26 +2,26 @@ source common.sh -rm -rf $TEST_ROOT/filterin -mkdir $TEST_ROOT/filterin -mkdir $TEST_ROOT/filterin/foo -touch $TEST_ROOT/filterin/foo/bar -touch $TEST_ROOT/filterin/xyzzy -touch $TEST_ROOT/filterin/b -touch $TEST_ROOT/filterin/bak -touch $TEST_ROOT/filterin/bla.c.bak -ln -s xyzzy $TEST_ROOT/filterin/link +rm -rf "$TEST_ROOT/filterin" +mkdir "$TEST_ROOT/filterin" +mkdir "$TEST_ROOT/filterin/foo" +touch "$TEST_ROOT/filterin/foo/bar" +touch "$TEST_ROOT/filterin/xyzzy" +touch "$TEST_ROOT/filterin/b" +touch "$TEST_ROOT/filterin/bak" +touch "$TEST_ROOT"/filterin/bla.c.bak +ln -s xyzzy "$TEST_ROOT/filterin/link" checkFilter() { - test ! -e $1/foo/bar - test -e $1/xyzzy - test -e $1/bak - test ! -e $1/bla.c.bak - test ! -L $1/link + test ! -e "$1/foo/bar" + test -e "$1/xyzzy" + test -e "$1/bak" + test ! -e "$1"/bla.c.bak + test ! -L "$1/link" } -nix-build ./filter-source.nix -o $TEST_ROOT/filterout1 -checkFilter $TEST_ROOT/filterout1 +nix-build ./filter-source.nix -o "$TEST_ROOT/filterout1" +checkFilter "$TEST_ROOT/filterout1" -nix-build ./path.nix -o $TEST_ROOT/filterout2 -checkFilter $TEST_ROOT/filterout2 +nix-build ./path.nix -o "$TEST_ROOT/filterout2" +checkFilter "$TEST_ROOT/filterout2" diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index 4d07d00cd..d98769e64 100755 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0)) diff --git a/tests/functional/flakes/absolute-attr-paths.sh b/tests/functional/flakes/absolute-attr-paths.sh index 8ed1755c4..b0e6225d8 100755 --- a/tests/functional/flakes/absolute-attr-paths.sh +++ b/tests/functional/flakes/absolute-attr-paths.sh @@ -4,8 +4,8 @@ source ./common.sh flake1Dir=$TEST_ROOT/flake1 -mkdir -p $flake1Dir -cat > $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/flake.nix < $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/flake.nix < $flake1Dir/foo +echo bar > "$flake1Dir/foo" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a1" [[ -e $TEST_ROOT/result/simple.nix ]] -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a2 -[[ $(cat $TEST_ROOT/result) = bar ]] +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a2" +[[ $(cat "$TEST_ROOT/result") = bar ]] -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a3 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a3" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a4 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a4" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a6" [[ -e $TEST_ROOT/result/simple.nix ]] -nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8 -diff common.sh $TEST_ROOT/result +nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a8" +diff common.sh "$TEST_ROOT/result" -expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9 \ +expectStderr 1 nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a9" \ | grepQuiet "has 0 entries in its context. It should only have exactly one entry" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a10 -[[ $(readlink -e $TEST_ROOT/result) = *simple.drv ]] +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir"#a10 +[[ $(readlink -e "$TEST_ROOT/result") = *simple.drv ]] -expectStderr 1 nix build --json --out-link $TEST_ROOT/result $flake1Dir#a11 \ +expectStderr 1 nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a11" \ | grepQuiet "has a context which refers to a complete source and binary closure" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a12 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a12" [[ -e $TEST_ROOT/result/hello ]] -expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a13 \ +expectStderr 1 nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a13" \ | grepQuiet "has 2 entries in its context. It should only have exactly one entry" # Test accessing output in installables with `.` (foobarbaz.) -nix build --json --no-link $flake1Dir#a14.foo | jq --exit-status ' +nix build --json --no-link "$flake1Dir"#a14.foo | jq --exit-status ' (.[0] | (.drvPath | match(".*dot-installable.drv")) and (.outputs | keys == ["foo"])) diff --git a/tests/functional/flakes/bundle.sh b/tests/functional/flakes/bundle.sh index 711691e0b..5e185cbf6 100755 --- a/tests/functional/flakes/bundle.sh +++ b/tests/functional/flakes/bundle.sh @@ -2,9 +2,9 @@ source common.sh -cp ../simple.nix ../simple.builder.sh ../config.nix $TEST_HOME +cp ../simple.nix ../simple.builder.sh ../config.nix "$TEST_HOME" -cd $TEST_HOME +cd "$TEST_HOME" cat < flake.nix { @@ -27,8 +27,8 @@ EOF nix build .# nix bundle --bundler .# .# -nix bundle --bundler .#bundlers.$system.default .#packages.$system.default -nix bundle --bundler .#bundlers.$system.simple .#packages.$system.default +nix bundle --bundler .#bundlers."$system".default .#packages."$system".default +nix bundle --bundler .#bundlers."$system".simple .#packages."$system".default -nix bundle --bundler .#bundlers.$system.default .#apps.$system.default -nix bundle --bundler .#bundlers.$system.simple .#apps.$system.default +nix bundle --bundler .#bundlers."$system".default .#apps."$system".default +nix bundle --bundler .#bundlers."$system".simple .#apps."$system".default diff --git a/tests/functional/flakes/circular.sh b/tests/functional/flakes/circular.sh index 6cab3a72b..5304496ba 100755 --- a/tests/functional/flakes/circular.sh +++ b/tests/functional/flakes/circular.sh @@ -8,10 +8,10 @@ requireGit flakeA=$TEST_ROOT/flakeA flakeB=$TEST_ROOT/flakeB -createGitRepo $flakeA -createGitRepo $flakeB +createGitRepo "$flakeA" +createGitRepo "$flakeB" -cat > $flakeA/flake.nix < "$flakeA"/flake.nix < $flakeA/flake.nix < $flakeB/flake.nix < "$flakeB"/flake.nix < $flakeB/flake.nix <"$flake1Dir/flake.nix" < \$out + ''; + }; + ifd = assert (import self.drv); self.drv; }; } EOF @@ -22,3 +32,8 @@ git -C "$flake1Dir" commit -m "Init" expect 1 nix build "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' expect 1 nix build "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' + +# Conditional error should not be cached +expect 1 nix build "$flake1Dir#ifd" --option allow-import-from-derivation false 2>&1 \ + | grepQuiet 'error: cannot build .* during evaluation because the option '\''allow-import-from-derivation'\'' is disabled' +nix build --no-link "$flake1Dir#ifd" diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 2988352a9..817f77783 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -15,6 +15,8 @@ source common.sh requireGit +TODO_NixOS + clearStore # Submodules can't be fetched locally by default. @@ -27,8 +29,8 @@ rootRepo=$TEST_ROOT/rootRepo subRepo=$TEST_ROOT/submodule -createGitRepo $subRepo -cat > $subRepo/flake.nix < "$subRepo"/flake.nix < $subRepo/flake.nix < $subRepo/sub.nix -git -C $subRepo add flake.nix sub.nix -git -C $subRepo commit -m Initial +echo '"expression in submodule"' > "$subRepo"/sub.nix +git -C "$subRepo" add flake.nix sub.nix +git -C "$subRepo" commit -m Initial -createGitRepo $rootRepo +createGitRepo "$rootRepo" -git -C $rootRepo submodule init -git -C $rootRepo submodule add $subRepo submodule -echo '"expression in root repo"' > $rootRepo/root.nix -git -C $rootRepo add root.nix -git -C $rootRepo commit -m "Add root.nix" +git -C "$rootRepo" submodule init +git -C "$rootRepo" submodule add "$subRepo" submodule +echo '"expression in root repo"' > "$rootRepo"/root.nix +git -C "$rootRepo" add root.nix +git -C "$rootRepo" commit -m "Add root.nix" flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule # Flake can live inside a submodule and can be accessed via ?dir=submodule -[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]] +[[ $(nix eval --json "$flakeref#sub" ) = '"expression in submodule"' ]] # The flake can access content outside of the submodule -[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]] +[[ $(nix eval --json "$flakeref#root" ) = '"expression in root repo"' ]] # Check that dirtying a submodule makes the entire thing dirty. -[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]] -echo '"foo"' > $rootRepo/submodule/sub.nix -[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]] -[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]] +[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]] +echo '"foo"' > "$rootRepo"/submodule/sub.nix +[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] +[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 3057c0293..26b91eda7 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -2,6 +2,8 @@ source ./common.sh +TODO_NixOS + requireGit clearStore @@ -17,13 +19,17 @@ flake7Dir=$TEST_ROOT/flake7 nonFlakeDir=$TEST_ROOT/nonFlake badFlakeDir=$TEST_ROOT/badFlake flakeGitBare=$TEST_ROOT/flakeGitBare +lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake -for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir"; do +for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir" "$lockfileSummaryFlake"; do # Give one repo a non-main initial branch. extraArgs= if [[ "$repo" == "$flake2Dir" ]]; then extraArgs="--initial-branch=main" fi + if [[ "$repo" == "$lockfileSummaryFlake" ]]; then + extraArgs="--initial-branch=main" + fi createGitRepo "$repo" "$extraArgs" done @@ -189,6 +195,7 @@ json=$(nix flake metadata flake1 --json | jq .) [[ -d $(echo "$json" | jq -r .path) ]] [[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) +[[ -n $(echo "$json" | jq -r .fingerprint) ]] echo foo > "$flake1Dir/foo" git -C "$flake1Dir" add $flake1Dir/foo @@ -642,3 +649,37 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock [[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] [[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' + +# Test that the --commit-lock-file-summary flag and its alias work +cat > "$lockfileSummaryFlake/flake.nix" < $templatesDir/flake.nix < "$templatesDir"/flake.nix < $templatesDir/flake.nix < $templatesDir/trivial/flake.nix < "$templatesDir"/trivial/flake.nix < $templatesDir/trivial/flake.nix < $templatesDir/trivial/a -echo b > $templatesDir/trivial/b +echo a > "$templatesDir/trivial/a" +echo b > "$templatesDir/trivial/b" -git -C $templatesDir add flake.nix trivial/ -git -C $templatesDir commit -m 'Initial' +git -C "$templatesDir" add flake.nix trivial/ +git -C "$templatesDir" commit -m 'Initial' nix flake check templates nix flake show templates nix flake show templates --json | jq -createGitRepo $flakeDir -(cd $flakeDir && nix flake init) -(cd $flakeDir && nix flake init) # check idempotence -git -C $flakeDir add flake.nix -nix flake check $flakeDir -nix flake show $flakeDir -nix flake show $flakeDir --json | jq -git -C $flakeDir commit -a -m 'Initial' +createGitRepo "$flakeDir" +(cd "$flakeDir" && nix flake init) +(cd "$flakeDir" && nix flake init) # check idempotence +git -C "$flakeDir" add flake.nix +nix flake check "$flakeDir" +nix flake show "$flakeDir" +nix flake show "$flakeDir" --json | jq +git -C "$flakeDir" commit -a -m 'Initial' # Test 'nix flake init' with benign conflicts createGitRepo "$flakeDir" -echo a > $flakeDir/a -(cd $flakeDir && nix flake init) # check idempotence +echo a > "$flakeDir/a" +(cd "$flakeDir" && nix flake init) # check idempotence # Test 'nix flake init' with conflicts createGitRepo "$flakeDir" -echo b > $flakeDir/a -pushd $flakeDir +echo b > "$flakeDir/a" +pushd "$flakeDir" (! nix flake init) |& grep "refusing to overwrite existing file '$flakeDir/a'" popd -git -C $flakeDir commit -a -m 'Changed' +git -C "$flakeDir" commit -a -m 'Changed' # Test 'nix flake new'. -rm -rf $flakeDir -nix flake new -t templates#trivial $flakeDir -nix flake new -t templates#trivial $flakeDir # check idempotence -nix flake check $flakeDir +rm -rf "$flakeDir" +nix flake new -t templates#trivial "$flakeDir" +nix flake new -t templates#trivial "$flakeDir" # check idempotence +nix flake check "$flakeDir" diff --git a/tests/functional/flakes/inputs.sh b/tests/functional/flakes/inputs.sh index 0327a3e9e..bc0603f1b 100755 --- a/tests/functional/flakes/inputs.sh +++ b/tests/functional/flakes/inputs.sh @@ -8,12 +8,12 @@ requireGit test_subdir_self_path() { baseDir=$TEST_ROOT/$RANDOM flakeDir=$baseDir/b-low - mkdir -p $flakeDir - writeSimpleFlake $baseDir - writeSimpleFlake $flakeDir + mkdir -p "$flakeDir" + writeSimpleFlake "$baseDir" + writeSimpleFlake "$flakeDir" - echo all good > $flakeDir/message - cat > $flakeDir/flake.nix < "$flakeDir/message" + cat > "$flakeDir"/flake.nix < $flakeDir/message - cat > $flakeDir/flake.nix < "$flakeDir/message" + cat > "$flakeDir"/flake.nix < $clientDir/flake.nix < "$clientDir"/flake.nix < foo/flake.nix @@ -27,11 +27,11 @@ success=("" . .# .#test ../subdir ../subdir#test "$PWD") failure=("path:$PWD" "../simple.nix") for i in "${success[@]}"; do - nix build $i || fail "flake should be found by searching up directories" + nix build "$i" || fail "flake should be found by searching up directories" done for i in "${failure[@]}"; do - ! nix build $i || fail "flake should not search up directories when using 'path:'" + ! nix build "$i" || fail "flake should not search up directories when using 'path:'" done popd @@ -45,7 +45,7 @@ if [[ -n $(type -p git) ]]; then pushd subdir git init for i in "${success[@]}" "${failure[@]}"; do - ! nix build $i || fail "flake should not search past a git repository" + ! nix build "$i" || fail "flake should not search past a git repository" done rm -rf .git popd diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index 680a1505c..a17a0c2af 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -7,26 +7,26 @@ requireGit flake1Dir=$TEST_ROOT/flake1 flake2Dir=$TEST_ROOT/flake2 -createGitRepo $flake1Dir -cat > $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/x.nix -git -C $flake1Dir add flake.nix x.nix -git -C $flake1Dir commit -m Initial +echo 123 > "$flake1Dir"/x.nix +git -C "$flake1Dir" add flake.nix x.nix +git -C "$flake1Dir" commit -m Initial -createGitRepo $flake2Dir -cat > $flake2Dir/flake.nix < "$flake2Dir"/flake.nix < $flake1Dir/x.nix +echo 456 > "$flake1Dir"/x.nix -[[ $(nix eval --json $flake2Dir#x --override-input flake1 $TEST_ROOT/flake1) = 456 ]] +[[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] diff --git a/tests/functional/fmt.sh b/tests/functional/fmt.sh index 8fc9a3979..b29fe64d6 100755 --- a/tests/functional/fmt.sh +++ b/tests/functional/fmt.sh @@ -2,7 +2,9 @@ source common.sh -clearStore +TODO_NixOS # Provide a `shell` variable. Try not to `export` it, perhaps. + +clearStoreIfPossible rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh ./config.nix $TEST_HOME @@ -31,5 +33,3 @@ EOF nix fmt ./file ./folder | grep 'Formatting: ./file ./folder' nix flake check nix flake show | grep -P "package 'formatter'" - -clearStore diff --git a/tests/functional/function-trace.sh b/tests/functional/function-trace.sh index 71f18b67f..7524afdf2 100755 --- a/tests/functional/function-trace.sh +++ b/tests/functional/function-trace.sh @@ -21,12 +21,12 @@ expect_trace() { <(echo "$expect") \ <(echo "$actual") ) && result=0 || result=$? - if [ $result -eq 0 ]; then + if [ "$result" -eq 0 ]; then echo " ok." else echo " failed. difference:" echo "$msg" - return $result + return "$result" fi } diff --git a/tests/functional/gc-auto.sh b/tests/functional/gc-auto.sh index 98bb7e60e..8f25be3e9 100755 --- a/tests/functional/gc-auto.sh +++ b/tests/functional/gc-auto.sh @@ -4,6 +4,8 @@ source common.sh needLocalStore "“min-free” and “max-free” are daemon options" +TODO_NixOS + clearStore garbage1=$(nix store add-path --name garbage1 ./nar-access.sh) diff --git a/tests/functional/gc-concurrent.sh b/tests/functional/gc-concurrent.sh index 67ea3dc74..df180b14f 100755 --- a/tests/functional/gc-concurrent.sh +++ b/tests/functional/gc-concurrent.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore lockFifo1=$TEST_ROOT/test1.fifo @@ -20,8 +22,8 @@ outPath3=$(nix-store -r $drvPath3) touch $outPath3.lock rm -f "$NIX_STATE_DIR"/gcroots/foo* -ln -s $drvPath2 "$NIX_STATE_DIR"/gcroots/foo -ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2 +ln -s $drvPath2 "$NIX_STATE_DIR/gcroots/foo" +ln -s $outPath3 "$NIX_STATE_DIR/gcroots/foo2" # Start build #1 in the background. It starts immediately. nix-store -rvv "$drvPath1" & diff --git a/tests/functional/gc-non-blocking.sh b/tests/functional/gc-non-blocking.sh index ecfa421fb..de10837eb 100755 --- a/tests/functional/gc-non-blocking.sh +++ b/tests/functional/gc-non-blocking.sh @@ -4,6 +4,8 @@ # parallel with it. source common.sh +TODO_NixOS + needLocalStore "the GC test needs a synchronisation point" clearStore diff --git a/tests/functional/gc-runtime.sh b/tests/functional/gc-runtime.sh index 2ee72b61e..0cccaaf16 100755 --- a/tests/functional/gc-runtime.sh +++ b/tests/functional/gc-runtime.sh @@ -11,28 +11,30 @@ esac set -m # enable job control, needed for kill +TODO_NixOS + profiles="$NIX_STATE_DIR"/profiles -rm -rf $profiles +rm -rf "$profiles" -nix-env -p $profiles/test -f ./gc-runtime.nix -i gc-runtime +nix-env -p "$profiles/test" -f ./gc-runtime.nix -i gc-runtime -outPath=$(nix-env -p $profiles/test -q --no-name --out-path gc-runtime) -echo $outPath +outPath=$(nix-env -p "$profiles/test" -q --no-name --out-path gc-runtime) +echo "$outPath" echo "backgrounding program..." -$profiles/test/program & +"$profiles"/test/program & sleep 2 # hack - wait for the program to get started child=$! echo PID=$child -nix-env -p $profiles/test -e gc-runtime -nix-env -p $profiles/test --delete-generations old +nix-env -p "$profiles/test" -e gc-runtime +nix-env -p "$profiles/test" --delete-generations old nix-store --gc kill -- -$child -if ! test -e $outPath; then +if ! test -e "$outPath"; then echo "running program was garbage collected!" exit 1 fi diff --git a/tests/functional/gc.sh b/tests/functional/gc.sh index 1f216ebc7..7707a7e27 100755 --- a/tests/functional/gc.sh +++ b/tests/functional/gc.sh @@ -2,14 +2,16 @@ source common.sh +TODO_NixOS + clearStore drvPath=$(nix-instantiate dependencies.nix) outPath=$(nix-store -rvv "$drvPath") # Set a GC root. -rm -f "$NIX_STATE_DIR"/gcroots/foo -ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo +rm -f "$NIX_STATE_DIR/gcroots/foo" +ln -sf $outPath "$NIX_STATE_DIR/gcroots/foo" [ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] @@ -42,7 +44,7 @@ cat $outPath/reference-to-input-2/bar # Check that the derivation has been GC'd. if test -e $drvPath; then false; fi -rm "$NIX_STATE_DIR"/gcroots/foo +rm "$NIX_STATE_DIR/gcroots/foo" nix-collect-garbage diff --git a/tests/functional/git-hashing/common.sh b/tests/functional/git-hashing/common.sh index 572cea438..29c518fea 100644 --- a/tests/functional/git-hashing/common.sh +++ b/tests/functional/git-hashing/common.sh @@ -1,5 +1,7 @@ source ../common.sh +TODO_NixOS # Need to enable git hashing feature and make sure test is ok for store we don't clear + clearStore clearCache diff --git a/tests/functional/hash-path.sh b/tests/functional/hash-path.sh index 12605ef71..86d782a95 100755 --- a/tests/functional/hash-path.sh +++ b/tests/functional/hash-path.sh @@ -3,7 +3,7 @@ source common.sh try () { - printf "%s" "$2" > $TEST_ROOT/vector + printf "%s" "$2" > "$TEST_ROOT/vector" hash="$(nix-hash --flat ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")" if ! (( "${NO_TEST_CLASSIC-}" )) && test "$hash" != "$3"; then echo "try nix-hash: hash $1, expected $3, got $hash" @@ -61,7 +61,7 @@ NO_TEST_NIX_COMMAND=1 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e NO_TEST_CLASSIC=1 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" try2 () { - hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) + hash=$(nix-hash --type "$1" "$TEST_ROOT/hash-path") if test "$hash" != "$2"; then echo "try nix-hash; hash $1, expected $2, got $hash" exit 1 @@ -73,22 +73,22 @@ try2 () { fi } -rm -rf $TEST_ROOT/hash-path -mkdir $TEST_ROOT/hash-path -echo "Hello World" > $TEST_ROOT/hash-path/hello +rm -rf "$TEST_ROOT/hash-path" +mkdir "$TEST_ROOT/hash-path" +echo "Hello World" > "$TEST_ROOT/hash-path/hello" try2 md5 "ea9b55537dd4c7e104515b2ccfaf4100" # Execute bit matters. -chmod +x $TEST_ROOT/hash-path/hello +chmod +x "$TEST_ROOT/hash-path/hello" try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" # Mtime and other bits don't. -touch -r . $TEST_ROOT/hash-path/hello -chmod 744 $TEST_ROOT/hash-path/hello +touch -r . "$TEST_ROOT/hash-path/hello" +chmod 744 "$TEST_ROOT/hash-path/hello" try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" # File type (e.g., symlink) does. -rm $TEST_ROOT/hash-path/hello -ln -s x $TEST_ROOT/hash-path/hello +rm "$TEST_ROOT/hash-path/hello" +ln -s x "$TEST_ROOT/hash-path/hello" try2 md5 "f78b733a68f5edbdf9413899339eaa4a" diff --git a/tests/functional/help.sh b/tests/functional/help.sh index 6436fb500..61efc1cb2 100755 --- a/tests/functional/help.sh +++ b/tests/functional/help.sh @@ -2,8 +2,6 @@ source common.sh -clearStore - # test help output nix-build --help diff --git a/tests/functional/import-derivation.sh b/tests/functional/import-derivation.sh index 53efa1f5d..68ddcfa4a 100755 --- a/tests/functional/import-derivation.sh +++ b/tests/functional/import-derivation.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible if nix-instantiate --readonly-mode ./import-derivation.nix; then echo "read-only evaluation of an imported derivation unexpectedly failed" @@ -11,4 +11,4 @@ fi outPath=$(nix-build ./import-derivation.nix --no-out-link) -[ "$(cat $outPath)" = FOO579 ] +[ "$(cat "$outPath")" = FOO579 ] diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index b59f73c77..5dea220fe 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -4,10 +4,12 @@ source common.sh requireDaemonNewerThan "2.8pre20220311" +TODO_NixOS + enableFeatures "ca-derivations impure-derivations" restartDaemon -clearStore +clearStoreIfPossible # Basic test of impure derivations: building one a second time should not use the previous result. printf 0 > $TEST_ROOT/counter diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh index 3c7df169e..ca32c1030 100755 --- a/tests/functional/impure-env.sh +++ b/tests/functional/impure-env.sh @@ -5,6 +5,8 @@ source common.sh # Needs the config option 'impure-env' to work requireDaemonNewerThan "2.19.0" +TODO_NixOS + enableFeatures "configurable-impure-env" restartDaemon @@ -20,13 +22,13 @@ startDaemon varTest env_name value --impure-env env_name=value -echo 'impure-env = set_in_config=config_value' >> "$NIX_CONF_DIR/nix.conf" +echo 'impure-env = set_in_config=config_value' >> "$test_nix_conf" set_in_config=daemon_value restartDaemon varTest set_in_config config_value varTest set_in_config client_value --impure-env set_in_config=client_value -sed -i -e '/^trusted-users =/d' "$NIX_CONF_DIR/nix.conf" +sed -i -e '/^trusted-users =/d' "$test_nix_conf" env_name=daemon_value restartDaemon diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 569e7082e..8cb8e98fb 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -4,7 +4,7 @@ source common.sh set -o pipefail -source lang/framework.sh +source characterisation/framework.sh # specialize function a bit function diffAndAccept() { @@ -138,32 +138,4 @@ for i in lang/eval-okay-*.nix; do fi done -if test -n "${_NIX_TEST_ACCEPT-}"; then - if (( "$badDiff" )); then - echo 'Output did mot match, but accepted output as the persisted expected output.' - echo 'That means the next time the tests are run, they should pass.' - else - echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' - echo 'indicating the unexpected output should be accepted as the expected output going forward,' - echo 'but no tests had unexpected output so there was no expected output to update.' - fi - if (( "$badExitCode" )); then - exit "$badExitCode" - else - skipTest "regenerating golden masters" - fi -else - if (( "$badDiff" )); then - echo '' - echo 'You can rerun this test with:' - echo '' - echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test' - echo '' - echo 'to regenerate the files containing the expected output,' - echo 'and then view the git diff to decide whether a change is' - echo 'good/intentional or bad/unintentional.' - echo 'If the diff contains arbitrary or impure information,' - echo 'please improve the normalization that the test applies to the output.' - fi - exit $(( "$badExitCode" + "$badDiff" )) -fi +characterisationTestExit diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp index 6f907106b..b262e814d 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp @@ -6,4 +6,4 @@ error: | ^ 10| - error: cannot coerce a set to a string: { a = { a = { a = { a = "ha"; b = "ha"; c = "ha"; d = "ha"; e = "ha"; f = "ha"; g = "ha"; h = "ha"; j = "ha"; }; «4294967295 attributes elided» }; «4294967294 attributes elided» }; «4294967293 attributes elided» } + error: cannot coerce a set to a string: { a = { a = { a = { a = "ha"; b = "ha"; c = "ha"; d = "ha"; e = "ha"; f = "ha"; g = "ha"; h = "ha"; j = "ha"; }; «8 attributes elided» }; «8 attributes elided» }; «8 attributes elided» } diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp new file mode 100644 index 000000000..eb2206df1 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the attribute 'outPath' + at :19:9: + 18| value = commonAttrs // { + 19| outPath = builtins.getAttr outputName strict; + | ^ + 20| drvPath = strict.drvPath; + + … while calling the 'getAttr' builtin + at :19:19: + 18| value = commonAttrs // { + 19| outPath = builtins.getAttr outputName strict; + | ^ + 20| drvPath = strict.drvPath; + + … while calling the 'derivationStrict' builtin + at :9:12: + 8| + 9| strict = derivationStrict drvAttrs; + | ^ + 10| + + … while evaluating derivation '~jiggle~' + whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:2:3 + + error: invalid derivation name: name '~jiggle~' contains illegal character '~'. Please pass a different 'name'. diff --git a/tests/functional/lang/eval-fail-derivation-name.nix b/tests/functional/lang/eval-fail-derivation-name.nix new file mode 100644 index 000000000..e779ad6ff --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.nix @@ -0,0 +1,5 @@ +derivation { + name = "~jiggle~"; + system = "some-system"; + builder = "/dontcare"; +} diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp new file mode 100644 index 000000000..30f8b6a35 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs-name.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/foo.tar.gz': name '~wobble~' contains illegal character '~'. Please change the value for the 'name' attribute passed to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix new file mode 100644 index 000000000..583805539 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp new file mode 100644 index 000000000..cef532e94 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/~wiggle~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please add a valid 'name' attribute to the argument for 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix new file mode 100644 index 000000000..068120edb --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/~wiggle~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp new file mode 100644 index 000000000..0950e8e70 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName.nix:1:1: + 1| builtins.fetchurl "https://example.com/~wiggle~" + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please pass an attribute set with 'url' and 'name' attributes to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.nix b/tests/functional/lang/eval-fail-fetchurl-baseName.nix new file mode 100644 index 000000000..965093843 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.nix @@ -0,0 +1 @@ +builtins.fetchurl "https://example.com/~wiggle~" diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp index 5d843d827..712dd75a8 100644 --- a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp @@ -29,7 +29,7 @@ error: | ^ 2| - (19997 duplicate frames omitted) + (197 duplicate frames omitted) error: stack overflow; max-call-depth exceeded at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14: diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags b/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags new file mode 100644 index 000000000..59e20ec9c --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags @@ -0,0 +1 @@ +--max-call-depth 100 \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-nested-list-items.err.exp b/tests/functional/lang/eval-fail-nested-list-items.err.exp new file mode 100644 index 000000000..90d439061 --- /dev/null +++ b/tests/functional/lang/eval-fail-nested-list-items.err.exp @@ -0,0 +1,9 @@ +error: + … while evaluating a path segment + at /pwd/lang/eval-fail-nested-list-items.nix:11:6: + 10| + 11| "" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v) + | ^ + 12| + + error: cannot coerce a list to a string: [ [ 1 2 3 4 5 6 7 8 ] [ 1 «3 items elided» ] ] diff --git a/tests/functional/lang/eval-fail-nested-list-items.nix b/tests/functional/lang/eval-fail-nested-list-items.nix new file mode 100644 index 000000000..af45b1dd4 --- /dev/null +++ b/tests/functional/lang/eval-fail-nested-list-items.nix @@ -0,0 +1,11 @@ +# This reproduces https://github.com/NixOS/nix/issues/10993, for lists +# $ nix run nix/2.23.1 -- eval --expr '"" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v)' +# error: +# … while evaluating a path segment +# at «string»:1:6: +# 1| "" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v) +# | ^ +# +# error: cannot coerce a list to a string: [ [ 1 2 3 4 5 6 7 8 ] [ 1 «4294967290 items elided» ] ] + +"" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v) diff --git a/tests/functional/lang/eval-okay-derivation-legacy.err.exp b/tests/functional/lang/eval-okay-derivation-legacy.err.exp new file mode 100644 index 000000000..94f0854dd --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.err.exp @@ -0,0 +1,6 @@ +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead diff --git a/tests/functional/lang/eval-okay-derivation-legacy.exp b/tests/functional/lang/eval-okay-derivation-legacy.exp new file mode 100644 index 000000000..4f374a1aa --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.exp @@ -0,0 +1 @@ +"/nix/store/mzgwvrjjir216ra58mwwizi8wj6y9ddr-eval-okay-derivation-legacy" diff --git a/tests/functional/lang/eval-okay-derivation-legacy.nix b/tests/functional/lang/eval-okay-derivation-legacy.nix new file mode 100644 index 000000000..b529cdf90 --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.nix @@ -0,0 +1,12 @@ +(builtins.derivationStrict { + name = "eval-okay-derivation-legacy"; + system = "x86_64-linux"; + builder = "/dontcare"; + __structuredAttrs = true; + allowedReferences = [ ]; + disallowedReferences = [ ]; + allowedRequisites = [ ]; + disallowedRequisites = [ ]; + maxSize = 1234; + maxClosureSize = 12345; +}).out diff --git a/tests/functional/lang/framework.sh b/tests/functional/lang/framework.sh deleted file mode 100644 index 9b886e983..000000000 --- a/tests/functional/lang/framework.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Golden test support -# -# Test that the output of the given test matches what is expected. If -# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so -# that next time the test succeeds. -function diffAndAcceptInner() { - local -r testName=$1 - local -r got="$2" - local -r expected="$3" - - # Absence of expected file indicates empty output expected. - if test -e "$expected"; then - local -r expectedOrEmpty="$expected" - else - local -r expectedOrEmpty=lang/empty.exp - fi - - # Diff so we get a nice message - if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then - echo "FAIL: evaluation result of $testName not as expected" - badDiff=1 - fi - - # Update expected if `_NIX_TEST_ACCEPT` is non-empty. - if test -n "${_NIX_TEST_ACCEPT-}"; then - cp "$got" "$expected" - # Delete empty expected files to avoid bloating the repo with - # empty files. - if ! test -s "$expected"; then - rm "$expected" - fi - fi -} diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index e71224f5e..653a3873f 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -4,6 +4,8 @@ source common.sh needLocalStore "the sandbox only runs on the builder side, so it makes no sense to test it with the daemon" +TODO_NixOS + clearStore requireSandboxSupport diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index 42a6d47f7..b7930e32e 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -15,6 +15,8 @@ declare -a storesBad=( "$storeBadRoot" "$storeBadLower" "$storeBadUpper" ) +TODO_NixOS + for i in "${storesBad[@]}"; do echo $i unshare --mount --map-root-user bash <> "$NIX_CONF_DIR/nix.conf" + echo "$1" >> "$test_nix_conf" } setupConfig () { diff --git a/tests/functional/local-overlay-store/gc-inner.sh b/tests/functional/local-overlay-store/gc-inner.sh index ea92154d2..687fed897 100644 --- a/tests/functional/local-overlay-store/gc-inner.sh +++ b/tests/functional/local-overlay-store/gc-inner.sh @@ -20,8 +20,8 @@ outPath=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg # Set a GC root. mkdir -p "$stateB" -rm -f "$stateB"/gcroots/foo -ln -sf $outPath "$stateB"/gcroots/foo +rm -f "$stateB/gcroots/foo" +ln -sf $outPath "$stateB/gcroots/foo" [ "$(nix-store -q --roots $outPath)" = "$stateB/gcroots/foo -> $outPath" ] @@ -46,7 +46,7 @@ nix-collect-garbage # Check that the root and its dependencies haven't been deleted. cat "$storeBRoot/$outPath" -rm "$stateB"/gcroots/foo +rm "$stateB/gcroots/foo" nix-collect-garbage diff --git a/tests/functional/local.mk b/tests/functional/local.mk index b379eeefe..49ee31284 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -23,7 +23,7 @@ nix_tests = \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ - lang-test-infra.sh \ + characterisation-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ @@ -106,6 +106,7 @@ nix_tests = \ eval-store.sh \ why-depends.sh \ derivation-json.sh \ + derivation-advanced-attributes.sh \ import-derivation.sh \ nix_path.sh \ case-hack.sh \ diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index 63752f6db..bd80a9163 100755 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore path=$(nix-build dependencies.nix --no-out-link) diff --git a/tests/functional/misc.sh b/tests/functional/misc.sh index 9eb80ad22..7d63756b7 100755 --- a/tests/functional/misc.sh +++ b/tests/functional/misc.sh @@ -13,6 +13,9 @@ source common.sh # Can we ask for the version number? nix-env --version | grep "$version" +nix_env=$(type -P nix-env) +(PATH=""; ! $nix_env --help 2>&1 ) | grepQuiet -F "The 'man' command was not found, but it is needed for 'nix-env' and some other 'nix-*' commands' help text. Perhaps you could install the 'man' command?" + # Usage errors. expect 1 nix-env --foo 2>&1 | grep "no operation" expect 1 nix-env -q --foo 2>&1 | grep "unknown flag" diff --git a/tests/functional/multiple-outputs.sh b/tests/functional/multiple-outputs.sh index af9f8af72..35a78d152 100755 --- a/tests/functional/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -2,7 +2,9 @@ source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible rm -f $TEST_ROOT/result* @@ -35,7 +37,7 @@ outPath=$(nix-store -q $drvPath) echo "building b..." outPath=$(nix-build multiple-outputs.nix -A b --no-out-link) echo "output path is $outPath" -[ "$(cat "$outPath"/file)" = "success" ] +[ "$(cat "$outPath/file")" = "success" ] # Test nix-build on a derivation with multiple outputs. outPath1=$(nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result) diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index 8839fd043..b254081cf 100755 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -27,6 +27,8 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +TODO_NixOS + # Check that 'nix store cat' fails on invalid store paths. invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" cp -r $storePath $invalidPath diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh index 44c3bb2bc..ae0256de2 100755 --- a/tests/functional/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -4,6 +4,8 @@ source common.sh # This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing. [[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" +TODO_NixOS + requireSandboxSupport source ./nested-sandboxing/command.sh diff --git a/tests/functional/nix-build.sh b/tests/functional/nix-build.sh index 45ff314c7..091e429e0 100755 --- a/tests/functional/nix-build.sh +++ b/tests/functional/nix-build.sh @@ -2,7 +2,9 @@ source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result) test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR diff --git a/tests/functional/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh index 07aaf61e9..119efe629 100755 --- a/tests/functional/nix-collect-garbage-d.sh +++ b/tests/functional/nix-collect-garbage-d.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore ## Test `nix-collect-garbage -d` diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh index cc8314ff7..5eea9612d 100644 --- a/tests/functional/nix-copy-ssh-common.sh +++ b/tests/functional/nix-copy-ssh-common.sh @@ -2,6 +2,8 @@ proto=$1 shift (( $# == 0 )) +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 1fd735b9d..41958c2c3 100755 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -4,6 +4,8 @@ source common.sh source nix-copy-ssh-common.sh "ssh-ng" +TODO_NixOS + clearStore clearRemoteStore diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 3e5846cf2..e2f19b99e 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearProfiles diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index c38107e64..65ff279f8 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible if [[ -n ${CONTENT_ADDRESSED:-} ]]; then shellDotNix="$PWD/ca-shell.nix" @@ -21,6 +21,10 @@ output=$(nix-shell --pure "$shellDotNix" -A shellDrv --run \ [ "$output" = " - foo - bar - true" ] +output=$(nix-shell --pure "$shellDotNix" -A shellDrv --option nix-shell-always-looks-for-shell-nix false --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') +[ "$output" = " - foo - bar - true" ] + # Test --keep output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') @@ -91,6 +95,55 @@ sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > $TEST_ROOT/shell.sheba chmod a+rx $TEST_ROOT/shell.shebang.nix $TEST_ROOT/shell.shebang.nix +mkdir $TEST_ROOT/lookup-test $TEST_ROOT/empty + +echo "import $shellDotNix" > $TEST_ROOT/lookup-test/shell.nix +cp config.nix $TEST_ROOT/lookup-test/ +echo 'abort "do not load default.nix!"' > $TEST_ROOT/lookup-test/default.nix + +nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' | grepQuiet "it works" +# https://github.com/NixOS/nix/issues/4529 +nix-shell -I "testRoot=$TEST_ROOT" '' -A shellDrv --run 'echo "it works"' | grepQuiet "it works" + +expectStderr 1 nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ + | grepQuiet -F "do not load default.nix!" # we did, because we chose to enable legacy behavior +expectStderr 1 nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ + | grepQuiet "Skipping .*lookup-test/shell\.nix.*, because the setting .*nix-shell-always-looks-for-shell-nix.* is disabled. This is a deprecated behavior\. Consider enabling .*nix-shell-always-looks-for-shell-nix.*" + +( + cd $TEST_ROOT/empty; + expectStderr 1 nix-shell | \ + grepQuiet "error.*no argument specified and no .*shell\.nix.* or .*default\.nix.* file found in the working directory" +) + +expectStderr 1 nix-shell -I "testRoot=$TEST_ROOT" '' | + grepQuiet "error.*neither .*shell\.nix.* nor .*default\.nix.* found in .*/empty" + +cat >$TEST_ROOT/lookup-test/shebangscript < $TEST_ROOT/marco/shell.nix +cat >$TEST_ROOT/marco/polo/default.nix < $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +TODO_NixOS # ignoring the client-specified setting 'auto-optimise-store', because it is a restricted setting and you are not a trusted user + # TODO: only continue when trusted user or root + inode1="$(stat --format=%i $outPath1/foo)" inode2="$(stat --format=%i $outPath2/foo)" if [ "$inode1" != "$inode2" ]; then diff --git a/tests/functional/output-normalization.sh b/tests/functional/output-normalization.sh index 2b319201a..c55f1b1d1 100755 --- a/tests/functional/output-normalization.sh +++ b/tests/functional/output-normalization.sh @@ -3,6 +3,7 @@ source common.sh testNormalization () { + TODO_NixOS clearStore outPath=$(nix-build ./simple.nix --no-out-link) test "$(stat -c %Y $outPath)" -eq 1 diff --git a/tests/functional/parallel.sh b/tests/functional/parallel.sh index 3b7bbe5a2..7e420688d 100644 --- a/tests/functional/parallel.sh +++ b/tests/functional/parallel.sh @@ -4,6 +4,8 @@ source common.sh # First, test that -jN performs builds in parallel. echo "testing nix-build -j..." +TODO_NixOS + clearStore rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max diff --git a/tests/functional/pass-as-file.sh b/tests/functional/pass-as-file.sh index 21d9ffc6d..6487bfffd 100755 --- a/tests/functional/pass-as-file.sh +++ b/tests/functional/pass-as-file.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible outPath=$(nix-build --no-out-link -E " with import ./config.nix; diff --git a/tests/functional/placeholders.sh b/tests/functional/placeholders.sh index f2b8bf6bf..33ec0c2b7 100755 --- a/tests/functional/placeholders.sh +++ b/tests/functional/placeholders.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc index e02fd68d5..7433ad190 100644 --- a/tests/functional/plugins/plugintest.cc +++ b/tests/functional/plugins/plugintest.cc @@ -1,4 +1,4 @@ -#include "config.hh" +#include "config-global.hh" #include "primops.hh" using namespace nix; diff --git a/tests/functional/post-hook.sh b/tests/functional/post-hook.sh index c0b1ab3aa..94a6d0d69 100755 --- a/tests/functional/post-hook.sh +++ b/tests/functional/post-hook.sh @@ -2,12 +2,14 @@ source common.sh +TODO_NixOS + clearStore rm -f $TEST_ROOT/result export REMOTE_STORE=file:$TEST_ROOT/remote_store -echo 'require-sigs = false' >> $NIX_CONF_DIR/nix.conf +echo 'require-sigs = false' >> $test_nix_conf restartDaemon diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh index 6d8aa35ec..250381099 100755 --- a/tests/functional/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible nix eval --expr 'assert 1 + 2 == 3; true' diff --git a/tests/functional/read-only-store.sh b/tests/functional/read-only-store.sh index ecc57642d..f6b6eaf32 100755 --- a/tests/functional/read-only-store.sh +++ b/tests/functional/read-only-store.sh @@ -6,6 +6,8 @@ enableFeatures "read-only-local-store" needLocalStore "cannot open store read-only when daemon has already opened it writeable" +TODO_NixOS + clearStore happy () { diff --git a/tests/functional/readfile-context.sh b/tests/functional/readfile-context.sh index 76fad9349..cb9ef6234 100755 --- a/tests/functional/readfile-context.sh +++ b/tests/functional/readfile-context.sh @@ -2,12 +2,14 @@ source common.sh +TODO_NixOS # NixOS doesn't provide $NIX_STATE_DIR (and shouldn't) + clearStore outPath=$(nix-build --no-out-link readfile-context.nix) # Set a GC root. -ln -s $outPath "$NIX_STATE_DIR"/gcroots/foo +ln -s $outPath "$NIX_STATE_DIR/gcroots/foo" # Check that file exists. [ "$(cat $(cat $outPath))" = "Hello World!" ] diff --git a/tests/functional/recursive.sh b/tests/functional/recursive.sh index a9966aabd..640fb92d2 100755 --- a/tests/functional/recursive.sh +++ b/tests/functional/recursive.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS # can't enable a sandbox feature easily + enableFeatures 'recursive-nix' restartDaemon diff --git a/tests/functional/referrers.sh b/tests/functional/referrers.sh index 898032e42..411cdb7c1 100755 --- a/tests/functional/referrers.sh +++ b/tests/functional/referrers.sh @@ -4,6 +4,8 @@ source common.sh needLocalStore "uses some low-level store manipulations that aren’t available through the daemon" +TODO_NixOS + clearStore max=500 @@ -31,7 +33,7 @@ echo "registering..." nix-store --register-validity < $TEST_ROOT/reg_info echo "collecting garbage..." -ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref +ln -sfn $reference "$NIX_STATE_DIR/gcroots/ref" nix-store --gc if [ -n "$(type -p sqlite3)" -a "$(sqlite3 $NIX_STATE_DIR/db/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index 171a5d391..841b6b27a 100755 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. diff --git a/tests/functional/repair.sh b/tests/functional/repair.sh index 552e04280..1f6004b2c 100755 --- a/tests/functional/repair.sh +++ b/tests/functional/repair.sh @@ -4,6 +4,8 @@ source common.sh needLocalStore "--repair needs a local store" +TODO_NixOS + clearStore path=$(nix-build dependencies.nix -o $TEST_ROOT/result) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index fca982807..86cd6f458 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -22,6 +22,8 @@ replUndefinedVariable=" import $testDir/undefined-variable.nix " +TODO_NixOS + testRepl () { local nixArgs nixArgs=("$@") diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index ab4cad5cf..915d973b0 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible nix-instantiate --restrict-eval --eval -E '1 + 2' (! nix-instantiate --eval --restrict-eval ./restricted.nix) diff --git a/tests/functional/search.sh b/tests/functional/search.sh index ce17411d2..3fadecd02 100755 --- a/tests/functional/search.sh +++ b/tests/functional/search.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible clearCache (( $(nix search -f search.nix '' hello | wc -l) > 0 )) diff --git a/tests/functional/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh index 7d81db58b..5cc4af435 100755 --- a/tests/functional/secure-drv-outputs.sh +++ b/tests/functional/secure-drv-outputs.sh @@ -6,6 +6,8 @@ source common.sh +TODO_NixOS + clearStore startDaemon diff --git a/tests/functional/selfref-gc.sh b/tests/functional/selfref-gc.sh index 37ce33089..518aea66b 100755 --- a/tests/functional/selfref-gc.sh +++ b/tests/functional/selfref-gc.sh @@ -4,7 +4,7 @@ source common.sh requireDaemonNewerThan "2.6.0pre20211215" -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix index 6a7dd7ad1..750cdf0bc 100644 --- a/tests/functional/shell.nix +++ b/tests/functional/shell.nix @@ -26,6 +26,9 @@ let pkgs = rec { fun() { echo blabla } + runHook() { + eval "''${!1}" + } ''; stdenv = mkDerivation { @@ -45,6 +48,18 @@ let pkgs = rec { TEST_inNixShell = if inNixShell then "true" else "false"; inherit stdenv; outputs = ["dev" "out"]; + } // { + shellHook = abort "Ignore non-drv shellHook attr"; + }; + + # https://github.com/NixOS/nix/issues/5431 + # See nix-shell.sh + polo = mkDerivation { + name = "polo"; + inherit stdenv; + shellHook = '' + echo Polo + ''; }; # Used by nix-shell -p diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index 1760eefff..b4c03f547 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index cf84ab377..890d1446f 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible clearCache nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 @@ -16,6 +16,7 @@ outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROO # Verify that the path got signed. info=$(nix path-info --json $outPath) echo $info | jq -e '.[] | .ultimate == true' +TODO_NixOS # looks like an actual bug? Following line fails on NixOS: echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index 846738cbd..86acca0c2 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -14,9 +14,11 @@ echo "output path is $outPath" (! [ -w $outPath ]) -text=$(cat "$outPath"/hello) +text=$(cat "$outPath/hello") if test "$text" != "Hello World!"; then exit 1; fi +TODO_NixOS + # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. nix-store --delete $outPath diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index 2398f5beb..f37889fbb 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -16,4 +16,6 @@ fi expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ fail "nix store info on a non-existent store should fail" +TODO_NixOS + [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index ba7f5967e..ec1282668 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -6,7 +6,7 @@ source common.sh # tests for the older versions requireDaemonNewerThan "2.4pre20210712" -clearStore +clearStoreIfPossible rm -f $TEST_ROOT/result @@ -21,6 +21,8 @@ env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' +TODO_NixOS # following line fails. + # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index 6ec1cd322..8db6f7b97 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible cd "$TEST_HOME" diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh index 9d474219f..5d329efc9 100755 --- a/tests/functional/supplementary-groups.sh +++ b/tests/functional/supplementary-groups.sh @@ -7,6 +7,8 @@ requireSandboxSupport if ! command -p -v unshare; then skipTest "Need unshare"; fi needLocalStore "The test uses --store always so we would just be bypassing the daemon" +TODO_NixOS + unshare --mount --map-root-user bash <&2 + exit 1 + fi + store_path2=$(nix store prefetch-file --json --unpack "file://$tarball" | jq -r .storePath) + diff_output=$(diff -r "$store_path2" "$tarroot") + if [ -n "$diff_output" ]; then + echo "prefetched tarball differs from original: $store_path2 vs $tarroot" >&2 + echo "$diff_output" + exit 1 + fi } test_tarball '' cat diff --git a/tests/functional/timeout.sh b/tests/functional/timeout.sh index 441c83b0e..f42354538 100755 --- a/tests/functional/timeout.sh +++ b/tests/functional/timeout.sh @@ -11,6 +11,10 @@ messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status= if [ $status -ne 101 ]; then echo "error: 'nix-store' exited with '$status'; should have exited 101" + + # FIXME: https://github.com/NixOS/nix/issues/4813 + skipTest "Do not block CI until fixed" + exit 1 fi diff --git a/tests/functional/user-envs-migration.sh b/tests/functional/user-envs-migration.sh index 992586b95..0f33074e1 100755 --- a/tests/functional/user-envs-migration.sh +++ b/tests/functional/user-envs-migration.sh @@ -13,6 +13,8 @@ fi killDaemon unset NIX_REMOTE +TODO_NixOS + clearStore clearProfiles rm -rf ~/.nix-profile diff --git a/tests/functional/user-envs-test-case.sh b/tests/functional/user-envs-test-case.sh index f4a90a675..117c6c7a4 100644 --- a/tests/functional/user-envs-test-case.sh +++ b/tests/functional/user-envs-test-case.sh @@ -11,6 +11,8 @@ outPath10=$(nix-env -f ./user-envs.nix -qa --out-path --no-name '*' | grep foo-1 drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1.0) [ -n "$outPath10" -a -n "$drvPath10" ] +TODO_NixOS + # Query with json nix-env -f ./user-envs.nix -qa --json | jq -e '.[] | select(.name == "bar-0.1") | [ .outputName == "out", diff --git a/tests/functional/why-depends.sh b/tests/functional/why-depends.sh index 69b365069..ce53546d8 100755 --- a/tests/functional/why-depends.sh +++ b/tests/functional/why-depends.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible cp ./dependencies.nix ./dependencies.builder0.sh ./config.nix $TEST_HOME diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh index 3bf9d5601..90fe58539 100755 --- a/tests/functional/zstd.sh +++ b/tests/functional/zstd.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 710f8a273..c0c7b42fd 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -7,7 +7,13 @@ let # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests runNixOSTestFor = system: test: (nixos-lib.runTest { - imports = [ test ]; + imports = [ + test + + # Add the quickBuild attribute to the check packages + ./quick-build.nix + ]; + hostPkgs = nixpkgsFor.${system}.native; defaults = { nixpkgs.pkgs = nixpkgsFor.${system}.native; @@ -132,4 +138,12 @@ in ca-fd-leak = runNixOSTestFor "x86_64-linux" ./ca-fd-leak; gzip-content-encoding = runNixOSTestFor "x86_64-linux" ./gzip-content-encoding.nix; + + functional_user = runNixOSTestFor "x86_64-linux" ./functional/as-user.nix; + + functional_trusted = runNixOSTestFor "x86_64-linux" ./functional/as-trusted-user.nix; + + functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; + + user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; } diff --git a/tests/nixos/functional/as-root.nix b/tests/nixos/functional/as-root.nix new file mode 100644 index 000000000..96be3d593 --- /dev/null +++ b/tests/nixos/functional/as-root.nix @@ -0,0 +1,12 @@ +{ + name = "functional-tests-on-nixos_root"; + + imports = [ ./common.nix ]; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + run-test-suite >&2 + """) + ''; +} diff --git a/tests/nixos/functional/as-trusted-user.nix b/tests/nixos/functional/as-trusted-user.nix new file mode 100644 index 000000000..d6f825697 --- /dev/null +++ b/tests/nixos/functional/as-trusted-user.nix @@ -0,0 +1,18 @@ +{ + name = "functional-tests-on-nixos_trusted-user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + nix.settings.trusted-users = [ "alice" ]; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + export TEST_TRUSTED_USER=1 + su --login --command "run-test-suite" alice >&2 + """) + ''; +} \ No newline at end of file diff --git a/tests/nixos/functional/as-user.nix b/tests/nixos/functional/as-user.nix new file mode 100644 index 000000000..1443f6e6c --- /dev/null +++ b/tests/nixos/functional/as-user.nix @@ -0,0 +1,16 @@ +{ + name = "functional-tests-on-nixos_user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + su --login --command "run-test-suite" alice >&2 + """) + ''; +} diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix new file mode 100644 index 000000000..51fd76884 --- /dev/null +++ b/tests/nixos/functional/common.nix @@ -0,0 +1,71 @@ +{ lib, ... }: + +let + # FIXME (roberth) reference issue + inputDerivation = pkg: (pkg.overrideAttrs (o: { + disallowedReferences = [ ]; + })).inputDerivation; + +in +{ + # We rarely change the script in a way that benefits from type checking, so + # we skip it to save time. + skipTypeCheck = true; + + nodes.machine = { config, pkgs, ... }: { + + virtualisation.writableStore = true; + system.extraDependencies = [ + (inputDerivation config.nix.package) + ]; + + nix.settings.substituters = lib.mkForce []; + + environment.systemPackages = let + run-test-suite = pkgs.writeShellApplication { + name = "run-test-suite"; + runtimeInputs = [ pkgs.gnumake pkgs.jq pkgs.git ]; + text = '' + set -x + cat /proc/sys/fs/file-max + ulimit -Hn + ulimit -Sn + cd ~ + cp -r ${pkgs.nix.overrideAttrs (o: { + name = "nix-configured-source"; + outputs = [ "out" ]; + separateDebugInfo = false; + disallowedReferences = [ ]; + buildPhase = ":"; + checkPhase = ":"; + installPhase = '' + cp -r . $out + ''; + installCheckPhase = ":"; + fixupPhase = ":"; + doInstallCheck = true; + })} nix + chmod -R +w nix + cd nix + + # Tests we don't need + echo >tests/functional/plugins/local.mk + sed -i tests/functional/local.mk \ + -e 's!nix_tests += plugins\.sh!!' \ + -e 's!nix_tests += test-libstoreconsumer\.sh!!' \ + ; + + export isTestOnNixOS=1 + export version=${config.nix.package.version} + export NIX_REMOTE_=daemon + export NIX_REMOTE=daemon + export NIX_STORE=${builtins.storeDir} + make -j1 installcheck --keep-going + ''; + }; + in [ + run-test-suite + pkgs.git + ]; + }; +} diff --git a/tests/nixos/quick-build.nix b/tests/nixos/quick-build.nix new file mode 100644 index 000000000..37169b4e2 --- /dev/null +++ b/tests/nixos/quick-build.nix @@ -0,0 +1,47 @@ +test@{ lib, extendModules, ... }: +let + inherit (lib) mkOption types; +in +{ + options = { + quickBuild = mkOption { + description = '' + Whether to perform a "quick" build of the Nix package to test. + + When iterating on the functional tests, it's recommended to "set" this + to `true`, so that changes to the functional tests don't require any + recompilation of the package. + You can do so by building the `.quickBuild` attribute on the check package, + e.g: + ```console + nix build .#hydraJobs.functional_user.quickBuild + ``` + + We don't enable this by default to avoid the mostly unnecessary work of + performing an additional build of the package in cases where we build + the package normally anyway, such as in our pre-merge CI. + ''; + type = types.bool; + default = false; + }; + }; + + config = { + passthru.quickBuild = + let withQuickBuild = extendModules { modules = [{ quickBuild = true; }]; }; + in withQuickBuild.config.test; + + defaults = { pkgs, ... }: { + config = lib.mkIf test.config.quickBuild { + nix.package = pkgs.nix_noTests; + + system.forbiddenDependenciesRegexes = [ + # This would indicate that the quickBuild feature is broken. + # It could happen if NixOS has a dependency on pkgs.nix instead of + # config.nix.package somewhere. + (builtins.unsafeDiscardStringContext pkgs.nix.outPath) + ]; + }; + }; + }; +} \ No newline at end of file diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index 2e7f98b5e..84cf377ec 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -70,12 +70,17 @@ in # Check that we got redirected to the immutable URL. assert info["locked"]["url"] == "http://localhost/stable/${nixpkgs.rev}.tar.gz" + # Check that we got a fingerprint for caching. + assert info["fingerprint"] + # Check that we got the rev and revCount attributes. assert info["revision"] == "${nixpkgs.rev}" assert info["revCount"] == 1234 - # Check that fetching with rev/revCount/narHash succeeds. + # Check that a 0-byte HTTP 304 "Not modified" result works. + machine.succeed("nix flake metadata --refresh --json http://localhost/tags/latest.tar.gz") + # Check that fetching with rev/revCount/narHash succeeds. machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz?rev=" + info["revision"]) machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz?revCount=" + str(info["revCount"])) machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz?narHash=" + info["locked"]["narHash"]) diff --git a/tests/nixos/user-sandboxing/attacker.c b/tests/nixos/user-sandboxing/attacker.c new file mode 100644 index 000000000..3bd729c04 --- /dev/null +++ b/tests/nixos/user-sandboxing/attacker.c @@ -0,0 +1,82 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include + +#define SYS_fchmodat2 452 + +int fchmodat2(int dirfd, const char *pathname, mode_t mode, int flags) { + return syscall(SYS_fchmodat2, dirfd, pathname, mode, flags); +} + +int main(int argc, char **argv) { + if (argc <= 1) { + // stage 1: place the setuid-builder executable + + // make the build directory world-accessible first + chmod(".", 0755); + + if (fchmodat2(AT_FDCWD, "attacker", 06755, AT_SYMLINK_NOFOLLOW) < 0) { + perror("Setting the suid bit on attacker"); + exit(-1); + } + + } else { + // stage 2: corrupt the victim derivation while it's building + + // prevent the kill + if (setresuid(-1, -1, getuid())) { + perror("setresuid"); + exit(-1); + } + + if (fork() == 0) { + + // wait for the victim to build + int fd = inotify_init(); + inotify_add_watch(fd, argv[1], IN_CREATE); + int dirfd = open(argv[1], O_DIRECTORY); + if (dirfd < 0) { + perror("opening the global build directory"); + exit(-1); + } + char buf[4096]; + fprintf(stderr, "Entering the inotify loop\n"); + for (;;) { + ssize_t len = read(fd, buf, sizeof(buf)); + struct inotify_event *ev; + for (char *pe = buf; pe < buf + len; + pe += sizeof(struct inotify_event) + ev->len) { + ev = (struct inotify_event *)pe; + fprintf(stderr, "folder %s created\n", ev->name); + // wait a bit to prevent racing against the creation + sleep(1); + int builddir = openat(dirfd, ev->name, O_DIRECTORY); + if (builddir < 0) { + perror("opening the build directory"); + continue; + } + int resultfile = openat(builddir, "build/result", O_WRONLY | O_TRUNC); + if (resultfile < 0) { + perror("opening the hijacked file"); + continue; + } + int writeres = write(resultfile, "bad\n", 4); + if (writeres < 0) { + perror("writing to the hijacked file"); + continue; + } + fprintf(stderr, "Hijacked the build for %s\n", ev->name); + return 0; + } + } + } + + exit(0); + } +} + diff --git a/tests/nixos/user-sandboxing/default.nix b/tests/nixos/user-sandboxing/default.nix new file mode 100644 index 000000000..8a16f44e8 --- /dev/null +++ b/tests/nixos/user-sandboxing/default.nix @@ -0,0 +1,129 @@ +{ config, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + attacker = pkgs.runCommandWith { + name = "attacker"; + stdenv = pkgs.pkgsStatic.stdenv; + } '' + $CC -static -o $out ${./attacker.c} + ''; + + try-open-build-dir = pkgs.writeScript "try-open-build-dir" '' + export PATH=${pkgs.coreutils}/bin:$PATH + + set -x + + chmod 700 . + # Shouldn't be able to open the root build directory + (! chmod 700 ..) + + touch foo + + # Synchronisation point: create a world-writable fifo and wait for someone + # to write into it + mkfifo syncPoint + chmod 777 syncPoint + cat syncPoint + + touch $out + + set +x + ''; + + create-hello-world = pkgs.writeScript "create-hello-world" '' + export PATH=${pkgs.coreutils}/bin:$PATH + + set -x + + echo "hello, world" > result + + # Synchronisation point: create a world-writable fifo and wait for someone + # to write into it + mkfifo syncPoint + chmod 777 syncPoint + cat syncPoint + + cp result $out + + set +x + ''; + +in +{ + name = "sandbox-setuid-leak"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + nix.settings.substituters = lib.mkForce [ ]; + nix.nrBuildUsers = 1; + virtualisation.additionalPaths = [ pkgs.busybox-sandbox-shell attacker try-open-build-dir create-hello-world pkgs.socat ]; + boot.kernelPackages = pkgs.linuxPackages_latest; + users.users.alice = { + isNormalUser = true; + }; + }; + + testScript = { nodes }: '' + start_all() + + with subtest("A builder can't give access to its build directory"): + # Make sure that a builder can't change the permissions on its build + # directory to the point of opening it up to external users + + # A derivation whose builder tries to make its build directory as open + # as possible and wait for someone to hijack it + machine.succeed(r""" + nix-build -v -E ' + builtins.derivation { + name = "open-build-dir"; + system = builtins.currentSystem; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ (builtins.storePath "${try-open-build-dir}") ]; + }' >&2 & + """.strip()) + + # Wait for the build to be ready + # This is OK because it runs as root, so we can access everything + machine.wait_for_file("/tmp/nix-build-open-build-dir.drv-0/build/syncPoint") + + # But Alice shouldn't be able to access the build directory + machine.fail("su alice -c 'ls /tmp/nix-build-open-build-dir.drv-0/build'") + machine.fail("su alice -c 'touch /tmp/nix-build-open-build-dir.drv-0/build/bar'") + machine.fail("su alice -c 'cat /tmp/nix-build-open-build-dir.drv-0/build/foo'") + + # Tell the user to finish the build + machine.succeed("echo foo > /tmp/nix-build-open-build-dir.drv-0/build/syncPoint") + + with subtest("Being able to execute stuff as the build user doesn't give access to the build dir"): + machine.succeed(r""" + nix-build -E ' + builtins.derivation { + name = "innocent"; + system = builtins.currentSystem; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ (builtins.storePath "${create-hello-world}") ]; + }' >&2 & + """.strip()) + machine.wait_for_file("/tmp/nix-build-innocent.drv-0/build/syncPoint") + + # The build ran as `nixbld1` (which is the only build user on the + # machine), but a process running as `nixbld1` outside the sandbox + # shouldn't be able to touch the build directory regardless + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'ls /tmp/nix-build-innocent.drv-0/build'") + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'echo pwned > /tmp/nix-build-innocent.drv-0/build/result'") + + # Finish the build + machine.succeed("echo foo > /tmp/nix-build-innocent.drv-0/build/syncPoint") + + # Check that the build was not affected + machine.succeed(r""" + cat ./result + test "$(cat ./result)" = "hello, world" + """.strip()) + ''; + +} + diff --git a/tests/unit/libexpr-support/.version b/tests/unit/libexpr-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libexpr-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libexpr-support/build-utils-meson b/tests/unit/libexpr-support/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libexpr-support/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libexpr-support/meson.build b/tests/unit/libexpr-support/meson.build new file mode 100644 index 000000000..705672204 --- /dev/null +++ b/tests/unit/libexpr-support/meson.build @@ -0,0 +1,74 @@ +project('nix-expr-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-util-test-support'), + dependency('nix-store'), + dependency('nix-store-test-support'), + dependency('nix-expr'), +] +subdir('build-utils-meson/subprojects') + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'tests/value/context.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/libexpr.hh', + 'tests/nix_api_expr.hh', + 'tests/value/context.hh', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nix-expr-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/tests/unit/libexpr-support/package.nix b/tests/unit/libexpr-support/package.nix new file mode 100644 index 000000000..0c966c55a --- /dev/null +++ b/tests/unit/libexpr-support/package.nix @@ -0,0 +1,81 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store-test-support +, nix-expr + +, rapidcheck + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-test-support"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-store-test-support + nix-expr + rapidcheck + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 1a4313990..eacbf0d5c 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -19,14 +19,14 @@ namespace nix { static void SetUpTestSuite() { LibStoreTest::SetUpTestSuite(); initGC(); - evalSettings.nixPath = {}; } protected: LibExprTest() : LibStoreTest() - , state({}, store) + , state({}, store, evalSettings, nullptr) { + evalSettings.nixPath = {}; } Value eval(std::string input, bool forceValue = true) { Value v; @@ -42,6 +42,8 @@ namespace nix { return state.symbols.create(value); } + bool readOnlyMode = true; + EvalSettings evalSettings{readOnlyMode}; EvalState state; }; diff --git a/tests/unit/libexpr-support/tests/nix_api_expr.hh b/tests/unit/libexpr-support/tests/nix_api_expr.hh index d1840d034..6ddca0d14 100644 --- a/tests/unit/libexpr-support/tests/nix_api_expr.hh +++ b/tests/unit/libexpr-support/tests/nix_api_expr.hh @@ -25,7 +25,7 @@ protected: } EvalState * state; - Value * value; + nix_value * value; }; } diff --git a/tests/unit/libexpr/.version b/tests/unit/libexpr/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libexpr/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libexpr/build-utils-meson b/tests/unit/libexpr/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libexpr/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libexpr/data/.gitkeep b/tests/unit/libexpr/data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk index 09a7dfca1..1617e2823 100644 --- a/tests/unit/libexpr/local.mk +++ b/tests/unit/libexpr/local.mk @@ -4,7 +4,7 @@ programs += libexpr-tests libexpr-tests_NAME := libnixexpr-tests -libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libexpr-tests.xml libexpr-tests_DIR := $(d) diff --git a/tests/unit/libexpr/meson.build b/tests/unit/libexpr/meson.build new file mode 100644 index 000000000..ee35258cf --- /dev/null +++ b/tests/unit/libexpr/meson.build @@ -0,0 +1,90 @@ +project('nix-expr-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-expr'), + dependency('nix-expr-c'), + dependency('nix-expr-test-support'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest') +deps_private += gtest + +gtest = dependency('gmock') +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'derived-path.cc', + 'error_traces.cc', + 'eval.cc', + 'json.cc', + 'main.cc', + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', + 'primops.cc', + 'search-path.cc', + 'trivial.cc', + 'value/context.cc', + 'value/print.cc', + 'value/value.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libexpr/nix_api_expr.cc b/tests/unit/libexpr/nix_api_expr.cc index 92a6a1175..8b97d6923 100644 --- a/tests/unit/libexpr/nix_api_expr.cc +++ b/tests/unit/libexpr/nix_api_expr.cc @@ -39,12 +39,12 @@ TEST_F(nix_api_expr_test, nix_expr_eval_drv) ASSERT_EQ(NIX_TYPE_ATTRS, nix_get_type(nullptr, value)); EvalState * stateFn = nix_state_create(nullptr, nullptr, store); - Value * valueFn = nix_alloc_value(nullptr, state); + nix_value * valueFn = nix_alloc_value(nullptr, state); nix_expr_eval_from_string(nullptr, stateFn, "builtins.toString", ".", valueFn); ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(nullptr, valueFn)); EvalState * stateResult = nix_state_create(nullptr, nullptr, store); - Value * valueResult = nix_alloc_value(nullptr, stateResult); + nix_value * valueResult = nix_alloc_value(nullptr, stateResult); nix_value_call(ctx, stateResult, valueFn, value, valueResult); ASSERT_EQ(NIX_TYPE_STRING, nix_get_type(nullptr, valueResult)); @@ -70,7 +70,7 @@ TEST_F(nix_api_expr_test, nix_build_drv) })"; nix_expr_eval_from_string(nullptr, state, expr, ".", value); - Value * drvPathValue = nix_get_attr_byname(nullptr, value, state, "drvPath"); + nix_value * drvPathValue = nix_get_attr_byname(nullptr, value, state, "drvPath"); std::string drvPath; nix_get_string(nullptr, drvPathValue, OBSERVE_STRING(drvPath)); @@ -84,7 +84,7 @@ TEST_F(nix_api_expr_test, nix_build_drv) StorePath * drvStorePath = nix_store_parse_path(ctx, store, drvPath.c_str()); ASSERT_EQ(true, nix_store_is_valid_path(ctx, store, drvStorePath)); - Value * outPathValue = nix_get_attr_byname(ctx, value, state, "outPath"); + nix_value * outPathValue = nix_get_attr_byname(ctx, value, state, "outPath"); std::string outPath; nix_get_string(ctx, outPathValue, OBSERVE_STRING(outPath)); @@ -193,7 +193,8 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context) const char * SAMPLE_USER_DATA = "whatever"; -static void primop_square(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret) +static void +primop_square(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) { assert(context); assert(state); @@ -207,17 +208,17 @@ TEST_F(nix_api_expr_test, nix_expr_primop) PrimOp * primop = nix_alloc_primop(ctx, primop_square, 1, "square", nullptr, "square an integer", (void *) SAMPLE_USER_DATA); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * three = nix_alloc_value(ctx, state); + nix_value * three = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, three, 3); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_value_call(ctx, state, primopValue, three, result); assert_ctx_ok(); @@ -226,7 +227,8 @@ TEST_F(nix_api_expr_test, nix_expr_primop) ASSERT_EQ(9, r); } -static void primop_repeat(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret) +static void +primop_repeat(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) { assert(context); assert(state); @@ -255,27 +257,27 @@ TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_multiple_calls) PrimOp * primop = nix_alloc_primop(ctx, primop_repeat, 2, "repeat", nullptr, "repeat a string", (void *) SAMPLE_USER_DATA); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * hello = nix_alloc_value(ctx, state); + nix_value * hello = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_string(ctx, hello, "hello"); assert_ctx_ok(); - Value * three = nix_alloc_value(ctx, state); + nix_value * three = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, three, 3); assert_ctx_ok(); - Value * partial = nix_alloc_value(ctx, state); + nix_value * partial = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_value_call(ctx, state, primopValue, hello, partial); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_value_call(ctx, state, partial, three, result); assert_ctx_ok(); @@ -290,22 +292,22 @@ TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_single_call) PrimOp * primop = nix_alloc_primop(ctx, primop_repeat, 2, "repeat", nullptr, "repeat a string", (void *) SAMPLE_USER_DATA); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * hello = nix_alloc_value(ctx, state); + nix_value * hello = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_string(ctx, hello, "hello"); assert_ctx_ok(); - Value * three = nix_alloc_value(ctx, state); + nix_value * three = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, three, 3); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); NIX_VALUE_CALL(ctx, state, result, primopValue, hello, three); assert_ctx_ok(); @@ -318,7 +320,7 @@ TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_single_call) } static void -primop_bad_no_return(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret) +primop_bad_no_return(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) { } @@ -327,17 +329,17 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_no_return) PrimOp * primop = nix_alloc_primop(ctx, primop_bad_no_return, 1, "badNoReturn", nullptr, "a broken primop", nullptr); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * three = nix_alloc_value(ctx, state); + nix_value * three = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, three, 3); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_value_call(ctx, state, primopValue, three, result); ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); @@ -348,8 +350,8 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_no_return) ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badNoReturn"))); } -static void -primop_bad_return_thunk(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret) +static void primop_bad_return_thunk( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) { nix_init_apply(context, ret, args[0], args[1]); } @@ -358,22 +360,22 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk) PrimOp * primop = nix_alloc_primop(ctx, primop_bad_return_thunk, 2, "badReturnThunk", nullptr, "a broken primop", nullptr); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * toString = nix_alloc_value(ctx, state); + nix_value * toString = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_expr_eval_from_string(ctx, state, "builtins.toString", ".", toString); assert_ctx_ok(); - Value * four = nix_alloc_value(ctx, state); + nix_value * four = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, four, 4); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); NIX_VALUE_CALL(ctx, state, result, primopValue, toString, four); @@ -387,11 +389,11 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk) TEST_F(nix_api_expr_test, nix_value_call_multi_no_args) { - Value * n = nix_alloc_value(ctx, state); + nix_value * n = nix_alloc_value(ctx, state); nix_init_int(ctx, n, 3); assert_ctx_ok(); - Value * r = nix_alloc_value(ctx, state); + nix_value * r = nix_alloc_value(ctx, state); nix_value_call_multi(ctx, state, n, 0, nullptr, r); assert_ctx_ok(); diff --git a/tests/unit/libexpr/nix_api_external.cc b/tests/unit/libexpr/nix_api_external.cc index 2391f8317..81ff285a4 100644 --- a/tests/unit/libexpr/nix_api_external.cc +++ b/tests/unit/libexpr/nix_api_external.cc @@ -49,10 +49,10 @@ TEST_F(nix_api_expr_test, nix_expr_eval_external) nix_init_external(ctx, value, val); EvalState * stateResult = nix_state_create(nullptr, nullptr, store); - Value * valueResult = nix_alloc_value(nullptr, stateResult); + nix_value * valueResult = nix_alloc_value(nullptr, stateResult); EvalState * stateFn = nix_state_create(nullptr, nullptr, store); - Value * valueFn = nix_alloc_value(nullptr, stateFn); + nix_value * valueFn = nix_alloc_value(nullptr, stateFn); nix_expr_eval_from_string(nullptr, state, "builtins.typeOf", ".", valueFn); diff --git a/tests/unit/libexpr/nix_api_value.cc b/tests/unit/libexpr/nix_api_value.cc index c71593c85..7fc8b4f64 100644 --- a/tests/unit/libexpr/nix_api_value.cc +++ b/tests/unit/libexpr/nix_api_value.cc @@ -4,16 +4,26 @@ #include "nix_api_util_internal.h" #include "nix_api_expr.h" #include "nix_api_value.h" +#include "nix_api_expr_internal.h" #include "tests/nix_api_expr.hh" #include "tests/string_callback.hh" #include "gmock/gmock.h" +#include #include #include namespace nixC { +TEST_F(nix_api_expr_test, as_nix_value_ptr) +{ + // nix_alloc_value casts nix::Value to nix_value + // It should be obvious from the decl that that works, but if it doesn't, + // the whole implementation would be utterly broken. + ASSERT_EQ(sizeof(nix::Value), sizeof(nix_value)); +} + TEST_F(nix_api_expr_test, nix_value_get_int_invalid) { ASSERT_EQ(0, nix_get_int(ctx, nullptr)); @@ -138,8 +148,8 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list) int size = 10; ListBuilder * builder = nix_make_list_builder(ctx, state, size); - Value * intValue = nix_alloc_value(ctx, state); - Value * intValue2 = nix_alloc_value(ctx, state); + nix_value * intValue = nix_alloc_value(ctx, state); + nix_value * intValue2 = nix_alloc_value(ctx, state); // `init` and `insert` can be called in any order nix_init_int(ctx, intValue, 42); @@ -194,10 +204,10 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, size); - Value * intValue = nix_alloc_value(ctx, state); + nix_value * intValue = nix_alloc_value(ctx, state); nix_init_int(ctx, intValue, 42); - Value * stringValue = nix_alloc_value(ctx, state); + nix_value * stringValue = nix_alloc_value(ctx, state); nix_init_string(ctx, stringValue, "foo"); nix_bindings_builder_insert(ctx, builder, "a", intValue); @@ -207,7 +217,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) ASSERT_EQ(2, nix_get_attrs_size(ctx, value)); - Value * out_value = nix_get_attr_byname(ctx, value, state, "a"); + nix_value * out_value = nix_get_attr_byname(ctx, value, state, "a"); ASSERT_EQ(42, nix_get_int(ctx, out_value)); nix_gc_decref(ctx, out_value); @@ -251,10 +261,10 @@ TEST_F(nix_api_expr_test, nix_value_init) // two = 2; // f = a: a * a; - Value * two = nix_alloc_value(ctx, state); + nix_value * two = nix_alloc_value(ctx, state); nix_init_int(ctx, two, 2); - Value * f = nix_alloc_value(ctx, state); + nix_value * f = nix_alloc_value(ctx, state); nix_expr_eval_from_string( ctx, state, @@ -268,7 +278,7 @@ TEST_F(nix_api_expr_test, nix_value_init) // r = f two; - Value * r = nix_alloc_value(ctx, state); + nix_value * r = nix_alloc_value(ctx, state); nix_init_apply(ctx, r, f, two); assert_ctx_ok(); @@ -297,11 +307,11 @@ TEST_F(nix_api_expr_test, nix_value_init) TEST_F(nix_api_expr_test, nix_value_init_apply_error) { - Value * some_string = nix_alloc_value(ctx, state); + nix_value * some_string = nix_alloc_value(ctx, state); nix_init_string(ctx, some_string, "some string"); assert_ctx_ok(); - Value * v = nix_alloc_value(ctx, state); + nix_value * v = nix_alloc_value(ctx, state); nix_init_apply(ctx, v, some_string, some_string); assert_ctx_ok(); @@ -326,7 +336,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) // r = f e // r should not throw an exception, because e is not evaluated - Value * f = nix_alloc_value(ctx, state); + nix_value * f = nix_alloc_value(ctx, state); nix_expr_eval_from_string( ctx, state, @@ -337,9 +347,9 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) f); assert_ctx_ok(); - Value * e = nix_alloc_value(ctx, state); + nix_value * e = nix_alloc_value(ctx, state); { - Value * g = nix_alloc_value(ctx, state); + nix_value * g = nix_alloc_value(ctx, state); nix_expr_eval_from_string( ctx, state, @@ -355,7 +365,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) nix_gc_decref(ctx, g); } - Value * r = nix_alloc_value(ctx, state); + nix_value * r = nix_alloc_value(ctx, state); nix_init_apply(ctx, r, f, e); assert_ctx_ok(); @@ -367,7 +377,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) ASSERT_EQ(1, n); // nix_get_attr_byname isn't lazy (it could have been) so it will throw the exception - Value * foo = nix_get_attr_byname(ctx, r, state, "foo"); + nix_value * foo = nix_get_attr_byname(ctx, r, state, "foo"); ASSERT_EQ(nullptr, foo); ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("error message for test case nix_value_init_apply_lazy_arg")); @@ -378,7 +388,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) TEST_F(nix_api_expr_test, nix_copy_value) { - Value * source = nix_alloc_value(ctx, state); + nix_value * source = nix_alloc_value(ctx, state); nix_init_int(ctx, source, 42); nix_copy_value(ctx, value, source); diff --git a/tests/unit/libexpr/package.nix b/tests/unit/libexpr/package.nix new file mode 100644 index 000000000..6b7e12c4a --- /dev/null +++ b/tests/unit/libexpr/package.nix @@ -0,0 +1,99 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-expr +, nix-expr-c +, nix-expr-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, resolvePath +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-expr-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-expr + nix-expr-c + nix-expr-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + } '' + PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + nix-expr-tests + touch $out + ''; + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libfetchers/.version b/tests/unit/libfetchers/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libfetchers/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libfetchers/build-utils-meson b/tests/unit/libfetchers/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libfetchers/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk index f4e56a501..30aa142a5 100644 --- a/tests/unit/libfetchers/local.mk +++ b/tests/unit/libfetchers/local.mk @@ -4,7 +4,7 @@ programs += libfetchers-tests libfetchers-tests_NAME = libnixfetchers-tests -libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libfetchers-tests.xml libfetchers-tests_DIR := $(d) diff --git a/tests/unit/libfetchers/meson.build b/tests/unit/libfetchers/meson.build new file mode 100644 index 000000000..d2de93829 --- /dev/null +++ b/tests/unit/libfetchers/meson.build @@ -0,0 +1,71 @@ +project('nix-fetchers-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-store-test-support'), + dependency('nix-fetchers'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'public-key.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libfetchers/package.nix b/tests/unit/libfetchers/package.nix new file mode 100644 index 000000000..9522f9639 --- /dev/null +++ b/tests/unit/libfetchers/package.nix @@ -0,0 +1,97 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-fetchers +, nix-store-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, resolvePath +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-fetchers-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-fetchers + nix-store-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + } '' + PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + nix-fetchers-tests + touch $out + ''; + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libflake/.version b/tests/unit/libflake/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libflake/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libflake/build-utils-meson b/tests/unit/libflake/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libflake/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libflake/data/.gitkeep b/tests/unit/libflake/data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/libexpr/flake/flakeref.cc b/tests/unit/libflake/flakeref.cc similarity index 100% rename from tests/unit/libexpr/flake/flakeref.cc rename to tests/unit/libflake/flakeref.cc diff --git a/tests/unit/libflake/local.mk b/tests/unit/libflake/local.mk new file mode 100644 index 000000000..590bcf7c0 --- /dev/null +++ b/tests/unit/libflake/local.mk @@ -0,0 +1,43 @@ +check: libflake-tests_RUN + +programs += libflake-tests + +libflake-tests_NAME := libnixflake-tests + +libflake-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libflake-tests.xml + +libflake-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libflake-tests_INSTALL_DIR := $(checkbindir) +else + libflake-tests_INSTALL_DIR := +endif + +libflake-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ + $(wildcard $(d)/flake/*.cc) + +libflake-tests_EXTRA_INCLUDES = \ + -I tests/unit/libflake-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libflake) \ + $(INCLUDE_libexpr) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libutil) \ + +libflake-tests_CXXFLAGS += $(libflake-tests_EXTRA_INCLUDES) + +libflake-tests_LIBS = \ + libexpr-test-support libstore-test-support libutil-test-support \ + libflake libexpr libfetchers libstore libutil + +libflake-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libflake-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libflake/meson.build b/tests/unit/libflake/meson.build new file mode 100644 index 000000000..2d6bbca0f --- /dev/null +++ b/tests/unit/libflake/meson.build @@ -0,0 +1,72 @@ +project('nix-flake-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-expr-test-support'), + dependency('nix-flake'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'flakeref.cc', + 'url-name.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libflake/package.nix b/tests/unit/libflake/package.nix new file mode 100644 index 000000000..859bc49d0 --- /dev/null +++ b/tests/unit/libflake/package.nix @@ -0,0 +1,97 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-flake +, nix-expr-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, resolvePath +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-flake-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-flake + nix-expr-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + } '' + PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + nix-flake-tests + touch $out + ''; + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libexpr/flake/url-name.cc b/tests/unit/libflake/url-name.cc similarity index 100% rename from tests/unit/libexpr/flake/url-name.cc rename to tests/unit/libflake/url-name.cc diff --git a/tests/unit/libstore-support/.version b/tests/unit/libstore-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libstore-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libstore-support/build-utils-meson b/tests/unit/libstore-support/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libstore-support/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libstore-support/meson.build b/tests/unit/libstore-support/meson.build new file mode 100644 index 000000000..ddb067c1b --- /dev/null +++ b/tests/unit/libstore-support/meson.build @@ -0,0 +1,76 @@ +project('nix-store-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-util-test-support'), + dependency('nix-store'), +] +subdir('build-utils-meson/subprojects') + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'tests/derived-path.cc', + 'tests/outputs-spec.cc', + 'tests/path.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/derived-path.hh', + 'tests/libstore.hh', + 'tests/nix_api_store.hh', + 'tests/outputs-spec.hh', + 'tests/path.hh', + 'tests/protocol.hh', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nix-store-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/tests/unit/libstore-support/package.nix b/tests/unit/libstore-support/package.nix new file mode 100644 index 000000000..cb15cdd5f --- /dev/null +++ b/tests/unit/libstore-support/package.nix @@ -0,0 +1,81 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-test-support +, nix-store + +, rapidcheck + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store-test-support"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-test-support + nix-store + rapidcheck + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libstore-support/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh index 267188224..84be52c23 100644 --- a/tests/unit/libstore-support/tests/libstore.hh +++ b/tests/unit/libstore-support/tests/libstore.hh @@ -8,19 +8,27 @@ namespace nix { -class LibStoreTest : public virtual ::testing::Test { - public: - static void SetUpTestSuite() { - initLibStore(false); - } +class LibStoreTest : public virtual ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } - protected: - LibStoreTest() - : store(openStore("dummy://")) - { } +protected: + LibStoreTest() + : store(openStore({ + .variant = + StoreReference::Specified{ + .scheme = "dummy", + }, + .params = {}, + })) + { + } - ref store; + ref store; }; - } /* namespace nix */ diff --git a/tests/unit/libstore/.version b/tests/unit/libstore/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libstore/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libstore/build-utils-meson b/tests/unit/libstore/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libstore/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc index d23805fc3..c8f6dd002 100644 --- a/tests/unit/libstore/common-protocol.cc +++ b/tests/unit/libstore/common-protocol.cc @@ -83,15 +83,15 @@ CHARACTERIZATION_TEST( "content-address", (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -178,7 +178,7 @@ CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, diff --git a/tests/unit/libstore/content-address.cc b/tests/unit/libstore/content-address.cc index cc1c7fcc6..72eb84fec 100644 --- a/tests/unit/libstore/content-address.cc +++ b/tests/unit/libstore/content-address.cc @@ -9,11 +9,11 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(ContentAddressMethod, testRoundTripPrintParse_1) { - for (const ContentAddressMethod & cam : { - ContentAddressMethod { TextIngestionMethod {} }, - ContentAddressMethod { FileIngestionMethod::Flat }, - ContentAddressMethod { FileIngestionMethod::Recursive }, - ContentAddressMethod { FileIngestionMethod::Git }, + for (ContentAddressMethod cam : { + ContentAddressMethod::Raw::Text, + ContentAddressMethod::Raw::Flat, + ContentAddressMethod::Raw::NixArchive, + ContentAddressMethod::Raw::Git, }) { EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); } diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv new file mode 120000 index 000000000..353090ad8 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json new file mode 100644 index 000000000..d58e7d5b5 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json @@ -0,0 +1,22 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "builder": "/bin/bash", + "name": "advanced-attributes-defaults", + "out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults", + "system": "my-system" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-defaults", + "outputs": { + "out": { + "path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..11713da12 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json new file mode 100644 index 000000000..473d006ac --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json @@ -0,0 +1,24 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}", + "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev", + "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs-defaults", + "outputs": { + "dev": { + "path": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" + }, + "out": { + "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..962f8ea3f --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json new file mode 100644 index 000000000..324428124 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json @@ -0,0 +1,41 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin", + "dev": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev", + "out": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + }, + "inputDrvs": { + "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + }, + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs", + "outputs": { + "bin": { + "path": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin" + }, + "dev": { + "path": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev" + }, + "out": { + "path": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes.drv b/tests/unit/libstore/data/derivation/advanced-attributes.drv new file mode 120000 index 000000000..2a53a05ca --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes.drv \ No newline at end of file diff --git a/tests/unit/libstore/derivation-advanced-attrs.cc b/tests/unit/libstore/derivation-advanced-attrs.cc new file mode 100644 index 000000000..26cf947a8 --- /dev/null +++ b/tests/unit/libstore/derivation-advanced-attrs.cc @@ -0,0 +1,234 @@ +#include +#include +#include + +#include "experimental-features.hh" +#include "derivations.hh" + +#include "tests/libstore.hh" +#include "tests/characterization.hh" +#include "parsed-derivations.hh" +#include "types.hh" + +namespace nix { + +using nlohmann::json; + +class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/derivation"; + +public: + Path goldenMaster(std::string_view testStem) const override + { + return unitTestData + "/" + testStem; + } +}; + +#define TEST_ATERM_JSON(STEM, NAME) \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_json) \ + { \ + readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*store, std::move(aterm), NAME); \ + Derivation got = Derivation::fromJSON(*store, encoded); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_to_json) \ + { \ + writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + return parseDerivation(*store, std::move(aterm), NAME).toJSON(*store); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_aterm) \ + { \ + readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(*store, json); \ + auto got = parseDerivation(*store, std::move(encoded), NAME); \ + EXPECT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + /* No corresponding write test, because we need to read the drv to write the json file */ + +TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs, "advanced-attributes-structured-attrs-defaults"); + +#undef TEST_ATERM_JSON + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) +{ + readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) +{ + readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedReferences"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedRequisites"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedReferences"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedRequisites"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs_defaults) +{ + readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_FALSE(outputChecks_); + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs) +{ + readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_TRUE(outputChecks_); + auto & outputChecks = *outputChecks_; + + { + auto output_ = get(outputChecks, "out"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "allowedReferences")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + get(output, "allowedRequisites")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + } + + { + auto output_ = get(outputChecks, "bin"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "disallowedReferences")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + get(output, "disallowedRequisites")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + } + + { + auto output_ = get(outputChecks, "dev"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ(get(output, "maxSize")->get(), 789); + EXPECT_EQ(get(output, "maxClosureSize")->get(), 5909); + } + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +} diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index 7a4b1403a..71979f885 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -108,7 +108,7 @@ TEST_JSON(DerivationTest, inputAddressed, TEST_JSON(DerivationTest, caFixedFlat, (DerivationOutput::CAFixed { .ca = { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -117,7 +117,7 @@ TEST_JSON(DerivationTest, caFixedFlat, TEST_JSON(DerivationTest, caFixedNAR, (DerivationOutput::CAFixed { .ca = { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -126,6 +126,7 @@ TEST_JSON(DerivationTest, caFixedNAR, TEST_JSON(DynDerivationTest, caFixedText, (DerivationOutput::CAFixed { .ca = { + .method = ContentAddressMethod::Raw::Text, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -133,7 +134,7 @@ TEST_JSON(DynDerivationTest, caFixedText, TEST_JSON(CaDerivationTest, caFloating, (DerivationOutput::CAFloating { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") @@ -144,7 +145,7 @@ TEST_JSON(DerivationTest, deferred, TEST_JSON(ImpureDerivationTest, impure, (DerivationOutput::Impure { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk index 0af1d2622..8d3d6b0af 100644 --- a/tests/unit/libstore/local.mk +++ b/tests/unit/libstore/local.mk @@ -4,7 +4,7 @@ programs += libstore-tests libstore-tests_NAME = libnixstore-tests -libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libstore-tests.xml libstore-tests_DIR := $(d) diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build new file mode 100644 index 000000000..90e7d3047 --- /dev/null +++ b/tests/unit/libstore/meson.build @@ -0,0 +1,95 @@ +project('nix-store-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-store'), + dependency('nix-store-c'), + dependency('nix-store-test-support'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/export-all-symbols') + +sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') +deps_private += sqlite + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +gtest = dependency('gmock') +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-util.h', + '-include', 'config-store.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'common-protocol.cc', + 'content-address.cc', + 'derivation-advanced-attrs.cc', + 'derivation.cc', + 'derived-path.cc', + 'downstream-placeholder.cc', + 'machines.cc', + 'nar-info-disk-cache.cc', + 'nar-info.cc', + 'nix_api_store.cc', + 'outputs-spec.cc', + 'path-info.cc', + 'path.cc', + 'references.cc', + 'serve-protocol.cc', + 'store-reference.cc', + 'worker-protocol.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc index bd10602e7..a6cb62de4 100644 --- a/tests/unit/libstore/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -25,7 +25,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { diff --git a/tests/unit/libstore/package.nix b/tests/unit/libstore/package.nix new file mode 100644 index 000000000..efffd0063 --- /dev/null +++ b/tests/unit/libstore/package.nix @@ -0,0 +1,110 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store +, nix-store-c +, nix-store-test-support +, sqlite + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, filesetToSource +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-store + nix-store-c + nix-store-test-support + sqlite + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = let + # Some data is shared with the functional tests: they create it, + # we consume it. + data = filesetToSource { + root = ../..; + fileset = lib.fileset.unions [ + ./data + ../../functional/derivation + ]; + }; + in runCommand "${finalAttrs.pname}-run" {} '' + PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + export _NIX_TEST_UNIT_DATA=${data + "/unit/libstore/data"} + nix-store-tests + touch $out + ''; + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 06c662b74..7637cb366 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -32,7 +32,7 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { diff --git a/tests/unit/libstore/path.cc b/tests/unit/libstore/path.cc index 213b6e95f..c4c055abf 100644 --- a/tests/unit/libstore/path.cc +++ b/tests/unit/libstore/path.cc @@ -26,10 +26,19 @@ static std::regex nameRegex { std::string { nameRegexStr } }; TEST_F(StorePathTest, bad_ ## NAME) { \ std::string_view str = \ STORE_DIR HASH_PART "-" STR; \ - ASSERT_THROW( \ - store->parseStorePath(str), \ - BadStorePath); \ + /* ASSERT_THROW generates a duplicate goto label */ \ + /* A lambda isolates those labels. */ \ + [&](){ \ + ASSERT_THROW( \ + store->parseStorePath(str), \ + BadStorePath); \ + }(); \ std::string name { STR }; \ + [&](){ \ + ASSERT_THROW( \ + nix::checkName(name), \ + BadStorePathName); \ + }(); \ EXPECT_FALSE(std::regex_match(name, nameRegex)); \ } @@ -54,6 +63,7 @@ TEST_DONT_PARSE(dot_dash_a, ".-a") STORE_DIR HASH_PART "-" STR; \ auto p = store->parseStorePath(str); \ std::string name { p.name() }; \ + EXPECT_EQ(p.name(), STR); \ EXPECT_TRUE(std::regex_match(name, nameRegex)); \ } diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index ebf0c52b0..2505c5a9a 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -55,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST( defaultVersion, (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -280,7 +280,7 @@ VERSIONED_CHARACTERIZATION_TEST( *LibStoreTest::store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { @@ -398,7 +398,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index 5907ea5a4..c15120010 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -1,4 +1,5 @@ #include +#include #include #include @@ -55,15 +56,15 @@ VERSIONED_CHARACTERIZATION_TEST( defaultVersion, (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -511,7 +512,7 @@ VERSIONED_CHARACTERIZATION_TEST( *LibStoreTest::store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { @@ -597,7 +598,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, diff --git a/tests/unit/libutil-support/.version b/tests/unit/libutil-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libutil-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libutil-support/build-utils-meson b/tests/unit/libutil-support/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libutil-support/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libutil-support/meson.build b/tests/unit/libutil-support/meson.build new file mode 100644 index 000000000..7d0e9c2fc --- /dev/null +++ b/tests/unit/libutil-support/meson.build @@ -0,0 +1,70 @@ +project('nix-util-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), +] +subdir('build-utils-meson/subprojects') + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'tests/hash.cc', + 'tests/string_callback.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/characterization.hh', + 'tests/hash.hh', + 'tests/nix_api_util.hh', + 'tests/string_callback.hh', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nix-util-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix new file mode 100644 index 000000000..fdecdec72 --- /dev/null +++ b/tests/unit/libutil-support/package.nix @@ -0,0 +1,79 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util + +, rapidcheck + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-test-support"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util + rapidcheck + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index c2f686dbf..19ba824ac 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "environment-variables.hh" +#include "file-system.hh" namespace nix { diff --git a/tests/unit/libutil/.version b/tests/unit/libutil/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libutil/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libutil/build-utils-meson b/tests/unit/libutil/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libutil/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libutil/file-content-address.cc b/tests/unit/libutil/file-content-address.cc index 294e39806..27d926a87 100644 --- a/tests/unit/libutil/file-content-address.cc +++ b/tests/unit/libutil/file-content-address.cc @@ -11,7 +11,7 @@ namespace nix { TEST(FileSerialisationMethod, testRoundTripPrintParse_1) { for (const FileSerialisationMethod fim : { FileSerialisationMethod::Flat, - FileSerialisationMethod::Recursive, + FileSerialisationMethod::NixArchive, }) { EXPECT_EQ(parseFileSerialisationMethod(renderFileSerialisationMethod(fim)), fim); } @@ -37,7 +37,7 @@ TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) { TEST(FileIngestionMethod, testRoundTripPrintParse_1) { for (const FileIngestionMethod fim : { FileIngestionMethod::Flat, - FileIngestionMethod::Recursive, + FileIngestionMethod::NixArchive, FileIngestionMethod::Git, }) { EXPECT_EQ(parseFileIngestionMethod(renderFileIngestionMethod(fim)), fim); diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index ff934c117..a0125d023 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -67,7 +67,7 @@ TEST_F(GitTest, blob_read) { StringSink out; RegularFileSink out2 { out }; ASSERT_EQ(parseObjectType(in, mockXpSettings), ObjectType::Blob); - parseBlob(out2, "", in, BlobMode::Regular, mockXpSettings); + parseBlob(out2, CanonPath::root, in, BlobMode::Regular, mockXpSettings); auto expected = readFile(goldenMaster("hello-world.bin")); @@ -132,8 +132,8 @@ TEST_F(GitTest, tree_read) { NullFileSystemObjectSink out; Tree got; ASSERT_EQ(parseObjectType(in, mockXpSettings), ObjectType::Tree); - parseTree(out, "", in, [&](auto & name, auto entry) { - auto name2 = name; + parseTree(out, CanonPath::root, in, [&](auto & name, auto entry) { + auto name2 = std::string{name.rel()}; if (entry.mode == Mode::Directory) name2 += '/'; got.insert_or_assign(name2, std::move(entry)); @@ -210,14 +210,14 @@ TEST_F(GitTest, both_roundrip) { MemorySink sinkFiles2 { *files2 }; - std::function mkSinkHook; + std::function mkSinkHook; mkSinkHook = [&](auto prefix, auto & hash, auto blobMode) { StringSource in { cas[hash] }; parse( sinkFiles2, prefix, in, blobMode, - [&](const Path & name, const auto & entry) { + [&](const CanonPath & name, const auto & entry) { mkSinkHook( - prefix + "/" + name, + prefix / name, entry.hash, // N.B. this cast would not be acceptable in real // code, because it would make an assert reachable, @@ -227,7 +227,7 @@ TEST_F(GitTest, both_roundrip) { mockXpSettings); }; - mkSinkHook("", root.hash, BlobMode::Regular); + mkSinkHook(CanonPath::root, root.hash, BlobMode::Regular); ASSERT_EQ(*files, *files2); } diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk index b9bddc24d..404f35cf1 100644 --- a/tests/unit/libutil/local.mk +++ b/tests/unit/libutil/local.mk @@ -4,7 +4,7 @@ programs += libutil-tests libutil-tests_NAME = libnixutil-tests -libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libutil-tests.xml libutil-tests_DIR := $(d) diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build new file mode 100644 index 000000000..4f055cabd --- /dev/null +++ b/tests/unit/libutil/meson.build @@ -0,0 +1,91 @@ +project('nix-util-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-util-c'), + dependency('nix-util-test-support'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-util.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'args.cc', + 'canon-path.cc', + 'chunked-vector.cc', + 'closure.cc', + 'compression.cc', + 'config.cc', + 'file-content-address.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'lru-cache.cc', + 'nix_api_util.cc', + 'pool.cc', + 'references.cc', + 'spawn.cc', + 'suggestions.cc', + 'tests.cc', + 'url.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libutil/nix_api_util.cc b/tests/unit/libutil/nix_api_util.cc index d2999f55b..2b7e38225 100644 --- a/tests/unit/libutil/nix_api_util.cc +++ b/tests/unit/libutil/nix_api_util.cc @@ -1,4 +1,4 @@ -#include "config.hh" +#include "config-global.hh" #include "args.hh" #include "nix_api_util.h" #include "nix_api_util_internal.h" diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix new file mode 100644 index 000000000..ad5ff7d1e --- /dev/null +++ b/tests/unit/libutil/package.nix @@ -0,0 +1,98 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-util-c +, nix-util-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-util + nix-util-c + nix-util-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + } '' + PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + export _NIX_TEST_UNIT_DATA=${./data} + nix-util-tests + touch $out + ''; + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libutil/spawn.cc b/tests/unit/libutil/spawn.cc new file mode 100644 index 000000000..c617acae0 --- /dev/null +++ b/tests/unit/libutil/spawn.cc @@ -0,0 +1,36 @@ +#include + +#include "processes.hh" + +namespace nix { + +#ifdef _WIN32 +TEST(SpawnTest, spawnEcho) +{ + auto output = runProgram(RunOptions{.program = "cmd.exe", .args = {"/C", "echo", "hello world"}}); + ASSERT_EQ(output.first, 0); + ASSERT_EQ(output.second, "\"hello world\"\r\n"); +} + +std::string windowsEscape(const std::string & str, bool cmd); + +TEST(SpawnTest, windowsEscape) +{ + auto empty = windowsEscape("", false); + ASSERT_EQ(empty, R"("")"); + // There's no quotes in this argument so the input should equal the output + auto backslashStr = R"(\\\\)"; + auto backslashes = windowsEscape(backslashStr, false); + ASSERT_EQ(backslashes, backslashStr); + + auto nestedQuotes = windowsEscape(R"(he said: "hello there")", false); + ASSERT_EQ(nestedQuotes, R"("he said: \"hello there\"")"); + + auto middleQuote = windowsEscape(R"( \\\" )", false); + ASSERT_EQ(middleQuote, R"(" \\\\\\\" ")"); + + auto space = windowsEscape("hello world", false); + ASSERT_EQ(space, R"("hello world")"); +} +#endif +}