From 372d5a441e1a6fe61740609ac622f53dc77229a3 Mon Sep 17 00:00:00 2001 From: Kirill Radzikhovskyy Date: Mon, 10 Jun 2024 11:17:41 +1000 Subject: [PATCH 01/67] darwin: allow ipc-sysv* in sandbox --- src/libstore/unix/build/sandbox-defaults.sb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/unix/build/sandbox-defaults.sb b/src/libstore/unix/build/sandbox-defaults.sb index 2ad5fb616..6da01b735 100644 --- a/src/libstore/unix/build/sandbox-defaults.sb +++ b/src/libstore/unix/build/sandbox-defaults.sb @@ -17,6 +17,9 @@ R""( ; Allow POSIX semaphores and shared memory. (allow ipc-posix*) +; Allow SYSV semaphores and shared memory. +(allow ipc-sysv*) + ; Allow socket creation. (allow system-socket) From b130d2f2e335994cad555a77d242804b65a87062 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 11 Jun 2024 17:52:33 +0200 Subject: [PATCH 02/67] add more context on the README (#9871) the thesis is still the defining document with all the motivation and explanations. adding it here for greater visibility. also more emphasis and clarity around the community aspect. the hydra build job seems a bit arbitrary right there. may be better for the contributing guide. --- README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e1cace3b4..931a60bba 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). -## Building And Developing +## Building and developing See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to set up a development environment and build Nix from source. @@ -22,12 +22,17 @@ See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacki Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. -## Additional Resources +## Additional resources -- [Nix manual](https://nixos.org/nix/manual) -- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) -- [NixOS Discourse](https://discourse.nixos.org/) -- [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) +Nix was created by Eelco Dolstra and developed as the subject of his PhD thesis [The Purely Functional Software Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf), published 2006. +Today, a world-wide developer community contributes to Nix and the ecosystem that has grown around it. + +- [The Nix, Nixpkgs, NixOS Community on nixos.org](https://nixos.org/) +- [Official documentation on nix.dev](https://nix.dev) +- [Nixpkgs](https://github.com/NixOS/nixpkgs) is [the largest, most up-to-date free software repository in the world](https://repology.org/repositories/graphs) +- [NixOS](https://github.com/NixOS/nixpkgs/tree/master/nixos) is a Linux distribution that can be configured fully declaratively +- [Discourse](https://discourse.nixos.org/) +- [Matrix](https://matrix.to/#/#nix:nixos.org) ## License From 258e2a32ec5f9661d79c0016e2510c0057f9ddd3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 12 Jun 2024 14:57:40 +0200 Subject: [PATCH 03/67] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index e9763f6bf..ad2261920 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.23.0 +2.24.0 From e74ce01b7f0920026502824922cfd71fca5eb525 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jun 2024 10:50:07 -0400 Subject: [PATCH 04/67] Fix precompiled headers building with clang Since 24.05 (I think) we need to pass `-c` or Clang thinks we want to compile *both* a final executable and precompiled header file, and complains that we cannot use `-o` with multiple outputs. `-c` seems fine with GCC too, so I just put it in there conditionally. --- mk/precompiled-headers.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) From 7c2981fc559e93131c845164907cec6cae11ce48 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jun 2024 10:02:20 -0400 Subject: [PATCH 05/67] Fix FreeBSD build This restores some CPP'd code that was added in c18911602eb4260d59acf8c17f1c3b4c7fcf7cee and accidentally lost in 2477e4e3b84b23b091befa1869a1fc6186fe74dc. Co-authored-by: Eelco Dolstra --- configure.ac | 8 +++++++ src/libexpr/eval.cc | 29 +++++++++++++++++++------- src/libstore/build/derivation-goal.cc | 6 +++++- tests/unit/libstore/worker-protocol.cc | 1 + 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 90a6d45d5..8211bec0b 100644 --- a/configure.ac +++ b/configure.ac @@ -369,6 +369,14 @@ if test "$gc" = yes; then PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS" AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.]) + + # See `fixupBoehmStackPointer`, for the integration between Boehm GC + # and Boost coroutines. + old_CFLAGS="$CFLAGS" + # Temporary set `-pthread` just for the next check + CFLAGS="$CFLAGS -pthread" + AC_CHECK_FUNCS([pthread_attr_get_np pthread_getattr_np]) + CFLAGS="$old_CFLAGS" fi AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f441977a5..42b26834c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -48,6 +48,9 @@ #define GC_INCLUDE_NEW #include +#if __FreeBSD__ +# include +#endif #include #include @@ -264,30 +267,42 @@ static BoehmGCStackAllocator boehmGCStackAllocator; * However, the implementation is quite lean, and usually we don't have active * coroutines during evaluation, so this is acceptable. */ -void fixupBoehmStackPointer(void ** sp_ptr, void * pthread_id) { +void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) { void *& sp = *sp_ptr; + auto pthread_id = reinterpret_cast(_pthread_id); pthread_attr_t pattr; size_t osStackSize; void * osStackLow; void * osStackBase; - #ifdef __APPLE__ - osStackSize = pthread_get_stacksize_np((pthread_t)pthread_id); - osStackLow = pthread_get_stackaddr_np((pthread_t)pthread_id); - #else +# ifdef __APPLE__ + osStackSize = pthread_get_stacksize_np(pthread_id); + osStackLow = pthread_get_stackaddr_np(pthread_id); +# else if (pthread_attr_init(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); } - if (pthread_getattr_np((pthread_t)pthread_id, &pattr)) { +# ifdef HAVE_PTHREAD_GETATTR_NP + if (pthread_getattr_np(pthread_id, &pattr)) { throw Error("fixupBoehmStackPointer: pthread_getattr_np failed"); } +# elif HAVE_PTHREAD_ATTR_GET_NP + if (!pthread_attr_init(&pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); + } + if (!pthread_attr_get_np(pthread_id, &pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_get_np failed"); + } +# else +# error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`" +# endif if (pthread_attr_getstack(&pattr, &osStackLow, &osStackSize)) { throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); } if (pthread_attr_destroy(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); } - #endif +# endif osStackBase = (char *)osStackLow + osStackSize; // NOTE: We assume the stack grows down, as it does on all architectures we support. // Architectures that grow the stack up are rare. diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 4226fb61a..e72ef4cf9 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -25,6 +25,10 @@ #include #include +#ifndef _WIN32 // TODO abstract over proc exit status +# include +#endif + #include namespace nix { @@ -1033,7 +1037,7 @@ void DerivationGoal::buildDone() BuildResult::Status st = BuildResult::MiscFailure; -#ifndef _WIN32 +#ifndef _WIN32 // TODO abstract over proc exit status if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) st = BuildResult::TimedOut; diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index 5907ea5a4..70e03a8ab 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -1,4 +1,5 @@ #include +#include #include #include From 5b53d8fec3149937d44b2302d51967aa95253d7a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jun 2024 10:17:39 -0400 Subject: [PATCH 06/67] Factor out GC initialization code This is not really part of the evaluator: it is just an integration between Boehm GC and Boost coroutines usable for any purpose. The evaluator (merely) optionally uses it. --- src/libexpr/eval-gc.cc | 226 +++++++++++++++++++++++++++++++++++++++++ src/libexpr/eval-gc.hh | 16 +++ src/libexpr/eval.cc | 210 ++------------------------------------ src/libexpr/eval.hh | 7 +- 4 files changed, 249 insertions(+), 210 deletions(-) create mode 100644 src/libexpr/eval-gc.cc create mode 100644 src/libexpr/eval-gc.hh diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc new file mode 100644 index 000000000..baf9df332 --- /dev/null +++ b/src/libexpr/eval-gc.cc @@ -0,0 +1,226 @@ +#include "error.hh" +#include "environment-variables.hh" +#include "serialise.hh" +#include "eval-gc.hh" + +#if HAVE_BOEHMGC + +# include +# if __FreeBSD__ +# include +# endif + +# include +# include +# include + +# include +# include +# include + +#endif + +namespace nix { + +#if HAVE_BOEHMGC +/* Called when the Boehm GC runs out of memory. */ +static void * oomHandler(size_t requested) +{ + /* Convert this to a proper C++ exception. */ + throw std::bad_alloc(); +} + +class BoehmGCStackAllocator : public StackAllocator +{ + boost::coroutines2::protected_fixedsize_stack stack{ + // We allocate 8 MB, the default max stack size on NixOS. + // A smaller stack might be quicker to allocate but reduces the stack + // depth available for source filter expressions etc. + std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024))}; + + // This is specific to boost::coroutines2::protected_fixedsize_stack. + // The stack protection page is included in sctx.size, so we have to + // subtract one page size from the stack size. + std::size_t pfss_usable_stack_size(boost::context::stack_context & sctx) + { + return sctx.size - boost::context::stack_traits::page_size(); + } + +public: + boost::context::stack_context allocate() override + { + auto sctx = stack.allocate(); + + // Stacks generally start at a high address and grow to lower addresses. + // Architectures that do the opposite are rare; in fact so rare that + // boost_routine does not implement it. + // So we subtract the stack size. + GC_add_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); + return sctx; + } + + void deallocate(boost::context::stack_context sctx) override + { + GC_remove_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); + stack.deallocate(sctx); + } +}; + +static BoehmGCStackAllocator boehmGCStackAllocator; + +/** + * When a thread goes into a coroutine, we lose its original sp until + * control flow returns to the thread. + * While in the coroutine, the sp points outside the thread stack, + * so we can detect this and push the entire thread stack instead, + * as an approximation. + * The coroutine's stack is covered by `BoehmGCStackAllocator`. + * This is not an optimal solution, because the garbage is scanned when a + * coroutine is active, for both the coroutine and the original thread stack. + * However, the implementation is quite lean, and usually we don't have active + * coroutines during evaluation, so this is acceptable. + */ +void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) +{ + void *& sp = *sp_ptr; + auto pthread_id = reinterpret_cast(_pthread_id); + pthread_attr_t pattr; + size_t osStackSize; + void * osStackLow; + void * osStackBase; + +# ifdef __APPLE__ + osStackSize = pthread_get_stacksize_np(pthread_id); + osStackLow = pthread_get_stackaddr_np(pthread_id); +# else + if (pthread_attr_init(&pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); + } +# ifdef HAVE_PTHREAD_GETATTR_NP + if (pthread_getattr_np(pthread_id, &pattr)) { + throw Error("fixupBoehmStackPointer: pthread_getattr_np failed"); + } +# elif HAVE_PTHREAD_ATTR_GET_NP + if (!pthread_attr_init(&pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); + } + if (!pthread_attr_get_np(pthread_id, &pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_get_np failed"); + } +# else +# error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`" +# endif + if (pthread_attr_getstack(&pattr, &osStackLow, &osStackSize)) { + throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); + } + if (pthread_attr_destroy(&pattr)) { + throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); + } +# endif + osStackBase = (char *) osStackLow + osStackSize; + // NOTE: We assume the stack grows down, as it does on all architectures we support. + // Architectures that grow the stack up are rare. + if (sp >= osStackBase || sp < osStackLow) { // lo is outside the os stack + sp = osStackBase; + } +} + +/* Disable GC while this object lives. Used by CoroutineContext. + * + * Boehm keeps a count of GC_disable() and GC_enable() calls, + * and only enables GC when the count matches. + */ +class BoehmDisableGC +{ +public: + BoehmDisableGC() + { + GC_disable(); + }; + ~BoehmDisableGC() + { + GC_enable(); + }; +}; + +static inline void initGCReal() +{ + /* Initialise the Boehm garbage collector. */ + + /* Don't look for interior pointers. This reduces the odds of + misdetection a bit. */ + GC_set_all_interior_pointers(0); + + /* We don't have any roots in data segments, so don't scan from + there. */ + GC_set_no_dls(1); + + GC_INIT(); + + GC_set_oom_fn(oomHandler); + + StackAllocator::defaultAllocator = &boehmGCStackAllocator; + +// TODO: Remove __APPLE__ condition. +// Comment suggests an implementation that works on darwin and windows +// https://github.com/ivmai/bdwgc/issues/362#issuecomment-1936672196 +# if GC_VERSION_MAJOR >= 8 && GC_VERSION_MINOR >= 2 && GC_VERSION_MICRO >= 4 && !defined(__APPLE__) + GC_set_sp_corrector(&fixupBoehmStackPointer); + + if (!GC_get_sp_corrector()) { + printTalkative("BoehmGC on this platform does not support sp_corrector; will disable GC inside coroutines"); + /* Used to disable GC when entering coroutines on macOS */ + create_coro_gc_hook = []() -> std::shared_ptr { return std::make_shared(); }; + } +# else +# warning \ + "BoehmGC version does not support GC while coroutine exists. GC will be disabled inside coroutines. Consider updating bdw-gc to 8.2.4 or later." +# endif + + /* Set the initial heap size to something fairly big (25% of + physical RAM, up to a maximum of 384 MiB) so that in most cases + we don't need to garbage collect at all. (Collection has a + fairly significant overhead.) The heap size can be overridden + through libgc's GC_INITIAL_HEAP_SIZE environment variable. We + should probably also provide a nix.conf setting for this. Note + that GC_expand_hp() causes a lot of virtual, but not physical + (resident) memory to be allocated. This might be a problem on + systems that don't overcommit. */ + if (!getEnv("GC_INITIAL_HEAP_SIZE")) { + size_t size = 32 * 1024 * 1024; +# if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) + size_t maxSize = 384 * 1024 * 1024; + long pageSize = sysconf(_SC_PAGESIZE); + long pages = sysconf(_SC_PHYS_PAGES); + if (pageSize != -1) + size = (pageSize * pages) / 4; // 25% of RAM + if (size > maxSize) + size = maxSize; +# endif + debug("setting initial heap size to %1% bytes", size); + GC_expand_hp(size); + } +} + +#endif + +static bool gcInitialised = false; + +void initGC() +{ + if (gcInitialised) + return; + +#if HAVE_BOEHMGC + initGCReal(); +#endif + + gcInitialised = true; +} + +void assertGCInitialized() +{ + assert(gcInitialised); +} + +} diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh new file mode 100644 index 000000000..cd4ea914d --- /dev/null +++ b/src/libexpr/eval-gc.hh @@ -0,0 +1,16 @@ +#pragma once +///@file + +namespace nix { + +/** + * Initialise the Boehm GC, if applicable. + */ +void initGC(); + +/** + * Make sure `initGC` has already been called. + */ +void assertGCInitialized(); + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 42b26834c..301b81a62 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -40,25 +40,16 @@ #include #ifndef _WIN32 // TODO use portable implementation -# include +# include #endif #if HAVE_BOEHMGC -#define GC_INCLUDE_NEW +# define GC_INCLUDE_NEW -#include -#if __FreeBSD__ -# include -#endif - -#include -#include -#include - -#include -#include -#include +# include +# include +# include #endif @@ -211,109 +202,6 @@ bool Value::isTrivial() const } -#if HAVE_BOEHMGC -/* Called when the Boehm GC runs out of memory. */ -static void * oomHandler(size_t requested) -{ - /* Convert this to a proper C++ exception. */ - throw std::bad_alloc(); -} - -class BoehmGCStackAllocator : public StackAllocator { - boost::coroutines2::protected_fixedsize_stack stack { - // We allocate 8 MB, the default max stack size on NixOS. - // A smaller stack might be quicker to allocate but reduces the stack - // depth available for source filter expressions etc. - std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) - }; - - // This is specific to boost::coroutines2::protected_fixedsize_stack. - // The stack protection page is included in sctx.size, so we have to - // subtract one page size from the stack size. - std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) { - return sctx.size - boost::context::stack_traits::page_size(); - } - - public: - boost::context::stack_context allocate() override { - auto sctx = stack.allocate(); - - // Stacks generally start at a high address and grow to lower addresses. - // Architectures that do the opposite are rare; in fact so rare that - // boost_routine does not implement it. - // So we subtract the stack size. - GC_add_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - return sctx; - } - - void deallocate(boost::context::stack_context sctx) override { - GC_remove_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - stack.deallocate(sctx); - } - -}; - -static BoehmGCStackAllocator boehmGCStackAllocator; - -/** - * When a thread goes into a coroutine, we lose its original sp until - * control flow returns to the thread. - * While in the coroutine, the sp points outside the thread stack, - * so we can detect this and push the entire thread stack instead, - * as an approximation. - * The coroutine's stack is covered by `BoehmGCStackAllocator`. - * This is not an optimal solution, because the garbage is scanned when a - * coroutine is active, for both the coroutine and the original thread stack. - * However, the implementation is quite lean, and usually we don't have active - * coroutines during evaluation, so this is acceptable. - */ -void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) { - void *& sp = *sp_ptr; - auto pthread_id = reinterpret_cast(_pthread_id); - pthread_attr_t pattr; - size_t osStackSize; - void * osStackLow; - void * osStackBase; - -# ifdef __APPLE__ - osStackSize = pthread_get_stacksize_np(pthread_id); - osStackLow = pthread_get_stackaddr_np(pthread_id); -# else - if (pthread_attr_init(&pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); - } -# ifdef HAVE_PTHREAD_GETATTR_NP - if (pthread_getattr_np(pthread_id, &pattr)) { - throw Error("fixupBoehmStackPointer: pthread_getattr_np failed"); - } -# elif HAVE_PTHREAD_ATTR_GET_NP - if (!pthread_attr_init(&pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); - } - if (!pthread_attr_get_np(pthread_id, &pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_get_np failed"); - } -# else -# error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`" -# endif - if (pthread_attr_getstack(&pattr, &osStackLow, &osStackSize)) { - throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); - } - if (pthread_attr_destroy(&pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); - } -# endif - osStackBase = (char *)osStackLow + osStackSize; - // NOTE: We assume the stack grows down, as it does on all architectures we support. - // Architectures that grow the stack up are rare. - if (sp >= osStackBase || sp < osStackLow) { // lo is outside the os stack - sp = osStackBase; - } -} - -#endif - - static Symbol getName(const AttrName & name, EvalState & state, Env & env) { if (name.symbol) { @@ -326,92 +214,6 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } } -#if HAVE_BOEHMGC -/* Disable GC while this object lives. Used by CoroutineContext. - * - * Boehm keeps a count of GC_disable() and GC_enable() calls, - * and only enables GC when the count matches. - */ -class BoehmDisableGC { -public: - BoehmDisableGC() { - GC_disable(); - }; - ~BoehmDisableGC() { - GC_enable(); - }; -}; -#endif - -static bool gcInitialised = false; - -void initGC() -{ - if (gcInitialised) return; - -#if HAVE_BOEHMGC - /* Initialise the Boehm garbage collector. */ - - /* Don't look for interior pointers. This reduces the odds of - misdetection a bit. */ - GC_set_all_interior_pointers(0); - - /* We don't have any roots in data segments, so don't scan from - there. */ - GC_set_no_dls(1); - - GC_INIT(); - - GC_set_oom_fn(oomHandler); - - StackAllocator::defaultAllocator = &boehmGCStackAllocator; - - // TODO: Remove __APPLE__ condition. - // Comment suggests an implementation that works on darwin and windows - // https://github.com/ivmai/bdwgc/issues/362#issuecomment-1936672196 - #if GC_VERSION_MAJOR >= 8 && GC_VERSION_MINOR >= 2 && GC_VERSION_MICRO >= 4 && !defined(__APPLE__) - GC_set_sp_corrector(&fixupBoehmStackPointer); - - if (!GC_get_sp_corrector()) { - printTalkative("BoehmGC on this platform does not support sp_corrector; will disable GC inside coroutines"); - /* Used to disable GC when entering coroutines on macOS */ - create_coro_gc_hook = []() -> std::shared_ptr { - return std::make_shared(); - }; - } - #else - #warning "BoehmGC version does not support GC while coroutine exists. GC will be disabled inside coroutines. Consider updating bdw-gc to 8.2.4 or later." - #endif - - - /* Set the initial heap size to something fairly big (25% of - physical RAM, up to a maximum of 384 MiB) so that in most cases - we don't need to garbage collect at all. (Collection has a - fairly significant overhead.) The heap size can be overridden - through libgc's GC_INITIAL_HEAP_SIZE environment variable. We - should probably also provide a nix.conf setting for this. Note - that GC_expand_hp() causes a lot of virtual, but not physical - (resident) memory to be allocated. This might be a problem on - systems that don't overcommit. */ - if (!getEnv("GC_INITIAL_HEAP_SIZE")) { - size_t size = 32 * 1024 * 1024; -#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) - size_t maxSize = 384 * 1024 * 1024; - long pageSize = sysconf(_SC_PAGESIZE); - long pages = sysconf(_SC_PHYS_PAGES); - if (pageSize != -1) - size = (pageSize * pages) / 4; // 25% of RAM - if (size > maxSize) size = maxSize; -#endif - debug("setting initial heap size to %1% bytes", size); - GC_expand_hp(size); - } - -#endif - - gcInitialised = true; -} - static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( @@ -508,7 +310,7 @@ EvalState::EvalState( countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; - assert(gcInitialised); + assertGCInitialized(); static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index dac763268..0e5e0433e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -3,6 +3,7 @@ #include "attr-set.hh" #include "eval-error.hh" +#include "eval-gc.hh" #include "types.hh" #include "value.hh" #include "nixexpr.hh" @@ -146,12 +147,6 @@ std::string printValue(EvalState & state, Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -/** - * Initialise the Boehm GC, if applicable. - */ -void initGC(); - - struct RegexCache; std::shared_ptr makeRegexCache(); From 7738b295e54e385d6ec54c805362403f0df25e41 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:18:42 -0700 Subject: [PATCH 07/67] housekeeping: shellcheck for tests/functional/bash-profile.sh --- maintainers/flake-module.nix | 1 - tests/functional/bash-profile.sh | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 3006d5e30..261da1766 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -507,7 +507,6 @@ ''^scripts/install-nix-from-closure\.sh$'' ''^scripts/install-systemd-multi-user\.sh$'' ''^src/nix/get-env\.sh$'' - ''^tests/functional/bash-profile\.sh$'' ''^tests/functional/binary-cache-build-remote\.sh$'' ''^tests/functional/binary-cache\.sh$'' ''^tests/functional/brotli\.sh$'' diff --git a/tests/functional/bash-profile.sh b/tests/functional/bash-profile.sh index 6cfa5bd9c..4228d4a20 100755 --- a/tests/functional/bash-profile.sh +++ b/tests/functional/bash-profile.sh @@ -2,10 +2,10 @@ source common.sh -sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > "$TEST_ROOT"/nix-profile.sh user=$(whoami) -rm -rf $TEST_HOME $TEST_ROOT/profile-var -mkdir -p $TEST_HOME +rm -rf "$TEST_HOME" "$TEST_ROOT/profile-var" +mkdir -p "$TEST_HOME" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency From 4a28ba7877f8c35cbf370fc45a1128d25c7ffe5b Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:18:53 -0700 Subject: [PATCH 08/67] housekeeping: shellcheck for tests/functional/binary-cache-build-remote.sh --- tests/functional/binary-cache-build-remote.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh index 0303e9410..4edda85b6 100755 --- a/tests/functional/binary-cache-build-remote.sh +++ b/tests/functional/binary-cache-build-remote.sh @@ -12,7 +12,7 @@ clearCacheCache outPath=$(nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix) # Test that the path exactly exists in the destination store. -nix path-info --store "file://$cacheDir" $outPath +nix path-info --store "file://$cacheDir" "$outPath" # Succeeds without any build capability because no-op nix-build --store "file://$cacheDir" -j0 dependencies.nix From aeed835a2ef8421b01d70d8e44a60fdade01eec4 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:39:01 -0700 Subject: [PATCH 09/67] housekeeping: shellcheck for tests/functional/binary-cache.sh Co-authored-by: Sandro --- tests/functional/binary-cache.sh | 163 ++++++++++++++++--------------- 1 file changed, 85 insertions(+), 78 deletions(-) diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 54a3687ca..23135359b 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -14,9 +14,9 @@ clearStore clearCache outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --to file://$cacheDir $outPath +nix copy --to file://"$cacheDir" "$outPath" -readarray -t paths < <(nix path-info --all --json --store file://$cacheDir | jq 'keys|sort|.[]' -r) +readarray -t paths < <(nix path-info --all --json --store file://"$cacheDir" | jq 'keys|sort|.[]' -r) [[ "${#paths[@]}" -eq 3 ]] for path in "${paths[@]}"; do [[ "$path" =~ -dependencies-input-0$ ]] \ @@ -25,16 +25,16 @@ for path in "${paths[@]}"; do done # Test copying build logs to the binary cache. -expect 1 nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available' -nix store copy-log --to file://$cacheDir $outPath -nix log --store file://$cacheDir $outPath | grep FOO -rm -rf $TEST_ROOT/var/log/nix -expect 1 nix log $outPath 2>&1 | grep 'is not available' -nix log --substituters file://$cacheDir $outPath | grep FOO +expect 1 nix log --store file://"$cacheDir" "$outPath" 2>&1 | grep 'is not available' +nix store copy-log --to file://"$cacheDir" "$outPath" +nix log --store file://"$cacheDir" "$outPath" | grep FOO +rm -rf "$TEST_ROOT/var/log/nix" +expect 1 nix log "$outPath" 2>&1 | grep 'is not available' +nix log --substituters file://"$cacheDir" "$outPath" | grep FOO # Test copying build logs from the binary cache. -nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath)^'*' -nix log $outPath | grep FOO +nix store copy-log --from file://"$cacheDir" "$(nix-store -qd "$outPath")"^'*' +nix log "$outPath" | grep FOO basicDownloadTests() { # No uploading tests bcause upload with force HTTP doesn't work. @@ -46,15 +46,15 @@ basicDownloadTests() { nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" - nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath" - [ -x $outPath/program ] + [ -x "$outPath/program" ] # But with the right configuration, "nix-env -qas" should also work. clearStore clearCacheCache - echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info + echo "WantMassQuery: 1" >> "$cacheDir/nix-cache-info" nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" @@ -62,12 +62,12 @@ basicDownloadTests() { x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only) [ -z "$x" ] - nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath" - nix-store --check-validity $outPath - nix-store -qR $outPath | grep input-2 + nix-store --check-validity "$outPath" + nix-store -qR "$outPath" | grep input-2 - echo "WantMassQuery: 0" >> $cacheDir/nix-cache-info + echo "WantMassQuery: 0" >> "$cacheDir/nix-cache-info" } @@ -83,22 +83,22 @@ basicDownloadTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) -mv $nar $nar.good -mkdir -p $TEST_ROOT/empty -nix-store --dump $TEST_ROOT/empty | xz > $nar +nar=$(find "$cacheDir/nar/" -type f -name "*.nar.xz" | head -n1) +mv "$nar" "$nar".good +mkdir -p "$TEST_ROOT/empty" +nix-store --dump "$TEST_ROOT/empty" | xz > "$nar" -expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grepQuiet "hash mismatch" $TEST_ROOT/log +expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "hash mismatch" "$TEST_ROOT/log" -mv $nar.good $nar +mv "$nar".good "$nar" # Test whether this unsigned cache is rejected if the user requires signed caches. clearStore clearCacheCache -if nix-store --substituters "file://$cacheDir" -r $outPath; then +if nix-store --substituters "file://$cacheDir" -r "$outPath"; then echo "unsigned binary cache incorrectly accepted" exit 1 fi @@ -107,131 +107,134 @@ fi # Test whether fallback works if a NAR has disappeared. This does not require --fallback. clearStore -mv $cacheDir/nar $cacheDir/nar2 +mv "$cacheDir/nar" "$cacheDir/nar2" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" -mv $cacheDir/nar2 $cacheDir/nar +mv "$cacheDir/nar2" "$cacheDir/nar" # Test whether fallback works if a NAR is corrupted. This does require --fallback. clearStore -mv $cacheDir/nar $cacheDir/nar2 -mkdir $cacheDir/nar -for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done +mv "$cacheDir/nar" "$cacheDir/nar2" +mkdir "$cacheDir/nar" +for i in $(cd "$cacheDir/nar2" && echo *); do touch "$cacheDir"/nar/"$i"; done -(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) +(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result") -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" --fallback -rm -rf $cacheDir/nar -mv $cacheDir/nar2 $cacheDir/nar +rm -rf "$cacheDir/nar" +mv "$cacheDir/nar2" "$cacheDir/nar" # Test whether building works if the binary cache contains an # incomplete closure. clearStore -rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) +rm -v "$(grep -l "StorePath:.*dependencies-input-2" "$cacheDir"/*.narinfo)" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grepQuiet "copying path.*input-0" $TEST_ROOT/log -grepQuiet "copying path.*input-2" $TEST_ROOT/log -grepQuiet "copying path.*top" $TEST_ROOT/log +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "copying path.*input-0" "$TEST_ROOT/log" +grepQuiet "copying path.*input-2" "$TEST_ROOT/log" +grepQuiet "copying path.*top" "$TEST_ROOT/log" # Idem, but without cached .narinfo. clearStore clearCacheCache -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grepQuiet "don't know how to build" $TEST_ROOT/log -grepQuiet "building.*input-1" $TEST_ROOT/log -grepQuiet "building.*input-2" $TEST_ROOT/log -grepQuiet "copying path.*input-0" $TEST_ROOT/log -grepQuiet "copying path.*top" $TEST_ROOT/log +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "don't know how to build" "$TEST_ROOT/log" +grepQuiet "building.*input-1" "$TEST_ROOT/log" +grepQuiet "building.*input-2" "$TEST_ROOT/log" +grepQuiet "copying path.*input-0" "$TEST_ROOT/log" +grepQuiet "copying path.*top" "$TEST_ROOT/log" # Create a signed binary cache. clearCache clearCacheCache -nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk1 -publicKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk1) +nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk1" +publicKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk1") -nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk2 -badKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk2) +nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk2" +badKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk2") -nix key generate-secret --key-name foo.nixos.org-1 > $TEST_ROOT/sk3 -otherKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk3) +nix key generate-secret --key-name foo.nixos.org-1 > "$TEST_ROOT/sk3" +otherKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk3") -_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP='' nix copy --to file://"$cacheDir"?secret-key="$TEST_ROOT"/sk1 "$outPath" # Downloading should fail if we don't provide a key. clearStore clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir") +(! nix-store -r "$outPath" --substituters "file://$cacheDir") # And it should fail if we provide an incorrect key. clearStore clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$badKey") +(! nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$badKey") # It should succeed if we provide the correct key. -nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" +nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" # It should fail if we corrupt the .narinfo. clearStore cacheDir2=$TEST_ROOT/binary-cache-2 -rm -rf $cacheDir2 -cp -r $cacheDir $cacheDir2 +rm -rf "$cacheDir2" +cp -r "$cacheDir" "$cacheDir2" -for i in $cacheDir2/*.narinfo; do - grep -v References $i > $i.tmp - mv $i.tmp $i +for i in "$cacheDir2"/*.narinfo; do + grep -v References "$i" > "$i".tmp + mv "$i".tmp "$i" done clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") +(! nix-store -r "$outPath" --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") # If we provide a bad and a good binary cache, it should succeed. -nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" +nix-store -r "$outPath" --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" unset _NIX_FORCE_HTTP # Test 'nix verify --all' on a binary cache. -nix store verify -vvvvv --all --store file://$cacheDir --no-trust +nix store verify -vvvvv --all --store file://"$cacheDir" --no-trust # Test local NAR caching. narCache=$TEST_ROOT/nar-cache -rm -rf $narCache -mkdir $narCache +rm -rf "$narCache" +mkdir "$narCache" -[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] rm -rfv "$cacheDir/nar" -[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] -(! nix store cat --store file://$cacheDir $outPath/foobar) +(! nix store cat --store file://"$cacheDir" "$outPath/foobar") # Test NAR listing generation. clearCache + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' with import ./config.nix; mkDerivation { @@ -240,16 +243,18 @@ outPath=$(nix-build --no-out-link -E ' } ') -nix copy --to file://$cacheDir?write-nar-listing=1 $outPath +nix copy --to file://"$cacheDir"?write-nar-listing=1 "$outPath" diff -u \ - <(jq -S < $cacheDir/$(basename $outPath | cut -c1-32).ls) \ + <(jq -S < "$cacheDir/$(basename "$outPath" | cut -c1-32).ls") \ <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S) # Test debug info index generation. clearCache +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' with import ./config.nix; mkDerivation { @@ -258,14 +263,16 @@ outPath=$(nix-build --no-out-link -E ' } ') -nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath +nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" "$outPath" diff -u \ - <(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \ + <(jq -S < "$cacheDir"/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) \ <(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S) # Test against issue https://github.com/NixOS/nix/issues/3964 -# + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 expr=' with import ./config.nix; mkDerivation { @@ -275,22 +282,22 @@ expr=' } ' outPath=$(nix-build --no-out-link -E "$expr") -docPath=$(nix-store -q --references $outPath) +docPath=$(nix-store -q --references "$outPath") # $ nix-store -q --tree $outPath # ...-multi-output # +---...-multi-output-doc -nix copy --to "file://$cacheDir" $outPath +nix copy --to "file://$cacheDir" "$outPath" hashpart() { basename "$1" | cut -c1-32 } # break the closure of out by removing doc -rm $cacheDir/$(hashpart $docPath).narinfo +rm "$cacheDir/$(hashpart "$docPath")".narinfo -nix-store --delete $outPath $docPath +nix-store --delete "$outPath" "$docPath" # -vvv is the level that logs during the loop timeout 60 nix-build --no-out-link -E "$expr" --option substituters "file://$cacheDir" \ --option trusted-binary-caches "file://$cacheDir" --no-require-sigs From 7186c68f75530f9590c8b6e7c14dd10d1bf6ad81 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:47:35 -0700 Subject: [PATCH 10/67] housekeeping: shellcheck for tests/functional/brotli.sh Co-authored-by: Sandro --- tests/functional/brotli.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/functional/brotli.sh b/tests/functional/brotli.sh index 02a2a0875..672e771c2 100755 --- a/tests/functional/brotli.sh +++ b/tests/functional/brotli.sh @@ -9,15 +9,15 @@ cacheURI="file://$cacheDir?compression=br" outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -HASH=$(nix hash path $outPath) +HASH=$(nix hash path "$outPath") clearStore clearCacheCache -nix copy --from $cacheURI $outPath --no-check-sigs +nix copy --from "$cacheURI" "$outPath" --no-check-sigs -HASH2=$(nix hash path $outPath) +HASH2=$(nix hash path "$outPath") -[[ $HASH = $HASH2 ]] +[[ $HASH == "$HASH2" ]] From f615489e0edd5113979d8506a8d0d8b54d5b8217 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:47:41 -0700 Subject: [PATCH 11/67] housekeeping: shellcheck for tests/functional/build-delete.sh --- tests/functional/build-delete.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh index 2ef3008f6..59cf95bd2 100755 --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -6,25 +6,25 @@ clearStore # https://github.com/NixOS/nix/issues/6572 issue_6572_independent_outputs() { - nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json + nix build -f multiple-outputs.nix --json independent --no-link > "$TEST_ROOT"/independent.json # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) nix-store --delete "$p" # Clean up for next test # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. - nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)" + nix-store --delete "$(jq -r <"$TEST_ROOT"/independent.json .[0].outputs.first)" p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) - cmp $p < $TEST_ROOT/a.json + nix build -f multiple-outputs.nix --json a --no-link > "$TEST_ROOT"/a.json # # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) nix-store --delete "$p" # Clean up for next test # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. - nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)" + nix-store --delete "$(jq -r <"$TEST_ROOT"/a.json .[0].outputs.second)" p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) - cmp $p < Date: Mon, 3 Jun 2024 13:47:43 -0700 Subject: [PATCH 12/67] housekeeping: shellcheck for tests/functional/build-dry.sh --- tests/functional/build-dry.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/functional/build-dry.sh b/tests/functional/build-dry.sh index 9336cf745..dca5888a6 100755 --- a/tests/functional/build-dry.sh +++ b/tests/functional/build-dry.sh @@ -35,17 +35,17 @@ clearStore clearCache RESULT=$TEST_ROOT/result-link -rm -f $RESULT +rm -f "$RESULT" -nix-build dependencies.nix -o $RESULT --dry-run +nix-build dependencies.nix -o "$RESULT" --dry-run [[ ! -h $RESULT ]] || fail "nix-build --dry-run created output link" -nix build -f dependencies.nix -o $RESULT --dry-run +nix build -f dependencies.nix -o "$RESULT" --dry-run [[ ! -h $RESULT ]] || fail "nix build --dry-run created output link" -nix build -f dependencies.nix -o $RESULT +nix build -f dependencies.nix -o "$RESULT" [[ -h $RESULT ]] @@ -58,12 +58,12 @@ RES=$(nix build -f dependencies.nix --dry-run --json) if [[ -z "${NIX_TESTS_CA_BY_DEFAULT-}" ]]; then echo "$RES" | jq '.[0] | [ - (.drvPath | test("'$NIX_STORE_DIR'.*\\.drv")), - (.outputs.out | test("'$NIX_STORE_DIR'")) + (.drvPath | test("'"$NIX_STORE_DIR"'.*\\.drv")), + (.outputs.out | test("'"$NIX_STORE_DIR"'")) ] | all' else echo "$RES" | jq '.[0] | [ - (.drvPath | test("'$NIX_STORE_DIR'.*\\.drv")), + (.drvPath | test("'"$NIX_STORE_DIR"'.*\\.drv")), .outputs.out == null ] | all' fi From 80c44138cb62f41f6b198d8a3a317a650f11f64e Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:50:30 -0700 Subject: [PATCH 13/67] housekeeping: shellcheck for tests/functional/ca/build-cache.sh Co-authored-by: Sandro --- .shellcheckrc | 2 ++ tests/functional/binary-cache.sh | 22 +++++++++++----------- tests/functional/ca/build-cache.sh | 7 ++++--- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.shellcheckrc b/.shellcheckrc index 662e2045c..de98055f7 100644 --- a/.shellcheckrc +++ b/.shellcheckrc @@ -1,2 +1,4 @@ external-sources=true source-path=SCRIPTDIR +# Hack for scripts in e.g. tests/functional/ca +source-path=SCRIPTDIR/.. diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 23135359b..5ef6d89d4 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -14,9 +14,9 @@ clearStore clearCache outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --to file://"$cacheDir" "$outPath" +nix copy --to "file://$cacheDir" "$outPath" -readarray -t paths < <(nix path-info --all --json --store file://"$cacheDir" | jq 'keys|sort|.[]' -r) +readarray -t paths < <(nix path-info --all --json --store "file://$cacheDir" | jq 'keys|sort|.[]' -r) [[ "${#paths[@]}" -eq 3 ]] for path in "${paths[@]}"; do [[ "$path" =~ -dependencies-input-0$ ]] \ @@ -25,15 +25,15 @@ for path in "${paths[@]}"; do done # Test copying build logs to the binary cache. -expect 1 nix log --store file://"$cacheDir" "$outPath" 2>&1 | grep 'is not available' -nix store copy-log --to file://"$cacheDir" "$outPath" -nix log --store file://"$cacheDir" "$outPath" | grep FOO +expect 1 nix log --store "file://$cacheDir" "$outPath" 2>&1 | grep 'is not available' +nix store copy-log --to "file://$cacheDir" "$outPath" +nix log --store "file://$cacheDir" "$outPath" | grep FOO rm -rf "$TEST_ROOT/var/log/nix" expect 1 nix log "$outPath" 2>&1 | grep 'is not available' -nix log --substituters file://"$cacheDir" "$outPath" | grep FOO +nix log --substituters "file://$cacheDir" "$outPath" | grep FOO # Test copying build logs from the binary cache. -nix store copy-log --from file://"$cacheDir" "$(nix-store -qd "$outPath")"^'*' +nix store copy-log --from "file://$cacheDir" "$(nix-store -qd "$outPath")"^'*' nix log "$outPath" | grep FOO basicDownloadTests() { @@ -166,7 +166,7 @@ badKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk2") nix key generate-secret --key-name foo.nixos.org-1 > "$TEST_ROOT/sk3" otherKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk3") -_NIX_FORCE_HTTP='' nix copy --to file://"$cacheDir"?secret-key="$TEST_ROOT"/sk1 "$outPath" +_NIX_FORCE_HTTP='' nix copy --to "file://$cacheDir"?secret-key="$TEST_ROOT"/sk1 "$outPath" # Downloading should fail if we don't provide a key. @@ -212,7 +212,7 @@ unset _NIX_FORCE_HTTP # Test 'nix verify --all' on a binary cache. -nix store verify -vvvvv --all --store file://"$cacheDir" --no-trust +nix store verify -vvvvv --all --store "file://$cacheDir" --no-trust # Test local NAR caching. @@ -226,7 +226,7 @@ rm -rfv "$cacheDir/nar" [[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] -(! nix store cat --store file://"$cacheDir" "$outPath/foobar") +(! nix store cat --store "file://$cacheDir" "$outPath/foobar") # Test NAR listing generation. @@ -243,7 +243,7 @@ outPath=$(nix-build --no-out-link -E ' } ') -nix copy --to file://"$cacheDir"?write-nar-listing=1 "$outPath" +nix copy --to "file://$cacheDir"?write-nar-listing=1 "$outPath" diff -u \ <(jq -S < "$cacheDir/$(basename "$outPath" | cut -c1-32).ls") \ diff --git a/tests/functional/ca/build-cache.sh b/tests/functional/ca/build-cache.sh index 6a4080fec..5cc71823e 100644 --- a/tests/functional/ca/build-cache.sh +++ b/tests/functional/ca/build-cache.sh @@ -26,7 +26,8 @@ copyAttr () { # Note: to copy CA derivations, we need to copy the realisations, which # currently requires naming the installables, not just the derivation output # path. - nix copy --to file://$cacheDir "${args[@]}" + + nix copy --to "file://$cacheDir" "${args[@]}" } testRemoteCacheFor () { @@ -35,7 +36,7 @@ testRemoteCacheFor () { copyAttr "$derivationPath" 1 clearStore # Check nothing gets built. - buildAttr "$derivationPath" 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse " will be built:" + buildAttr "$derivationPath" 1 --option substituters "file://$cacheDir" --no-require-sigs |& grepQuietInverse " will be built:" } testRemoteCache () { @@ -48,4 +49,4 @@ testRemoteCache () { } clearStore -testRemoteCache \ No newline at end of file +testRemoteCache From 627176fd54ebbd7a8411fd736f5ca6c7cd02e197 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:50:50 -0700 Subject: [PATCH 14/67] housekeeping: shellcheck for tests/functional/ca/build.sh --- tests/functional/ca/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/ca/build.sh b/tests/functional/ca/build.sh index e1a8a7625..e5ad9d2a0 100644 --- a/tests/functional/ca/build.sh +++ b/tests/functional/ca/build.sh @@ -20,11 +20,11 @@ testDeterministicCA () { testCutoffFor () { local out1 out2 - out1=$(buildAttr $1 1) + out1=$(buildAttr "$1" 1) # The seed only changes the root derivation, and not it's output, so the # dependent derivations should only need to be built once. buildAttr rootCA 2 - out2=$(buildAttr $1 2 -j0) + out2=$(buildAttr "$1" 2 -j0) test "$out1" == "$out2" } @@ -41,7 +41,7 @@ testGC () { nix-instantiate ./content-addressed.nix -A rootCA --arg seed 5 nix-collect-garbage --option keep-derivations true clearStore - buildAttr rootCA 1 --out-link $TEST_ROOT/rootCA + buildAttr rootCA 1 --out-link "$TEST_ROOT"/rootCA nix-collect-garbage buildAttr rootCA 1 -j0 } @@ -55,7 +55,7 @@ testNixCommand () { testNormalization () { clearStore outPath=$(buildAttr rootCA 1) - test "$(stat -c %Y $outPath)" -eq 1 + test "$(stat -c %Y "$outPath")" -eq 1 } clearStore From 2dfbba3e5e075ecbf6ddd3fb90ecee3003db85ec Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:52:29 -0700 Subject: [PATCH 15/67] housekeeping: shellcheck for tests/functional/ca/derivation-json.sh --- tests/functional/ca/derivation-json.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/functional/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh index c1480fd17..1e2a8fe35 100644 --- a/tests/functional/ca/derivation-json.sh +++ b/tests/functional/ca/derivation-json.sh @@ -1,29 +1,31 @@ +#!/usr/bin/env bash +# source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 drvPath=$(nix-instantiate ../simple.nix) -nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json +nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/simple.json -drvPath2=$(nix derivation add < $TEST_HOME/simple.json) +drvPath2=$(nix derivation add < "$TEST_HOME"/simple.json) [[ "$drvPath" = "$drvPath2" ]] # Content-addressed derivations can be renamed. -jq '.name = "foo"' < $TEST_HOME/simple.json > $TEST_HOME/foo.json -drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) +jq '.name = "foo"' < "$TEST_HOME"/simple.json > "$TEST_HOME"/foo.json +drvPath3=$(nix derivation add --dry-run < "$TEST_HOME"/foo.json) # With --dry-run nothing is actually written [[ ! -e "$drvPath3" ]] # But the JSON is rejected without the experimental feature -expectStderr 1 nix derivation add < $TEST_HOME/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" +expectStderr 1 nix derivation add < "$TEST_HOME"/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" # Without --dry-run it is actually written -drvPath4=$(nix derivation add < $TEST_HOME/foo.json) +drvPath4=$(nix derivation add < "$TEST_HOME"/foo.json) [[ "$drvPath4" = "$drvPath3" ]] [[ -e "$drvPath3" ]] # The modified derivation read back as JSON matches -nix derivation show $drvPath3 | jq .[] > $TEST_HOME/foo-read.json -diff $TEST_HOME/foo.json $TEST_HOME/foo-read.json +nix derivation show "$drvPath3" | jq .[] > "$TEST_HOME"/foo-read.json +diff "$TEST_HOME"/foo.json "$TEST_HOME"/foo-read.json From 195c0da849e924ac0eee3583cace931eca158a3f Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:54:17 -0700 Subject: [PATCH 16/67] housekeeping: shellcheck for tests/functional/ca/duplicate-realisation-in-closure.sh --- tests/functional/ca/duplicate-realisation-in-closure.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh index da9cd8fb4..0baf15cc2 100644 --- a/tests/functional/ca/duplicate-realisation-in-closure.sh +++ b/tests/functional/ca/duplicate-realisation-in-closure.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source ./common.sh requireDaemonNewerThan "2.4pre20210625" @@ -5,7 +7,7 @@ requireDaemonNewerThan "2.4pre20210625" export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" -rm -rf $REMOTE_STORE_DIR +rm -rf "$REMOTE_STORE_DIR" clearStore # Build dep1 and push that to the binary cache. From deacc421eb5bc53260ece90ce8cd1593ea7f7769 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:54:41 -0700 Subject: [PATCH 17/67] housekeeping: shellcheck for tests/functional/ca/nix-copy.sh --- tests/functional/ca/nix-copy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/ca/nix-copy.sh b/tests/functional/ca/nix-copy.sh index 7a8307a4e..f77b00030 100755 --- a/tests/functional/ca/nix-copy.sh +++ b/tests/functional/ca/nix-copy.sh @@ -15,13 +15,13 @@ testOneCopy () { rm -rf "$REMOTE_STORE_DIR" attrPath="$1" - nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix + nix copy --to "$REMOTE_STORE" "$attrPath" --file ./content-addressed.nix ensureCorrectlyCopied "$attrPath" # Ensure that we can copy back what we put in the store clearStore - nix copy --from $REMOTE_STORE \ + nix copy --from "$REMOTE_STORE" \ --file ./content-addressed.nix "$attrPath" \ --no-check-sigs } From 8f8553762960de9b1ff0d8add46ab9bc18cd76c5 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:54:42 -0700 Subject: [PATCH 18/67] housekeeping: shellcheck for tests/functional/ca/nix-run.sh --- tests/functional/ca/nix-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/ca/nix-run.sh b/tests/functional/ca/nix-run.sh index 5f46518e8..920950c11 100755 --- a/tests/functional/ca/nix-run.sh +++ b/tests/functional/ca/nix-run.sh @@ -4,4 +4,4 @@ source common.sh FLAKE_PATH=path:$PWD -nix run --no-write-lock-file $FLAKE_PATH#runnable +nix run --no-write-lock-file "$FLAKE_PATH#runnable" From 04876c39e46758a7445d390aa679b090248f7c4f Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 13:57:25 -0700 Subject: [PATCH 19/67] housekeeping: shellcheck for tests/functional/ca/signatures.sh --- tests/functional/ca/signatures.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/functional/ca/signatures.sh b/tests/functional/ca/signatures.sh index eb18a4130..f69a205d2 100644 --- a/tests/functional/ca/signatures.sh +++ b/tests/functional/ca/signatures.sh @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + source common.sh clearStore clearCache -nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 -pk1=$(cat $TEST_ROOT/pk1) +nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT/sk1" "$TEST_ROOT/pk1" +pk1=$(cat "$TEST_ROOT/pk1") export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" @@ -19,16 +21,16 @@ testOneCopy () { rm -rf "$REMOTE_STORE_DIR" attrPath="$1" - nix copy -vvvv --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix \ + nix copy -vvvv --to "$REMOTE_STORE" "$attrPath" --file ./content-addressed.nix \ --secret-key-files "$TEST_ROOT/sk1" --show-trace ensureCorrectlyCopied "$attrPath" # Ensure that we can copy back what we put in the store clearStore - nix copy --from $REMOTE_STORE \ + nix copy --from "$REMOTE_STORE" \ --file ./content-addressed.nix "$attrPath" \ - --trusted-public-keys $pk1 + --trusted-public-keys "$pk1" } for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do From 259b502773849afe385b473c48e852eb7bb7e9bd Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:14:23 -0700 Subject: [PATCH 20/67] housekeeping: shellcheck for tests/functional/ca/substitute.sh --- tests/functional/ca/substitute.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index ea981adc4..76cedb4bc 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -4,9 +4,10 @@ source common.sh +# shellcheck disable=SC1111 needLocalStore "“--no-require-sigs” can’t be used with the daemon" -rm -rf $TEST_ROOT/binary_cache +rm -rf "$TEST_ROOT"/binary_cache export REMOTE_STORE_DIR=$TEST_ROOT/binary_cache export REMOTE_STORE=file://$REMOTE_STORE_DIR @@ -17,11 +18,11 @@ buildDrvs () { # Populate the remote cache clearStore -nix copy --to $REMOTE_STORE --file ./content-addressed.nix +nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix # Restart the build on an empty store, ensuring that we don't build clearStore -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA # Check that the thing we’ve just substituted has its realisation stored nix realisation info --file ./content-addressed.nix transitivelyDependentCA # Check that its dependencies have it too @@ -63,9 +64,9 @@ clearStore # Add the realisations of rootCA to the cachecache clearCacheCache export _NIX_FORCE_HTTP=1 -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm $REMOTE_STORE_DIR/realisations/* -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +rm "$REMOTE_STORE_DIR"/realisations/* +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 From d7bb5bde48acbc9be46933344cc6c7d47c43a1df Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:14:46 -0700 Subject: [PATCH 21/67] housekeeping: shellcheck for tests/functional/check-refs.sh --- tests/functional/check-refs.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/functional/check-refs.sh b/tests/functional/check-refs.sh index 2cebdd84d..6534e55c6 100755 --- a/tests/functional/check-refs.sh +++ b/tests/functional/check-refs.sh @@ -6,42 +6,42 @@ clearStore RESULT=$TEST_ROOT/result -dep=$(nix-build -o $RESULT check-refs.nix -A dep) +dep=$(nix-build -o "$RESULT" check-refs.nix -A dep) # test1 references dep, not itself. -test1=$(nix-build -o $RESULT check-refs.nix -A test1) -nix-store -q --references $test1 | grepQuietInverse $test1 -nix-store -q --references $test1 | grepQuiet $dep +test1=$(nix-build -o "$RESULT" check-refs.nix -A test1) +nix-store -q --references "$test1" | grepQuietInverse "$test1" +nix-store -q --references "$test1" | grepQuiet "$dep" # test2 references src, not itself nor dep. -test2=$(nix-build -o $RESULT check-refs.nix -A test2) -nix-store -q --references $test2 | grepQuietInverse $test2 -nix-store -q --references $test2 | grepQuietInverse $dep -nix-store -q --references $test2 | grepQuiet aux-ref +test2=$(nix-build -o "$RESULT" check-refs.nix -A test2) +nix-store -q --references "$test2" | grepQuietInverse "$test2" +nix-store -q --references "$test2" | grepQuietInverse "$dep" +nix-store -q --references "$test2" | grepQuiet aux-ref # test3 should fail (unallowed ref). -(! nix-build -o $RESULT check-refs.nix -A test3) +(! nix-build -o "$RESULT" check-refs.nix -A test3) # test4 should succeed. -nix-build -o $RESULT check-refs.nix -A test4 +nix-build -o "$RESULT" check-refs.nix -A test4 # test5 should succeed. -nix-build -o $RESULT check-refs.nix -A test5 +nix-build -o "$RESULT" check-refs.nix -A test5 # test6 should fail (unallowed self-ref). -(! nix-build -o $RESULT check-refs.nix -A test6) +(! nix-build -o "$RESULT" check-refs.nix -A test6) # test7 should succeed (allowed self-ref). -nix-build -o $RESULT check-refs.nix -A test7 +nix-build -o "$RESULT" check-refs.nix -A test7 # test8 should fail (toFile depending on derivation output). -(! nix-build -o $RESULT check-refs.nix -A test8) +(! nix-build -o "$RESULT" check-refs.nix -A test8) # test9 should fail (disallowed reference). -(! nix-build -o $RESULT check-refs.nix -A test9) +(! nix-build -o "$RESULT" check-refs.nix -A test9) # test10 should succeed (no disallowed references). -nix-build -o $RESULT check-refs.nix -A test10 +nix-build -o "$RESULT" check-refs.nix -A test10 if isDaemonNewer 2.12pre20230103; then if ! isDaemonNewer 2.16.0; then @@ -50,6 +50,6 @@ if isDaemonNewer 2.12pre20230103; then fi # test11 should succeed. - test11=$(nix-build -o $RESULT check-refs.nix -A test11) + test11=$(nix-build -o "$RESULT" check-refs.nix -A test11) [[ -z $(nix-store -q --references "$test11") ]] fi From 4f04006bc14c865acae67ee1890b72af2bd430a6 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:14:48 -0700 Subject: [PATCH 22/67] housekeeping: shellcheck for tests/functional/check-reqs.sh --- tests/functional/check-reqs.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/functional/check-reqs.sh b/tests/functional/check-reqs.sh index 2bcd558fd..4d795391e 100755 --- a/tests/functional/check-reqs.sh +++ b/tests/functional/check-reqs.sh @@ -6,13 +6,13 @@ clearStore RESULT=$TEST_ROOT/result -nix-build -o $RESULT check-reqs.nix -A test1 +nix-build -o "$RESULT" check-reqs.nix -A test1 -(! nix-build -o $RESULT check-reqs.nix -A test2) -(! nix-build -o $RESULT check-reqs.nix -A test3) -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep1' -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep2' -(! nix-build -o $RESULT check-reqs.nix -A test5) -(! nix-build -o $RESULT check-reqs.nix -A test6) +(! nix-build -o "$RESULT" check-reqs.nix -A test2) +(! nix-build -o "$RESULT" check-reqs.nix -A test3) +(! nix-build -o "$RESULT" check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep1' +(! nix-build -o "$RESULT" check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep2' +(! nix-build -o "$RESULT" check-reqs.nix -A test5) +(! nix-build -o "$RESULT" check-reqs.nix -A test6) -nix-build -o $RESULT check-reqs.nix -A test7 +nix-build -o "$RESULT" check-reqs.nix -A test7 From 63272235e282ad0f075677883eb2812a9c68cec2 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:51:07 -0700 Subject: [PATCH 23/67] housekeeping: shellcheck for tests/functional/case-hacks.sh Co-authored-by: Sandro --- tests/functional/case-hack.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/functional/case-hack.sh b/tests/functional/case-hack.sh index fbc8242ff..48a2ab13f 100755 --- a/tests/functional/case-hack.sh +++ b/tests/functional/case-hack.sh @@ -4,18 +4,19 @@ source common.sh clearStore -rm -rf $TEST_ROOT/case +rm -rf "$TEST_ROOT/case" -opts="--option use-case-hack true" +opts=("--option" "use-case-hack" "true") # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. -nix-store $opts --restore $TEST_ROOT/case < case.nar -nix-store $opts --dump $TEST_ROOT/case > $TEST_ROOT/case.nar -cmp case.nar $TEST_ROOT/case.nar -[ "$(nix-hash $opts --type sha256 $TEST_ROOT/case)" = "$(nix-hash --flat --type sha256 case.nar)" ] + +nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case.nar +nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > "$TEST_ROOT/case.nar" +cmp case.nar "$TEST_ROOT/case.nar" +[ "$(nix-hash "${opts[@]}" --type sha256 "$TEST_ROOT/case")" = "$(nix-hash --flat --type sha256 case.nar)" ] # Check whether we detect true collisions (e.g. those remaining after # removal of the suffix). touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" -(! nix-store $opts --dump $TEST_ROOT/case > /dev/null) +(! nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null) From 48520cb71ee2b068e66b26b37554cba8ff3b69cc Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 12:23:17 -0700 Subject: [PATCH 24/67] housekeeping: shellcheck for tests/functional/chroot-store.sh --- tests/functional/chroot-store.sh | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 60b9c50a7..741907fca 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -2,34 +2,34 @@ source common.sh -echo example > $TEST_ROOT/example.txt -mkdir -p $TEST_ROOT/x +echo example > "$TEST_ROOT"/example.txt +mkdir -p "$TEST_ROOT/x" export NIX_STORE_DIR=/nix2/store -CORRECT_PATH=$(cd $TEST_ROOT && nix-store --store ./x --add example.txt) +CORRECT_PATH=$(cd "$TEST_ROOT" && nix-store --store ./x --add example.txt) [[ $CORRECT_PATH =~ ^/nix2/store/.*-example.txt$ ]] -PATH1=$(cd $TEST_ROOT && nix path-info --store ./x $CORRECT_PATH) -[ $CORRECT_PATH == $PATH1 ] +PATH1=$(cd "$TEST_ROOT" && nix path-info --store ./x "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH1" ] -PATH2=$(nix path-info --store "$TEST_ROOT/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH2 ] +PATH2=$(nix path-info --store "$TEST_ROOT/x" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH2" ] -PATH3=$(nix path-info --store "local?root=$TEST_ROOT/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH3 ] +PATH3=$(nix path-info --store "local?root=$TEST_ROOT/x" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH3" ] # Ensure store info trusted works with local store -nix --store $TEST_ROOT/x store info --json | jq -e '.trusted' +nix --store "$TEST_ROOT/x" store info --json | jq -e '.trusted' # Test building in a chroot store. if canUseSandbox; then flakeDir=$TEST_ROOT/flake - mkdir -p $flakeDir + mkdir -p "$flakeDir" - cat > $flakeDir/flake.nix < "$flakeDir"/flake.nix < Date: Tue, 4 Jun 2024 13:32:30 -0700 Subject: [PATCH 25/67] housekeeping: shellcheck for tests/functional/compression-levels.sh --- tests/functional/compression-levels.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/compression-levels.sh b/tests/functional/compression-levels.sh index 34f66c531..6a2111f10 100755 --- a/tests/functional/compression-levels.sh +++ b/tests/functional/compression-levels.sh @@ -9,16 +9,16 @@ outPath=$(nix-build dependencies.nix --no-out-link) cacheURI="file://$cacheDir?compression=xz&compression-level=0" -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -FILESIZES=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') +FILESIZES=$(cat "${cacheDir}"/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') clearCache cacheURI="file://$cacheDir?compression=xz&compression-level=5" -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -FILESIZES2=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') +FILESIZES2=$(cat "${cacheDir}"/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') [[ $FILESIZES -gt $FILESIZES2 ]] From 847842c4bbb33dcde0fbdad557b3776482a28a87 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:32:33 -0700 Subject: [PATCH 26/67] housekeeping: shellcheck for tests/functional/derivation-json.sh --- tests/functional/derivation-json.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/derivation-json.sh b/tests/functional/derivation-json.sh index 59c77e6c5..06f934cfe 100755 --- a/tests/functional/derivation-json.sh +++ b/tests/functional/derivation-json.sh @@ -4,11 +4,11 @@ source common.sh drvPath=$(nix-instantiate simple.nix) -nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json +nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/simple.json -drvPath2=$(nix derivation add < $TEST_HOME/simple.json) +drvPath2=$(nix derivation add < "$TEST_HOME"/simple.json) [[ "$drvPath" = "$drvPath2" ]] # Input addressed derivations cannot be renamed. -jq '.name = "foo"' < $TEST_HOME/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" +jq '.name = "foo"' < "$TEST_HOME"/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" From 1c1abefdd2c7da4873f78c8f71583294a94ae187 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:32:35 -0700 Subject: [PATCH 27/67] housekeeping: shellcheck for tests/functional/dyn-drv/text-hashed-output.sh --- tests/functional/dyn-drv/text-hashed-output.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/dyn-drv/text-hashed-output.sh b/tests/functional/dyn-drv/text-hashed-output.sh index f3e5aa93b..2cc877219 100644 --- a/tests/functional/dyn-drv/text-hashed-output.sh +++ b/tests/functional/dyn-drv/text-hashed-output.sh @@ -20,7 +20,7 @@ nix show-derivation "$drvProducingDrv" out1=$(nix-build ./text-hashed-output.nix -A producingDrv --no-out-link) -nix path-info $drv --derivation --json | jq -nix path-info $out1 --derivation --json | jq +nix path-info "$drv" --derivation --json | jq +nix path-info "$out1" --derivation --json | jq -test $out1 == $drv +test "$out1" == "$drv" From 823d53c643d195f1663cc3dc5a0aa8e6184505d4 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:32:40 -0700 Subject: [PATCH 28/67] housekeeping: shellcheck for tests/functional/experimental-features.sh --- tests/functional/experimental-features.sh | 28 +++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh index 38f198eee..d7216992d 100755 --- a/tests/functional/experimental-features.sh +++ b/tests/functional/experimental-features.sh @@ -33,35 +33,35 @@ source common.sh NIX_CONFIG=' experimental-features = nix-command accept-flake-config = true -' expect 1 nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -[[ $(cat $TEST_ROOT/stdout) = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr -grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr +' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +[[ $(cat "$TEST_ROOT/stdout") = '' ]] +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" # 'flakes' experimental-feature is disabled after, ignore and warn NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command -' expect 1 nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -[[ $(cat $TEST_ROOT/stdout) = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr -grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr +' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +[[ $(cat "$TEST_ROOT/stdout") = '' ]] +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" # 'flakes' experimental-feature is enabled before, process NIX_CONFIG=' experimental-features = nix-command flakes accept-flake-config = true -' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -grepQuiet "true" $TEST_ROOT/stdout -grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr +' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +grepQuiet "true" "$TEST_ROOT/stdout" +grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" # 'flakes' experimental-feature is enabled after, process NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command flakes -' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -grepQuiet "true" $TEST_ROOT/stdout -grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr +' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +grepQuiet "true" "$TEST_ROOT/stdout" +grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" function exit_code_both_ways { expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null From f0492a61976ba41063565e903370c0c73e85a8ae Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:32:57 -0700 Subject: [PATCH 29/67] housekeeping: shellcheck for tests/functional/fetchPath.sh --- tests/functional/fetchPath.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/fetchPath.sh b/tests/functional/fetchPath.sh index e466e4494..560a270c1 100755 --- a/tests/functional/fetchPath.sh +++ b/tests/functional/fetchPath.sh @@ -2,7 +2,7 @@ source common.sh -touch $TEST_ROOT/foo -t 202211111111 +touch "$TEST_ROOT/foo" -t 202211111111 # We only check whether 2022-11-1* **:**:** is the last modified date since # `lastModified` is transformed into UTC in `builtins.fetchTarball`. [[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]] From 224f5515b91075e368259102cff40ba1a6b6494d Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:32:58 -0700 Subject: [PATCH 30/67] housekeeping: shellcheck for tests/functional/fetchTree-file.sh Co-authored-by: Sandro --- tests/functional/fetchTree-file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/fetchTree-file.sh b/tests/functional/fetchTree-file.sh index 9c9532876..6faccd282 100755 --- a/tests/functional/fetchTree-file.sh +++ b/tests/functional/fetchTree-file.sh @@ -90,7 +90,7 @@ EOF EOF # Test tarball URLs on the command line. - [[ $(nix flake metadata --json file://$PWD/test_input_no_ext | jq -r .resolved.type) = tarball ]] + [[ $(nix flake metadata --json "file://$PWD/test_input_no_ext" | jq -r .resolved.type) = tarball ]] popd From ae6a842c550e3fbb9c75e3ee93a78cf387fe4ec3 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:32:59 -0700 Subject: [PATCH 31/67] housekeeping: shellcheck for tests/functional/filter-source.sh --- tests/functional/filter-source.sh | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/functional/filter-source.sh b/tests/functional/filter-source.sh index c5e10be93..b32f5b59d 100755 --- a/tests/functional/filter-source.sh +++ b/tests/functional/filter-source.sh @@ -2,26 +2,26 @@ source common.sh -rm -rf $TEST_ROOT/filterin -mkdir $TEST_ROOT/filterin -mkdir $TEST_ROOT/filterin/foo -touch $TEST_ROOT/filterin/foo/bar -touch $TEST_ROOT/filterin/xyzzy -touch $TEST_ROOT/filterin/b -touch $TEST_ROOT/filterin/bak -touch $TEST_ROOT/filterin/bla.c.bak -ln -s xyzzy $TEST_ROOT/filterin/link +rm -rf "$TEST_ROOT/filterin" +mkdir "$TEST_ROOT/filterin" +mkdir "$TEST_ROOT/filterin/foo" +touch "$TEST_ROOT/filterin/foo/bar" +touch "$TEST_ROOT/filterin/xyzzy" +touch "$TEST_ROOT/filterin/b" +touch "$TEST_ROOT/filterin/bak" +touch "$TEST_ROOT"/filterin/bla.c.bak +ln -s xyzzy "$TEST_ROOT/filterin/link" checkFilter() { - test ! -e $1/foo/bar - test -e $1/xyzzy - test -e $1/bak - test ! -e $1/bla.c.bak - test ! -L $1/link + test ! -e "$1/foo/bar" + test -e "$1/xyzzy" + test -e "$1/bak" + test ! -e "$1"/bla.c.bak + test ! -L "$1/link" } -nix-build ./filter-source.nix -o $TEST_ROOT/filterout1 -checkFilter $TEST_ROOT/filterout1 +nix-build ./filter-source.nix -o "$TEST_ROOT/filterout1" +checkFilter "$TEST_ROOT/filterout1" -nix-build ./path.nix -o $TEST_ROOT/filterout2 -checkFilter $TEST_ROOT/filterout2 +nix-build ./path.nix -o "$TEST_ROOT/filterout2" +checkFilter "$TEST_ROOT/filterout2" From d81fd4a1c38a7a06cfbad1728aa9bfcc3753dc6b Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:09 -0700 Subject: [PATCH 32/67] housekeeping: shellcheck for tests/functional/flakes/absolute-attr-paths.sh --- tests/functional/flakes/absolute-attr-paths.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/functional/flakes/absolute-attr-paths.sh b/tests/functional/flakes/absolute-attr-paths.sh index 8ed1755c4..b0e6225d8 100755 --- a/tests/functional/flakes/absolute-attr-paths.sh +++ b/tests/functional/flakes/absolute-attr-paths.sh @@ -4,8 +4,8 @@ source ./common.sh flake1Dir=$TEST_ROOT/flake1 -mkdir -p $flake1Dir -cat > $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/flake.nix < Date: Tue, 4 Jun 2024 13:33:10 -0700 Subject: [PATCH 33/67] housekeeping: shellcheck for tests/functional/flakes/build-paths.sh --- tests/functional/flakes/build-paths.sh | 44 +++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/functional/flakes/build-paths.sh b/tests/functional/flakes/build-paths.sh index a336471f0..f8486528b 100755 --- a/tests/functional/flakes/build-paths.sh +++ b/tests/functional/flakes/build-paths.sh @@ -5,15 +5,15 @@ source ./common.sh flake1Dir=$TEST_ROOT/flake1 flake2Dir=$TEST_ROOT/flake2 -mkdir -p $flake1Dir $flake2Dir +mkdir -p "$flake1Dir" "$flake2Dir" -writeSimpleFlake $flake2Dir -tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT flake2 -hash=$(nix hash path $flake2Dir) +writeSimpleFlake "$flake2Dir" +tar cfz "$TEST_ROOT"/flake.tar.gz -C "$TEST_ROOT" flake2 +hash=$(nix hash path "$flake2Dir") dep=$(nix store add-path ./common.sh) -cat > $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/flake.nix < $flake1Dir/foo +echo bar > "$flake1Dir/foo" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a1" [[ -e $TEST_ROOT/result/simple.nix ]] -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a2 -[[ $(cat $TEST_ROOT/result) = bar ]] +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a2" +[[ $(cat "$TEST_ROOT/result") = bar ]] -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a3 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a3" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a4 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a4" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a6" [[ -e $TEST_ROOT/result/simple.nix ]] -nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8 -diff common.sh $TEST_ROOT/result +nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a8" +diff common.sh "$TEST_ROOT/result" -expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9 \ +expectStderr 1 nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a9" \ | grepQuiet "has 0 entries in its context. It should only have exactly one entry" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a10 -[[ $(readlink -e $TEST_ROOT/result) = *simple.drv ]] +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir"#a10 +[[ $(readlink -e "$TEST_ROOT/result") = *simple.drv ]] -expectStderr 1 nix build --json --out-link $TEST_ROOT/result $flake1Dir#a11 \ +expectStderr 1 nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a11" \ | grepQuiet "has a context which refers to a complete source and binary closure" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a12 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a12" [[ -e $TEST_ROOT/result/hello ]] -expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a13 \ +expectStderr 1 nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a13" \ | grepQuiet "has 2 entries in its context. It should only have exactly one entry" # Test accessing output in installables with `.` (foobarbaz.) -nix build --json --no-link $flake1Dir#a14.foo | jq --exit-status ' +nix build --json --no-link "$flake1Dir"#a14.foo | jq --exit-status ' (.[0] | (.drvPath | match(".*dot-installable.drv")) and (.outputs | keys == ["foo"])) From ece86b719150fe5122f78b24e3f46602b6656973 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:11 -0700 Subject: [PATCH 34/67] housekeeping: shellcheck for tests/functional/flakes/bundle.sh --- tests/functional/flakes/bundle.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/functional/flakes/bundle.sh b/tests/functional/flakes/bundle.sh index 711691e0b..5e185cbf6 100755 --- a/tests/functional/flakes/bundle.sh +++ b/tests/functional/flakes/bundle.sh @@ -2,9 +2,9 @@ source common.sh -cp ../simple.nix ../simple.builder.sh ../config.nix $TEST_HOME +cp ../simple.nix ../simple.builder.sh ../config.nix "$TEST_HOME" -cd $TEST_HOME +cd "$TEST_HOME" cat < flake.nix { @@ -27,8 +27,8 @@ EOF nix build .# nix bundle --bundler .# .# -nix bundle --bundler .#bundlers.$system.default .#packages.$system.default -nix bundle --bundler .#bundlers.$system.simple .#packages.$system.default +nix bundle --bundler .#bundlers."$system".default .#packages."$system".default +nix bundle --bundler .#bundlers."$system".simple .#packages."$system".default -nix bundle --bundler .#bundlers.$system.default .#apps.$system.default -nix bundle --bundler .#bundlers.$system.simple .#apps.$system.default +nix bundle --bundler .#bundlers."$system".default .#apps."$system".default +nix bundle --bundler .#bundlers."$system".simple .#apps."$system".default From 3b853e795beab3f5f7117b47ae0b3be42882fac4 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:12 -0700 Subject: [PATCH 35/67] housekeeping: shellcheck for tests/functional/flakes/circular.sh --- tests/functional/flakes/circular.sh | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/functional/flakes/circular.sh b/tests/functional/flakes/circular.sh index 6cab3a72b..5304496ba 100755 --- a/tests/functional/flakes/circular.sh +++ b/tests/functional/flakes/circular.sh @@ -8,10 +8,10 @@ requireGit flakeA=$TEST_ROOT/flakeA flakeB=$TEST_ROOT/flakeB -createGitRepo $flakeA -createGitRepo $flakeB +createGitRepo "$flakeA" +createGitRepo "$flakeB" -cat > $flakeA/flake.nix < "$flakeA"/flake.nix < $flakeA/flake.nix < $flakeB/flake.nix < "$flakeB"/flake.nix < $flakeB/flake.nix < Date: Tue, 4 Jun 2024 13:33:15 -0700 Subject: [PATCH 36/67] housekeeping: shellcheck for tests/functional/flakes/flake-in-submodule.sh --- tests/functional/flakes/flake-in-submodule.sh | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 2988352a9..08f751216 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -27,8 +27,8 @@ rootRepo=$TEST_ROOT/rootRepo subRepo=$TEST_ROOT/submodule -createGitRepo $subRepo -cat > $subRepo/flake.nix < "$subRepo"/flake.nix < $subRepo/flake.nix < $subRepo/sub.nix -git -C $subRepo add flake.nix sub.nix -git -C $subRepo commit -m Initial +echo '"expression in submodule"' > "$subRepo"/sub.nix +git -C "$subRepo" add flake.nix sub.nix +git -C "$subRepo" commit -m Initial -createGitRepo $rootRepo +createGitRepo "$rootRepo" -git -C $rootRepo submodule init -git -C $rootRepo submodule add $subRepo submodule -echo '"expression in root repo"' > $rootRepo/root.nix -git -C $rootRepo add root.nix -git -C $rootRepo commit -m "Add root.nix" +git -C "$rootRepo" submodule init +git -C "$rootRepo" submodule add "$subRepo" submodule +echo '"expression in root repo"' > "$rootRepo"/root.nix +git -C "$rootRepo" add root.nix +git -C "$rootRepo" commit -m "Add root.nix" flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule # Flake can live inside a submodule and can be accessed via ?dir=submodule -[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]] +[[ $(nix eval --json "$flakeref#sub" ) = '"expression in submodule"' ]] # The flake can access content outside of the submodule -[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]] +[[ $(nix eval --json "$flakeref#root" ) = '"expression in root repo"' ]] # Check that dirtying a submodule makes the entire thing dirty. -[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]] -echo '"foo"' > $rootRepo/submodule/sub.nix -[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]] -[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]] +[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]] +echo '"foo"' > "$rootRepo"/submodule/sub.nix +[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] +[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] From d95adb531ec55970c3eeae964c734015985e19b2 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:23 -0700 Subject: [PATCH 37/67] housekeeping: shellcheck for tests/functional/flakes/init.sh --- tests/functional/flakes/init.sh | 58 ++++++++++++++++----------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/functional/flakes/init.sh b/tests/functional/flakes/init.sh index f8d51e819..6787b9e79 100755 --- a/tests/functional/flakes/init.sh +++ b/tests/functional/flakes/init.sh @@ -8,16 +8,16 @@ templatesDir=$TEST_ROOT/templates flakeDir=$TEST_ROOT/flake nixpkgsDir=$TEST_ROOT/nixpkgs -nix registry add --registry $registry templates git+file://$templatesDir -nix registry add --registry $registry nixpkgs git+file://$nixpkgsDir +nix registry add --registry "$registry" templates git+file://"$templatesDir" +nix registry add --registry "$registry" nixpkgs git+file://"$nixpkgsDir" -createGitRepo $nixpkgsDir -createSimpleGitFlake $nixpkgsDir +createGitRepo "$nixpkgsDir" +createSimpleGitFlake "$nixpkgsDir" # Test 'nix flake init'. -createGitRepo $templatesDir +createGitRepo "$templatesDir" -cat > $templatesDir/flake.nix < "$templatesDir"/flake.nix < $templatesDir/flake.nix < $templatesDir/trivial/flake.nix < "$templatesDir"/trivial/flake.nix < $templatesDir/trivial/flake.nix < $templatesDir/trivial/a -echo b > $templatesDir/trivial/b +echo a > "$templatesDir/trivial/a" +echo b > "$templatesDir/trivial/b" -git -C $templatesDir add flake.nix trivial/ -git -C $templatesDir commit -m 'Initial' +git -C "$templatesDir" add flake.nix trivial/ +git -C "$templatesDir" commit -m 'Initial' nix flake check templates nix flake show templates nix flake show templates --json | jq -createGitRepo $flakeDir -(cd $flakeDir && nix flake init) -(cd $flakeDir && nix flake init) # check idempotence -git -C $flakeDir add flake.nix -nix flake check $flakeDir -nix flake show $flakeDir -nix flake show $flakeDir --json | jq -git -C $flakeDir commit -a -m 'Initial' +createGitRepo "$flakeDir" +(cd "$flakeDir" && nix flake init) +(cd "$flakeDir" && nix flake init) # check idempotence +git -C "$flakeDir" add flake.nix +nix flake check "$flakeDir" +nix flake show "$flakeDir" +nix flake show "$flakeDir" --json | jq +git -C "$flakeDir" commit -a -m 'Initial' # Test 'nix flake init' with benign conflicts createGitRepo "$flakeDir" -echo a > $flakeDir/a -(cd $flakeDir && nix flake init) # check idempotence +echo a > "$flakeDir/a" +(cd "$flakeDir" && nix flake init) # check idempotence # Test 'nix flake init' with conflicts createGitRepo "$flakeDir" -echo b > $flakeDir/a -pushd $flakeDir +echo b > "$flakeDir/a" +pushd "$flakeDir" (! nix flake init) |& grep "refusing to overwrite existing file '$flakeDir/a'" popd -git -C $flakeDir commit -a -m 'Changed' +git -C "$flakeDir" commit -a -m 'Changed' # Test 'nix flake new'. -rm -rf $flakeDir -nix flake new -t templates#trivial $flakeDir -nix flake new -t templates#trivial $flakeDir # check idempotence -nix flake check $flakeDir +rm -rf "$flakeDir" +nix flake new -t templates#trivial "$flakeDir" +nix flake new -t templates#trivial "$flakeDir" # check idempotence +nix flake check "$flakeDir" From c7b3468968c56e34a38096419d59be5e71941e6f Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:24 -0700 Subject: [PATCH 38/67] housekeeping: shellcheck for tests/functional/flakes/inputs.sh --- tests/functional/flakes/inputs.sh | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/functional/flakes/inputs.sh b/tests/functional/flakes/inputs.sh index 0327a3e9e..bc0603f1b 100755 --- a/tests/functional/flakes/inputs.sh +++ b/tests/functional/flakes/inputs.sh @@ -8,12 +8,12 @@ requireGit test_subdir_self_path() { baseDir=$TEST_ROOT/$RANDOM flakeDir=$baseDir/b-low - mkdir -p $flakeDir - writeSimpleFlake $baseDir - writeSimpleFlake $flakeDir + mkdir -p "$flakeDir" + writeSimpleFlake "$baseDir" + writeSimpleFlake "$flakeDir" - echo all good > $flakeDir/message - cat > $flakeDir/flake.nix < "$flakeDir/message" + cat > "$flakeDir"/flake.nix < $flakeDir/message - cat > $flakeDir/flake.nix < "$flakeDir/message" + cat > "$flakeDir"/flake.nix < $clientDir/flake.nix < "$clientDir"/flake.nix < Date: Tue, 4 Jun 2024 13:33:29 -0700 Subject: [PATCH 39/67] housekeeping: shellcheck for tests/functional/flakes/mercurial.sh --- tests/functional/flakes/mercurial.sh | 46 ++++++++++++++-------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/functional/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh index 0e9f2d626..1b095790b 100755 --- a/tests/functional/flakes/mercurial.sh +++ b/tests/functional/flakes/mercurial.sh @@ -5,41 +5,41 @@ source ./common.sh [[ $(type -p hg) ]] || skipTest "Mercurial not installed" flake1Dir=$TEST_ROOT/flake-hg1 -mkdir -p $flake1Dir -writeSimpleFlake $flake1Dir -hg init $flake1Dir +mkdir -p "$flake1Dir" +writeSimpleFlake "$flake1Dir" +hg init "$flake1Dir" -nix registry add --registry $registry flake1 hg+file://$flake1Dir +nix registry add --registry "$registry" flake1 hg+file://"$flake1Dir" flake2Dir=$TEST_ROOT/flake-hg2 -mkdir -p $flake2Dir -writeDependentFlake $flake2Dir -hg init $flake2Dir +mkdir -p "$flake2Dir" +writeDependentFlake "$flake2Dir" +hg init "$flake2Dir" -hg add $flake1Dir/* -hg commit --config ui.username=foobar@example.org $flake1Dir -m 'Initial commit' +hg add "$flake1Dir"/* +hg commit --config ui.username=foobar@example.org "$flake1Dir" -m 'Initial commit' -hg add $flake2Dir/flake.nix -hg commit --config ui.username=foobar@example.org $flake2Dir -m 'Initial commit' +hg add "$flake2Dir"/flake.nix +hg commit --config ui.username=foobar@example.org "$flake2Dir" -m 'Initial commit' -nix build -o $TEST_ROOT/result hg+file://$flake2Dir +nix build -o "$TEST_ROOT/result" hg+file://"$flake2Dir" [[ -e $TEST_ROOT/result/hello ]] -(! nix flake metadata --json hg+file://$flake2Dir | jq -e -r .revision) +(! nix flake metadata --json hg+file://"$flake2Dir" | jq -e -r .revision) -nix eval hg+file://$flake2Dir#expr +nix eval hg+file://"$flake2Dir"#expr -nix eval hg+file://$flake2Dir#expr +nix eval hg+file://"$flake2Dir"#expr -(! nix eval hg+file://$flake2Dir#expr --no-allow-dirty) +(! nix eval hg+file://"$flake2Dir"#expr --no-allow-dirty) -(! nix flake metadata --json hg+file://$flake2Dir | jq -e -r .revision) +(! nix flake metadata --json hg+file://"$flake2Dir" | jq -e -r .revision) -hg commit --config ui.username=foobar@example.org $flake2Dir -m 'Add lock file' +hg commit --config ui.username=foobar@example.org "$flake2Dir" -m 'Add lock file' -nix flake metadata --json hg+file://$flake2Dir --refresh | jq -e -r .revision -nix flake metadata --json hg+file://$flake2Dir -[[ $(nix flake metadata --json hg+file://$flake2Dir | jq -e -r .revCount) = 1 ]] +nix flake metadata --json hg+file://"$flake2Dir" --refresh | jq -e -r .revision +nix flake metadata --json hg+file://"$flake2Dir" +[[ $(nix flake metadata --json hg+file://"$flake2Dir" | jq -e -r .revCount) = 1 ]] -nix build -o $TEST_ROOT/result hg+file://$flake2Dir --no-registries --no-allow-dirty -nix build -o $TEST_ROOT/result hg+file://$flake2Dir --no-use-registries --no-allow-dirty +nix build -o "$TEST_ROOT/result" hg+file://"$flake2Dir" --no-registries --no-allow-dirty +nix build -o "$TEST_ROOT/result" hg+file://"$flake2Dir" --no-use-registries --no-allow-dirty From e1ce349d05608ea4acdcf02385f7fb634caa6a4b Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:30 -0700 Subject: [PATCH 40/67] housekeeping: shellcheck for tests/functional/flakes/search-root.sh --- tests/functional/flakes/search-root.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/functional/flakes/search-root.sh b/tests/functional/flakes/search-root.sh index c2337edc0..600dcf937 100755 --- a/tests/functional/flakes/search-root.sh +++ b/tests/functional/flakes/search-root.sh @@ -4,8 +4,8 @@ source common.sh clearStore -writeSimpleFlake $TEST_HOME -cd $TEST_HOME +writeSimpleFlake "$TEST_HOME" +cd "$TEST_HOME" mkdir -p foo/subdir echo '{ outputs = _: {}; }' > foo/flake.nix @@ -27,11 +27,11 @@ success=("" . .# .#test ../subdir ../subdir#test "$PWD") failure=("path:$PWD" "../simple.nix") for i in "${success[@]}"; do - nix build $i || fail "flake should be found by searching up directories" + nix build "$i" || fail "flake should be found by searching up directories" done for i in "${failure[@]}"; do - ! nix build $i || fail "flake should not search up directories when using 'path:'" + ! nix build "$i" || fail "flake should not search up directories when using 'path:'" done popd @@ -45,7 +45,7 @@ if [[ -n $(type -p git) ]]; then pushd subdir git init for i in "${success[@]}" "${failure[@]}"; do - ! nix build $i || fail "flake should not search past a git repository" + ! nix build "$i" || fail "flake should not search past a git repository" done rm -rf .git popd From b764dd9aa40f3dcc5664d920393d277b97db3923 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:31 -0700 Subject: [PATCH 41/67] housekeeping: shellcheck for tests/functional/flakes/unlocked-override.sh --- tests/functional/flakes/unlocked-override.sh | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index 680a1505c..a17a0c2af 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -7,26 +7,26 @@ requireGit flake1Dir=$TEST_ROOT/flake1 flake2Dir=$TEST_ROOT/flake2 -createGitRepo $flake1Dir -cat > $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/x.nix -git -C $flake1Dir add flake.nix x.nix -git -C $flake1Dir commit -m Initial +echo 123 > "$flake1Dir"/x.nix +git -C "$flake1Dir" add flake.nix x.nix +git -C "$flake1Dir" commit -m Initial -createGitRepo $flake2Dir -cat > $flake2Dir/flake.nix < "$flake2Dir"/flake.nix < $flake1Dir/x.nix +echo 456 > "$flake1Dir"/x.nix -[[ $(nix eval --json $flake2Dir#x --override-input flake1 $TEST_ROOT/flake1) = 456 ]] +[[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] From cd46ec17f9bc353ba3b604e5291202e8c5aa175d Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:32 -0700 Subject: [PATCH 42/67] housekeeping: shellcheck for tests/functional/function-trace.sh --- tests/functional/function-trace.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/function-trace.sh b/tests/functional/function-trace.sh index 71f18b67f..7524afdf2 100755 --- a/tests/functional/function-trace.sh +++ b/tests/functional/function-trace.sh @@ -21,12 +21,12 @@ expect_trace() { <(echo "$expect") \ <(echo "$actual") ) && result=0 || result=$? - if [ $result -eq 0 ]; then + if [ "$result" -eq 0 ]; then echo " ok." else echo " failed. difference:" echo "$msg" - return $result + return "$result" fi } From d1c476865a3cab6527f8f04aba8e285ff0ddd48c Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:49 -0700 Subject: [PATCH 43/67] housekeeping: shellcheck for tests/functional/gc-runtime.sh --- tests/functional/gc-runtime.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/functional/gc-runtime.sh b/tests/functional/gc-runtime.sh index 2ee72b61e..b5f6f769c 100755 --- a/tests/functional/gc-runtime.sh +++ b/tests/functional/gc-runtime.sh @@ -12,27 +12,27 @@ esac set -m # enable job control, needed for kill profiles="$NIX_STATE_DIR"/profiles -rm -rf $profiles +rm -rf "$profiles" -nix-env -p $profiles/test -f ./gc-runtime.nix -i gc-runtime +nix-env -p "$profiles/test" -f ./gc-runtime.nix -i gc-runtime -outPath=$(nix-env -p $profiles/test -q --no-name --out-path gc-runtime) -echo $outPath +outPath=$(nix-env -p "$profiles/test" -q --no-name --out-path gc-runtime) +echo "$outPath" echo "backgrounding program..." -$profiles/test/program & +"$profiles"/test/program & sleep 2 # hack - wait for the program to get started child=$! echo PID=$child -nix-env -p $profiles/test -e gc-runtime -nix-env -p $profiles/test --delete-generations old +nix-env -p "$profiles/test" -e gc-runtime +nix-env -p "$profiles/test" --delete-generations old nix-store --gc kill -- -$child -if ! test -e $outPath; then +if ! test -e "$outPath"; then echo "running program was garbage collected!" exit 1 fi From 1c9336098916d23d1510efb848ecdec8ce6eb70f Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:54 -0700 Subject: [PATCH 44/67] housekeeping: shellcheck for tests/functional/hash-path.sh --- tests/functional/hash-path.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/functional/hash-path.sh b/tests/functional/hash-path.sh index 12605ef71..86d782a95 100755 --- a/tests/functional/hash-path.sh +++ b/tests/functional/hash-path.sh @@ -3,7 +3,7 @@ source common.sh try () { - printf "%s" "$2" > $TEST_ROOT/vector + printf "%s" "$2" > "$TEST_ROOT/vector" hash="$(nix-hash --flat ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")" if ! (( "${NO_TEST_CLASSIC-}" )) && test "$hash" != "$3"; then echo "try nix-hash: hash $1, expected $3, got $hash" @@ -61,7 +61,7 @@ NO_TEST_NIX_COMMAND=1 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e NO_TEST_CLASSIC=1 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" try2 () { - hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) + hash=$(nix-hash --type "$1" "$TEST_ROOT/hash-path") if test "$hash" != "$2"; then echo "try nix-hash; hash $1, expected $2, got $hash" exit 1 @@ -73,22 +73,22 @@ try2 () { fi } -rm -rf $TEST_ROOT/hash-path -mkdir $TEST_ROOT/hash-path -echo "Hello World" > $TEST_ROOT/hash-path/hello +rm -rf "$TEST_ROOT/hash-path" +mkdir "$TEST_ROOT/hash-path" +echo "Hello World" > "$TEST_ROOT/hash-path/hello" try2 md5 "ea9b55537dd4c7e104515b2ccfaf4100" # Execute bit matters. -chmod +x $TEST_ROOT/hash-path/hello +chmod +x "$TEST_ROOT/hash-path/hello" try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" # Mtime and other bits don't. -touch -r . $TEST_ROOT/hash-path/hello -chmod 744 $TEST_ROOT/hash-path/hello +touch -r . "$TEST_ROOT/hash-path/hello" +chmod 744 "$TEST_ROOT/hash-path/hello" try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" # File type (e.g., symlink) does. -rm $TEST_ROOT/hash-path/hello -ln -s x $TEST_ROOT/hash-path/hello +rm "$TEST_ROOT/hash-path/hello" +ln -s x "$TEST_ROOT/hash-path/hello" try2 md5 "f78b733a68f5edbdf9413899339eaa4a" From 2d467b4731365065b22456066bcdf4cc1d548c13 Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:33:59 -0700 Subject: [PATCH 45/67] housekeeping: shellcheck for tests/functional/import-derivation.sh --- tests/functional/import-derivation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/import-derivation.sh b/tests/functional/import-derivation.sh index 53efa1f5d..96cc30646 100755 --- a/tests/functional/import-derivation.sh +++ b/tests/functional/import-derivation.sh @@ -11,4 +11,4 @@ fi outPath=$(nix-build ./import-derivation.nix --no-out-link) -[ "$(cat $outPath)" = FOO579 ] +[ "$(cat "$outPath")" = FOO579 ] From 1afac8fbbc49dabe793e5f6c62ffa6ee334a169e Mon Sep 17 00:00:00 2001 From: Cameron Dart <8763518+SkamDart@users.noreply.github.com> Date: Tue, 4 Jun 2024 23:37:45 -0700 Subject: [PATCH 46/67] remove tests from pre-commit excludes --- maintainers/flake-module.nix | 38 ------------------------------------ 1 file changed, 38 deletions(-) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 261da1766..b1d21e8fe 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -507,49 +507,30 @@ ''^scripts/install-nix-from-closure\.sh$'' ''^scripts/install-systemd-multi-user\.sh$'' ''^src/nix/get-env\.sh$'' - ''^tests/functional/binary-cache-build-remote\.sh$'' - ''^tests/functional/binary-cache\.sh$'' - ''^tests/functional/brotli\.sh$'' - ''^tests/functional/build-delete\.sh$'' - ''^tests/functional/build-dry\.sh$'' ''^tests/functional/build\.sh$'' - ''^tests/functional/ca/build-cache\.sh$'' ''^tests/functional/ca/build-dry\.sh$'' ''^tests/functional/ca/build-with-garbage-path\.sh$'' - ''^tests/functional/ca/build\.sh$'' ''^tests/functional/ca/common\.sh$'' ''^tests/functional/ca/concurrent-builds\.sh$'' - ''^tests/functional/ca/derivation-json\.sh$'' - ''^tests/functional/ca/duplicate-realisation-in-closure\.sh$'' ''^tests/functional/ca/eval-store\.sh$'' ''^tests/functional/ca/gc\.sh$'' ''^tests/functional/ca/import-derivation\.sh$'' ''^tests/functional/ca/new-build-cmd\.sh$'' - ''^tests/functional/ca/nix-copy\.sh$'' - ''^tests/functional/ca/nix-run\.sh$'' ''^tests/functional/ca/nix-shell\.sh$'' ''^tests/functional/ca/post-hook\.sh$'' ''^tests/functional/ca/recursive\.sh$'' ''^tests/functional/ca/repl\.sh$'' ''^tests/functional/ca/selfref-gc\.sh$'' - ''^tests/functional/ca/signatures\.sh$'' - ''^tests/functional/ca/substitute\.sh$'' ''^tests/functional/ca/why-depends\.sh$'' - ''^tests/functional/case-hack\.sh$'' - ''^tests/functional/check-refs\.sh$'' - ''^tests/functional/check-reqs\.sh$'' ''^tests/functional/check\.sh$'' - ''^tests/functional/chroot-store\.sh$'' ''^tests/functional/common/vars-and-functions\.sh$'' ''^tests/functional/completions\.sh$'' - ''^tests/functional/compression-levels\.sh$'' ''^tests/functional/compute-levels\.sh$'' ''^tests/functional/config\.sh$'' ''^tests/functional/db-migration\.sh$'' ''^tests/functional/debugger\.sh$'' ''^tests/functional/dependencies\.builder0\.sh$'' ''^tests/functional/dependencies\.sh$'' - ''^tests/functional/derivation-json\.sh$'' ''^tests/functional/dump-db\.sh$'' ''^tests/functional/dyn-drv/build-built-drv\.sh$'' ''^tests/functional/dyn-drv/common\.sh$'' @@ -557,10 +538,8 @@ ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' - ''^tests/functional/dyn-drv/text-hashed-output\.sh$'' ''^tests/functional/eval-store\.sh$'' ''^tests/functional/eval\.sh$'' - ''^tests/functional/experimental-features\.sh$'' ''^tests/functional/export-graph\.sh$'' ''^tests/functional/export\.sh$'' ''^tests/functional/extra-sandbox-profile\.sh$'' @@ -570,49 +549,32 @@ ''^tests/functional/fetchGitSubmodules\.sh$'' ''^tests/functional/fetchGitVerification\.sh$'' ''^tests/functional/fetchMercurial\.sh$'' - ''^tests/functional/fetchPath\.sh$'' - ''^tests/functional/fetchTree-file\.sh$'' ''^tests/functional/fetchurl\.sh$'' - ''^tests/functional/filter-source\.sh$'' ''^tests/functional/fixed\.builder1\.sh$'' ''^tests/functional/fixed\.builder2\.sh$'' ''^tests/functional/fixed\.sh$'' - ''^tests/functional/flakes/absolute-attr-paths\.sh$'' ''^tests/functional/flakes/absolute-paths\.sh$'' - ''^tests/functional/flakes/build-paths\.sh$'' - ''^tests/functional/flakes/bundle\.sh$'' ''^tests/functional/flakes/check\.sh$'' - ''^tests/functional/flakes/circular\.sh$'' ''^tests/functional/flakes/common\.sh$'' ''^tests/functional/flakes/config\.sh$'' ''^tests/functional/flakes/develop\.sh$'' - ''^tests/functional/flakes/flake-in-submodule\.sh$'' ''^tests/functional/flakes/flakes\.sh$'' ''^tests/functional/flakes/follow-paths\.sh$'' - ''^tests/functional/flakes/init\.sh$'' - ''^tests/functional/flakes/inputs\.sh$'' - ''^tests/functional/flakes/mercurial\.sh$'' ''^tests/functional/flakes/prefetch\.sh$'' ''^tests/functional/flakes/run\.sh$'' - ''^tests/functional/flakes/search-root\.sh$'' ''^tests/functional/flakes/show\.sh$'' - ''^tests/functional/flakes/unlocked-override\.sh$'' ''^tests/functional/fmt\.sh$'' ''^tests/functional/fmt\.simple\.sh$'' - ''^tests/functional/function-trace\.sh$'' ''^tests/functional/gc-auto\.sh$'' ''^tests/functional/gc-concurrent\.builder\.sh$'' ''^tests/functional/gc-concurrent\.sh$'' ''^tests/functional/gc-concurrent2\.builder\.sh$'' ''^tests/functional/gc-non-blocking\.sh$'' - ''^tests/functional/gc-runtime\.sh$'' ''^tests/functional/gc\.sh$'' ''^tests/functional/git-hashing/common\.sh$'' ''^tests/functional/git-hashing/simple\.sh$'' ''^tests/functional/hash-convert\.sh$'' - ''^tests/functional/hash-path\.sh$'' ''^tests/functional/help\.sh$'' - ''^tests/functional/import-derivation\.sh$'' ''^tests/functional/impure-derivations\.sh$'' ''^tests/functional/impure-env\.sh$'' ''^tests/functional/impure-eval\.sh$'' From d8ae28617db90fb500103fccb7df0b3499d22099 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jun 2024 12:42:38 -0400 Subject: [PATCH 47/67] Try to fix quotes that don't go to end with sed --- tests/functional/ca/substitute.sh | 2 +- tests/functional/common/init.sh | 2 +- tests/functional/dependencies.sh | 2 +- tests/functional/fetchGitRefs.sh | 2 +- tests/functional/gc-concurrent.sh | 4 ++-- tests/functional/gc.sh | 6 +++--- tests/functional/local-overlay-store/gc-inner.sh | 6 +++--- tests/functional/multiple-outputs.sh | 2 +- tests/functional/readfile-context.sh | 2 +- tests/functional/referrers.sh | 2 +- tests/functional/simple.sh | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index 76cedb4bc..9728470f0 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -7,7 +7,7 @@ source common.sh # shellcheck disable=SC1111 needLocalStore "“--no-require-sigs” can’t be used with the daemon" -rm -rf "$TEST_ROOT"/binary_cache +rm -rf "$TEST_ROOT/binary_cache" export REMOTE_STORE_DIR=$TEST_ROOT/binary_cache export REMOTE_STORE=file://$REMOTE_STORE_DIR diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index dda1ecd41..4f2a393af 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -13,7 +13,7 @@ mkdir "$TEST_HOME" mkdir "$NIX_STORE_DIR" mkdir "$NIX_LOCALSTATE_DIR" -mkdir -p "$NIX_LOG_DIR"/drvs +mkdir -p "$NIX_LOG_DIR/drvs" mkdir "$NIX_STATE_DIR" mkdir "$NIX_CONF_DIR" diff --git a/tests/functional/dependencies.sh b/tests/functional/dependencies.sh index 5922a1f98..1b266935d 100755 --- a/tests/functional/dependencies.sh +++ b/tests/functional/dependencies.sh @@ -33,7 +33,7 @@ nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' echo "output path is $outPath" -text=$(cat "$outPath"/foobar) +text=$(cat "$outPath/foobar") if test "$text" != "FOOBAR"; then exit 1; fi deps=$(nix-store -quR "$drvPath") diff --git a/tests/functional/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh index b17cc2090..9373146cd 100755 --- a/tests/functional/fetchGitRefs.sh +++ b/tests/functional/fetchGitRefs.sh @@ -14,7 +14,7 @@ git init "$repo" git -C "$repo" config user.email "foobar@example.com" git -C "$repo" config user.name "Foobar" -echo utrecht > "$repo"/hello +echo utrecht > "$repo/hello" git -C "$repo" add hello git -C "$repo" commit -m 'Bla1' diff --git a/tests/functional/gc-concurrent.sh b/tests/functional/gc-concurrent.sh index 67ea3dc74..128271287 100755 --- a/tests/functional/gc-concurrent.sh +++ b/tests/functional/gc-concurrent.sh @@ -20,8 +20,8 @@ outPath3=$(nix-store -r $drvPath3) touch $outPath3.lock rm -f "$NIX_STATE_DIR"/gcroots/foo* -ln -s $drvPath2 "$NIX_STATE_DIR"/gcroots/foo -ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2 +ln -s $drvPath2 "$NIX_STATE_DIR/gcroots/foo" +ln -s $outPath3 "$NIX_STATE_DIR/gcroots/foo2" # Start build #1 in the background. It starts immediately. nix-store -rvv "$drvPath1" & diff --git a/tests/functional/gc.sh b/tests/functional/gc.sh index 1f216ebc7..7594312bb 100755 --- a/tests/functional/gc.sh +++ b/tests/functional/gc.sh @@ -8,8 +8,8 @@ drvPath=$(nix-instantiate dependencies.nix) outPath=$(nix-store -rvv "$drvPath") # Set a GC root. -rm -f "$NIX_STATE_DIR"/gcroots/foo -ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo +rm -f "$NIX_STATE_DIR/gcroots/foo" +ln -sf $outPath "$NIX_STATE_DIR/gcroots/foo" [ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] @@ -42,7 +42,7 @@ cat $outPath/reference-to-input-2/bar # Check that the derivation has been GC'd. if test -e $drvPath; then false; fi -rm "$NIX_STATE_DIR"/gcroots/foo +rm "$NIX_STATE_DIR/gcroots/foo" nix-collect-garbage diff --git a/tests/functional/local-overlay-store/gc-inner.sh b/tests/functional/local-overlay-store/gc-inner.sh index ea92154d2..687fed897 100644 --- a/tests/functional/local-overlay-store/gc-inner.sh +++ b/tests/functional/local-overlay-store/gc-inner.sh @@ -20,8 +20,8 @@ outPath=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg # Set a GC root. mkdir -p "$stateB" -rm -f "$stateB"/gcroots/foo -ln -sf $outPath "$stateB"/gcroots/foo +rm -f "$stateB/gcroots/foo" +ln -sf $outPath "$stateB/gcroots/foo" [ "$(nix-store -q --roots $outPath)" = "$stateB/gcroots/foo -> $outPath" ] @@ -46,7 +46,7 @@ nix-collect-garbage # Check that the root and its dependencies haven't been deleted. cat "$storeBRoot/$outPath" -rm "$stateB"/gcroots/foo +rm "$stateB/gcroots/foo" nix-collect-garbage diff --git a/tests/functional/multiple-outputs.sh b/tests/functional/multiple-outputs.sh index af9f8af72..31ce2a3a4 100755 --- a/tests/functional/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -35,7 +35,7 @@ outPath=$(nix-store -q $drvPath) echo "building b..." outPath=$(nix-build multiple-outputs.nix -A b --no-out-link) echo "output path is $outPath" -[ "$(cat "$outPath"/file)" = "success" ] +[ "$(cat "$outPath/file")" = "success" ] # Test nix-build on a derivation with multiple outputs. outPath1=$(nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result) diff --git a/tests/functional/readfile-context.sh b/tests/functional/readfile-context.sh index 76fad9349..d0644471d 100755 --- a/tests/functional/readfile-context.sh +++ b/tests/functional/readfile-context.sh @@ -7,7 +7,7 @@ clearStore outPath=$(nix-build --no-out-link readfile-context.nix) # Set a GC root. -ln -s $outPath "$NIX_STATE_DIR"/gcroots/foo +ln -s $outPath "$NIX_STATE_DIR/gcroots/foo" # Check that file exists. [ "$(cat $(cat $outPath))" = "Hello World!" ] diff --git a/tests/functional/referrers.sh b/tests/functional/referrers.sh index 898032e42..0fda97378 100755 --- a/tests/functional/referrers.sh +++ b/tests/functional/referrers.sh @@ -31,7 +31,7 @@ echo "registering..." nix-store --register-validity < $TEST_ROOT/reg_info echo "collecting garbage..." -ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref +ln -sfn $reference "$NIX_STATE_DIR/gcroots/ref" nix-store --gc if [ -n "$(type -p sqlite3)" -a "$(sqlite3 $NIX_STATE_DIR/db/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index 846738cbd..4e7d37f59 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -14,7 +14,7 @@ echo "output path is $outPath" (! [ -w $outPath ]) -text=$(cat "$outPath"/hello) +text=$(cat "$outPath/hello") if test "$text" != "Hello World!"; then exit 1; fi # Directed delete: $outPath is not reachable from a root, so it should From 33241887d1d30fb1d5b679162b3f26e67567d7fb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jun 2024 17:47:54 -0400 Subject: [PATCH 48/67] More quote coalescing --- tests/functional/flakes/init.sh | 4 ++-- tests/functional/flakes/mercurial.sh | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/functional/flakes/init.sh b/tests/functional/flakes/init.sh index 6787b9e79..9e484f71c 100755 --- a/tests/functional/flakes/init.sh +++ b/tests/functional/flakes/init.sh @@ -8,8 +8,8 @@ templatesDir=$TEST_ROOT/templates flakeDir=$TEST_ROOT/flake nixpkgsDir=$TEST_ROOT/nixpkgs -nix registry add --registry "$registry" templates git+file://"$templatesDir" -nix registry add --registry "$registry" nixpkgs git+file://"$nixpkgsDir" +nix registry add --registry "$registry" templates "git+file://$templatesDir" +nix registry add --registry "$registry" nixpkgs "git+file://$nixpkgsDir" createGitRepo "$nixpkgsDir" createSimpleGitFlake "$nixpkgsDir" diff --git a/tests/functional/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh index 1b095790b..b9045bf6b 100755 --- a/tests/functional/flakes/mercurial.sh +++ b/tests/functional/flakes/mercurial.sh @@ -9,7 +9,7 @@ mkdir -p "$flake1Dir" writeSimpleFlake "$flake1Dir" hg init "$flake1Dir" -nix registry add --registry "$registry" flake1 hg+file://"$flake1Dir" +nix registry add --registry "$registry" flake1 "hg+file://$flake1Dir" flake2Dir=$TEST_ROOT/flake-hg2 mkdir -p "$flake2Dir" @@ -22,24 +22,24 @@ hg commit --config ui.username=foobar@example.org "$flake1Dir" -m 'Initial commi hg add "$flake2Dir"/flake.nix hg commit --config ui.username=foobar@example.org "$flake2Dir" -m 'Initial commit' -nix build -o "$TEST_ROOT/result" hg+file://"$flake2Dir" +nix build -o "$TEST_ROOT/result" "hg+file://$flake2Dir" [[ -e $TEST_ROOT/result/hello ]] -(! nix flake metadata --json hg+file://"$flake2Dir" | jq -e -r .revision) +(! nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revision) -nix eval hg+file://"$flake2Dir"#expr +nix eval "hg+file://$flake2Dir"#expr -nix eval hg+file://"$flake2Dir"#expr +nix eval "hg+file://$flake2Dir"#expr -(! nix eval hg+file://"$flake2Dir"#expr --no-allow-dirty) +(! nix eval "hg+file://$flake2Dir"#expr --no-allow-dirty) -(! nix flake metadata --json hg+file://"$flake2Dir" | jq -e -r .revision) +(! nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revision) hg commit --config ui.username=foobar@example.org "$flake2Dir" -m 'Add lock file' -nix flake metadata --json hg+file://"$flake2Dir" --refresh | jq -e -r .revision -nix flake metadata --json hg+file://"$flake2Dir" -[[ $(nix flake metadata --json hg+file://"$flake2Dir" | jq -e -r .revCount) = 1 ]] +nix flake metadata --json "hg+file://$flake2Dir" --refresh | jq -e -r .revision +nix flake metadata --json "hg+file://$flake2Dir" +[[ $(nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revCount) = 1 ]] -nix build -o "$TEST_ROOT/result" hg+file://"$flake2Dir" --no-registries --no-allow-dirty -nix build -o "$TEST_ROOT/result" hg+file://"$flake2Dir" --no-use-registries --no-allow-dirty +nix build -o "$TEST_ROOT/result" "hg+file://$flake2Dir" --no-registries --no-allow-dirty +nix build -o "$TEST_ROOT/result" "hg+file://$flake2Dir" --no-use-registries --no-allow-dirty From 28d2af4ea69fbc7f1803de4a3ab1e6dd49dfe519 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Jun 2024 09:28:27 -0400 Subject: [PATCH 49/67] Build `nix-util` with Meson MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The idea is two-fold: - Replace autotools with Meson - Build each library in its own derivation The interaction of these two features is that Meson's "subprojects" feature (https://mesonbuild.com/Subprojects) allows us to have single dev shell for building all libraries still, while also building things separately. This allows us to break up the build without a huge productivity lost. I tested the Linux native build, and NetBSD and Windows cross builds. Also do some clean ups of the Flake in the process of supporting new jobs. Special thanks to everyone that has worked on a Meson port so far, @p01arst0rm and @Qyriad in particular. Co-Authored-By: p01arst0rm Co-Authored-By: Artemis Tosini Co-Authored-By: Artemis Tosini Co-Authored-By: Felix Uhl Co-Authored-By: Jade Lovelace Co-Authored-By: Lunaphied Co-Authored-By: Maximilian Bosch Co-Authored-By: Pierre Bourdon Co-Authored-By: Qyriad Co-Authored-By: Rebecca Turner Co-Authored-By: Winter Co-Authored-By: eldritch horrors Co-Authored-By: jade Co-Authored-By: julia Co-Authored-By: rebecca “wiggles” turner Co-Authored-By: wiggles dog Co-Authored-By: fricklerhandwerk Co-authored-By: Eli Schwartz Co-authored-by: Robert Hensing --- .gitignore | 2 + flake.nix | 75 +++++--- {build => maintainers}/hydra.nix | 12 +- meson.build | 9 + package.nix | 3 +- src/libutil/.version | 1 + src/libutil/linux/meson.build | 11 ++ src/libutil/meson.build | 288 +++++++++++++++++++++++++++++++ src/libutil/meson.options | 5 + src/libutil/package.nix | 144 ++++++++++++++++ src/libutil/unix/meson.build | 17 ++ src/libutil/windows/meson.build | 19 ++ 12 files changed, 553 insertions(+), 33 deletions(-) rename {build => maintainers}/hydra.nix (93%) create mode 100644 meson.build create mode 120000 src/libutil/.version create mode 100644 src/libutil/linux/meson.build create mode 100644 src/libutil/meson.build create mode 100644 src/libutil/meson.options create mode 100644 src/libutil/package.nix create mode 100644 src/libutil/unix/meson.build create mode 100644 src/libutil/windows/meson.build diff --git a/.gitignore b/.gitignore index 28f853715..1bf540ba2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ perl/Makefile.config /svn-revision /libtool /config/config.* +# Default meson build dir +/build # /doc/manual/ /doc/manual/*.1 diff --git a/flake.nix b/flake.nix index 5dbc554fc..0e29c6e6f 100644 --- a/flake.nix +++ b/flake.nix @@ -160,21 +160,23 @@ }; }); - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; + nix-util = final.callPackage ./src/libutil/package.nix { + inherit + fileset + stdenv + officialRelease + versionSuffix + ; + }; - in final.callPackage ./package.nix { + nix = + final.callPackage ./package.nix { inherit fileset stdenv + officialRelease versionSuffix ; - officialRelease = false; boehmgc = final.boehmgc-nix; libgit2 = final.libgit2-nix; libseccomp = final.libseccomp-nix; @@ -203,7 +205,7 @@ # 'nix.perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = import ./build/hydra.nix { + hydraJobs = import ./maintainers/hydra.nix { inherit inputs binaryTarball @@ -236,11 +238,29 @@ } // devFlake.checks.${system} or {} ); - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + packages = forAllSystems (system: { + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + } // lib.concatMapAttrs + (pkgName: {}: { + "${pkgName}" = nixpkgsFor.${system}.native.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName}; + } // lib.concatMapAttrs + (crossSystem: {}: { + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}; + }) + (lib.genAttrs crossSystems (_: { })) + // lib.concatMapAttrs + (stdenvName: {}: { + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName}; + }) + (lib.genAttrs stdenvs (_: { }))) + { + "nix" = { }; + "nix-util" = { }; + } + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; @@ -255,18 +275,7 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: @@ -274,6 +283,11 @@ modular = devFlake.getSystem stdenv.buildPlatform.system; in { pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -288,12 +302,19 @@ src = null; env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.boost}/lib"; # For `make format`, to work without installing pre-commit _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; }; + inherit (pkgs.nix-util) mesonFlags; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nix-util.nativeBuildInputs ++ [ modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" diff --git a/build/hydra.nix b/maintainers/hydra.nix similarity index 93% rename from build/hydra.nix rename to maintainers/hydra.nix index 857b7f1f0..2541b061e 100644 --- a/build/hydra.nix +++ b/maintainers/hydra.nix @@ -32,6 +32,8 @@ let doBuild = false; }; + + forAllPackages = lib.genAttrs [ "nix" "nix-util" ]; in { # Binary package for various platforms. @@ -39,10 +41,12 @@ in shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.${pkgName})); - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs [ "x86_64-linux" ] (system: self.packages.${system}."nix-${crossSystem}")); + buildCross = forAllPackages (pkgName: + forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}))); buildNoGc = forAllSystems (system: self.packages.${system}.nix.override { enableGC = false; } @@ -76,7 +80,7 @@ in binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: forAllCrossSystems (crossSystem: binaryTarball - self.packages.${system}."nix-${crossSystem}" + nixpkgsFor.${system}.cross.${crossSystem}.nix nixpkgsFor.${system}.cross.${crossSystem})); # The first half of the installation script. This is uploaded diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..fc6441877 --- /dev/null +++ b/meson.build @@ -0,0 +1,9 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +subproject('libutil') diff --git a/package.nix b/package.nix index cf1654c6a..677ee73c7 100644 --- a/package.nix +++ b/package.nix @@ -385,8 +385,7 @@ in { separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/src/libutil/.version b/src/libutil/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil/linux/meson.build b/src/libutil/linux/meson.build new file mode 100644 index 000000000..a1ded76ca --- /dev/null +++ b/src/libutil/linux/meson.build @@ -0,0 +1,11 @@ +sources += files( + 'cgroup.cc', + 'namespaces.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'cgroup.hh', + 'namespaces.hh', +) diff --git a/src/libutil/meson.build b/src/libutil/meson.build new file mode 100644 index 000000000..8644c69f4 --- /dev/null +++ b/src/libutil/meson.build @@ -0,0 +1,288 @@ +project('nix-util', 'cpp', + version : run_command('cat', './.version', check : true).stdout().strip(), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 0.64.0', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +deps_private = [ ] +deps_public = [ ] +deps_other = [ ] + +configdata = configuration_data() + +# Check for each of these functions, and create a define like `#define HAVE_LUTIMES 1`. +check_funcs = [ + # Optionally used for changing the mtime of symlinks. + 'lutimes', + 'pipe2', + 'posix_fallocate', + 'strsignal', + 'sysconf', +] +foreach funcspec : check_funcs + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) +endforeach + +# Conditional to work around https://github.com/mesonbuild/meson/issues/13293 +if host_machine.system() != 'windows' and cxx.get_id() == 'gcc' + deps_private += dependency('threads') +endif + +if host_machine.system() == 'windows' + socket = cxx.find_library('ws2_32') + deps_other += socket +elif host_machine.system() == 'sunos' + socket = cxx.find_library('socket') + network_service_library = cxx.find_library('nsl') + deps_other += [socket, network_service_library] +endif + +boost = dependency( + 'boost', + modules : ['context', 'coroutine'], +) +# Actually public, but wrong type of dep for pkg config +deps_other += boost + +openssl = dependency( + 'libcrypto', + 'openssl', + version : '>= 1.1.1', +) +deps_private += openssl + +libarchive = dependency('libarchive', version : '>= 3.1.2') +deps_public += libarchive +if get_option('default_library') == 'static' + # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed + add_project_arguments('-lz', language : 'cpp') +endif + +sodium = dependency('libsodium', 'sodium') +deps_private += sodium + +brotli = [ + dependency('libbrotlicommon'), + dependency('libbrotlidec'), + dependency('libbrotlienc'), +] +deps_private += brotli + +# cpuid only makes sense on x86_64 +cpuid_required = host_machine.cpu_family() == 'x86_64' ? get_option('cpuid') : false +cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required) +configdata.set('HAVE_LIBCPUID', cpuid.found().to_int()) +deps_private += cpuid + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +config_util_h = configure_file( + configuration : configdata, + output : 'config-util.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.h', + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + # FIXME: remove when we get meson 1.4.0 which will default this to on for us: + # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions + '-D_GLIBCXX_ASSERTIONS=1', + language : 'cpp', +) + +sources = files( + 'archive.cc', + 'args.cc', + 'canon-path.cc', + 'compression.cc', + 'compute-levels.cc', + 'config.cc', + 'current-process.cc', + 'english.cc', + 'environment-variables.cc', + 'error.cc', + 'exit.cc', + 'experimental-features.cc', + 'file-content-address.cc', + 'file-descriptor.cc', + 'file-system.cc', + 'fs-sink.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'memory-source-accessor.cc', + 'position.cc', + 'posix-source-accessor.cc', + 'references.cc', + 'serialise.cc', + 'signature/local-keys.cc', + 'signature/signer.cc', + 'source-accessor.cc', + 'source-path.cc', + 'suggestions.cc', + 'tarfile.cc', + 'terminal.cc', + 'thread-pool.cc', + 'unix-domain-socket.cc', + 'url.cc', + 'users.cc', + 'util.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_util_h] + files( + 'abstract-setting-to-json.hh', + 'ansicolor.hh', + 'archive.hh', + 'args.hh', + 'args/root.hh', + 'callback.hh', + 'canon-path.hh', + 'chunked-vector.hh', + 'closure.hh', + 'comparator.hh', + 'compression.hh', + 'compute-levels.hh', + 'config-impl.hh', + 'config.hh', + 'current-process.hh', + 'english.hh', + 'environment-variables.hh', + 'error.hh', + 'exit.hh', + 'experimental-features.hh', + 'file-content-address.hh', + 'file-descriptor.hh', + 'file-path-impl.hh', + 'file-path.hh', + 'file-system.hh', + 'finally.hh', + 'fmt.hh', + 'fs-sink.hh', + 'git.hh', + 'hash.hh', + 'hilite.hh', + 'json-impls.hh', + 'json-utils.hh', + 'logging.hh', + 'lru-cache.hh', + 'memory-source-accessor.hh', + 'muxable-pipe.hh', + 'pool.hh', + 'position.hh', + 'posix-source-accessor.hh', + 'processes.hh', + 'ref.hh', + 'references.hh', + 'regex-combinators.hh', + 'repair-flag.hh', + 'serialise.hh', + 'signals.hh', + 'signature/local-keys.hh', + 'signature/signer.hh', + 'source-accessor.hh', + 'source-path.hh', + 'split.hh', + 'suggestions.hh', + 'sync.hh', + 'tarfile.hh', + 'terminal.hh', + 'thread-pool.hh', + 'topo-sort.hh', + 'types.hh', + 'unix-domain-socket.hh', + 'url-parts.hh', + 'url.hh', + 'users.hh', + 'util.hh', + 'variant-wrapper.hh', + 'xml-writer.hh', +) + +if host_machine.system() == 'linux' + subdir('linux') +endif + +if host_machine.system() == 'windows' + subdir('windows') +else + subdir('unix') +endif + +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif + +libutil = library( + 'nixutil', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = ['-lboost_context', '-lboost_coroutine'] +if host_machine.system() == 'windows' + # `libraries_private` cannot contain ad-hoc dependencies (from + # `find_library), so we need to do this manually + libraries_private += ['-lws2_32'] +endif + +import('pkgconfig').generate( + libutil, + filebase : 'nix-util', + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : deps_public, + requires_private : deps_private, + # avoid absolute paths, use vendored ones + libraries_private : libraries_private, +) + +nix_util = declare_dependency( + include_directories : include_dirs, + link_with : libutil, + compile_args : ['-std=c++2a'], + dependencies : [], +) +meson.override_dependency('nix-util', nix_util) diff --git a/src/libutil/meson.options b/src/libutil/meson.options new file mode 100644 index 000000000..21883af01 --- /dev/null +++ b/src/libutil/meson.options @@ -0,0 +1,5 @@ +# vim: filetype=meson + +option('cpuid', type : 'feature', + description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)', +) diff --git a/src/libutil/package.nix b/src/libutil/package.nix new file mode 100644 index 000000000..6575b37a6 --- /dev/null +++ b/src/libutil/package.nix @@ -0,0 +1,144 @@ +{ lib +, stdenv +, releaseTools +, fileset + +, meson +, ninja +, pkg-config + +, boost +, brotli +, libarchive +, libcpuid +, libsodium +, nlohmann_json +, openssl + +# Configuration Options + +, versionSuffix ? "" +, officialRelease ? false + +# Check test coverage of Nix. Probably want to use with at least +# one of `doCheck` or `doInstallCheck` enabled. +, withCoverageChecks ? false +}: + +let + version = lib.fileContents ./.version + versionSuffix; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: { + pname = "nix-util"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions [ + ./meson.build + ./meson.options + ./linux/meson.build + ./unix/meson.build + ./windows/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + }; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + boost + brotli + libsodium + openssl + ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + ; + + propagatedBuildInputs = [ + libarchive + nlohmann_json + ]; + + disallowedReferences = [ boost ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix + '' + echo ${version} > .version + '' + # Copy libboost_context so we don't get all of Boost in our closure. + + # https://github.com/NixOS/nixpkgs/issues/45462 + + lib.optionalString (!stdenv.hostPlatform.isStatic) ('' + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + '' + lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + '' + lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + '' + ); + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + postInstall = + # Remove absolute path to boost libs + '' + sed -i "$out/lib/pkgconfig/nix-util.pc" -e 's, ${lib.getLib boost}[^ ]*,,g' + '' + + lib.optionalString stdenv.isDarwin '' + install_name_tool \ + -change ${boost}/lib/libboost_context.dylib \ + $out/lib/libboost_context.dylib \ + $out/lib/libnixutil.dylib + ''; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = [ "fortify" ]; +}) diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build new file mode 100644 index 000000000..38e5cd3aa --- /dev/null +++ b/src/libutil/unix/meson.build @@ -0,0 +1,17 @@ +sources += files( + 'environment-variables.cc', + 'file-descriptor.cc', + 'file-path.cc', + 'file-system.cc', + 'muxable-pipe.cc', + 'processes.cc', + 'signals.cc', + 'users.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'monitor-fd.hh', + 'signals-impl.hh', +) diff --git a/src/libutil/windows/meson.build b/src/libutil/windows/meson.build new file mode 100644 index 000000000..00320877f --- /dev/null +++ b/src/libutil/windows/meson.build @@ -0,0 +1,19 @@ +sources += files( + 'environment-variables.cc', + 'file-descriptor.cc', + 'file-path.cc', + 'file-system.cc', + 'muxable-pipe.cc', + 'processes.cc', + 'users.cc', + 'windows-async-pipe.cc', + 'windows-error.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'signals-impl.hh', + 'windows-async-pipe.hh', + 'windows-error.hh', +) From 25a98949431af72bfdc00cd7e4e6bab18cdd1ec5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jun 2024 18:33:28 -0400 Subject: [PATCH 50/67] hash: Compare hash algo second for back compat Previously (in cfc18a77395b5ef49763f77c3cc67a95f762a0eb), we forgot to compare the algo at all. This means we keep the same ordering as before by making the stuff we always have compared take priority. --- src/libutil/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2f2ed8138..7064e96e6 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -52,11 +52,11 @@ bool Hash::operator == (const Hash & h2) const std::strong_ordering Hash::operator <=> (const Hash & h) const { - if (auto cmp = algo <=> h.algo; cmp != 0) return cmp; if (auto cmp = hashSize <=> h.hashSize; cmp != 0) return cmp; for (unsigned int i = 0; i < hashSize; i++) { if (auto cmp = hash[i] <=> h.hash[i]; cmp != 0) return cmp; } + if (auto cmp = algo <=> h.algo; cmp != 0) return cmp; return std::strong_ordering::equivalent; } From 1dc7c8e59991b15a78504be52931f8561dcff32b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 13 Jun 2024 13:55:42 +0200 Subject: [PATCH 51/67] eval-fail-infinite-recursion-lambda: Reduce recursion depth This prevents the test from failing in environments with a smaller configured stack size. --- .../functional/lang/eval-fail-infinite-recursion-lambda.err.exp | 2 +- tests/functional/lang/eval-fail-infinite-recursion-lambda.flags | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 tests/functional/lang/eval-fail-infinite-recursion-lambda.flags diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp index 5d843d827..712dd75a8 100644 --- a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp @@ -29,7 +29,7 @@ error: | ^ 2| - (19997 duplicate frames omitted) + (197 duplicate frames omitted) error: stack overflow; max-call-depth exceeded at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14: diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags b/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags new file mode 100644 index 000000000..59e20ec9c --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags @@ -0,0 +1 @@ +--max-call-depth 100 \ No newline at end of file From ff87c1a318c8ea5ed8714e2d363d4e080fc93f02 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jun 2024 10:34:44 -0400 Subject: [PATCH 52/67] Put some file descriptor functions in unix and windows namespaces It is misleading when platform-specific functions are in the overall `nix` namespace. More namespaces also makes for nicer doxygen. --- src/libstore/unix/build/local-derivation-goal.cc | 4 ++-- src/libutil/file-descriptor.hh | 10 +++++++--- src/libutil/file-system.cc | 2 +- src/libutil/unix-domain-socket.cc | 2 +- src/libutil/unix/file-descriptor.cc | 8 ++++---- src/libutil/windows/file-descriptor.cc | 4 ++-- src/nix/unix/daemon.cc | 4 ++-- 7 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 16095cf5d..a99439738 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1500,7 +1500,7 @@ void LocalDerivationGoal::startDaemon() throw SysError("accepting connection"); } - closeOnExec(remote.get()); + unix::closeOnExec(remote.get()); debug("received daemon connection"); @@ -1961,7 +1961,7 @@ void LocalDerivationGoal::runChild() throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + unix::closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); #if __linux__ linux::setPersonality(drv->platform); diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index 84786e95a..492b67d74 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -140,6 +140,7 @@ public: }; #ifndef _WIN32 // Not needed on Windows, where we don't fork +namespace unix { /** * Close all file descriptors except those listed in the given set. @@ -152,13 +153,16 @@ void closeMostFDs(const std::set & exceptions); */ void closeOnExec(Descriptor fd); +} // namespace unix #endif -#ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 +#if defined(_WIN32) && _WIN32_WINNT >= 0x0600 +namespace windows ( + Path handleToPath(Descriptor handle); std::wstring handleToFileName(Descriptor handle); -# endif + +} // namespace windows #endif MakeError(EndOfFile, Error); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 919bf5d50..9d6142556 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -546,7 +546,7 @@ std::pair createTempFile(const Path & prefix) if (!fd) throw SysError("creating temporary file '%s'", tmpl); #ifndef _WIN32 - closeOnExec(fd.get()); + unix::closeOnExec(fd.get()); #endif return {std::move(fd), tmpl}; } diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc index 87914bb83..1707fdb75 100644 --- a/src/libutil/unix-domain-socket.cc +++ b/src/libutil/unix-domain-socket.cc @@ -24,7 +24,7 @@ AutoCloseFD createUnixDomainSocket() if (!fdSocket) throw SysError("cannot create Unix domain socket"); #ifndef _WIN32 - closeOnExec(fdSocket.get()); + unix::closeOnExec(fdSocket.get()); #endif return fdSocket; } diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index a74f16ce1..963e75785 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -110,8 +110,8 @@ void Pipe::create() if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); #else if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); + unix::closeOnExec(fds[0]); + unix::closeOnExec(fds[1]); #endif readSide = fds[0]; writeSide = fds[1]; @@ -120,7 +120,7 @@ void Pipe::create() ////////////////////////////////////////////////////////////////////// -void closeMostFDs(const std::set & exceptions) +void unix::closeMostFDs(const std::set & exceptions) { #if __linux__ try { @@ -146,7 +146,7 @@ void closeMostFDs(const std::set & exceptions) } -void closeOnExec(int fd) +void unix::closeOnExec(int fd) { int prev; if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index b5c21ad32..16773e3ea 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -122,7 +122,7 @@ void Pipe::create() #if _WIN32_WINNT >= 0x0600 -std::wstring handleToFileName(HANDLE handle) { +std::wstring windows::handleToFileName(HANDLE handle) { std::vector buf(0x100); DWORD dw = GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED); if (dw == 0) { @@ -141,7 +141,7 @@ std::wstring handleToFileName(HANDLE handle) { } -Path handleToPath(HANDLE handle) { +Path windows::handleToPath(HANDLE handle) { return os_string_to_string(handleToFileName(handle)); } diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index de77a7b6b..f1fc51682 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -295,7 +295,7 @@ static void daemonLoop(std::optional forceTrustClientOpt) if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") throw Error("unexpected systemd environment variables"); fdSocket = SD_LISTEN_FDS_START; - closeOnExec(fdSocket.get()); + unix::closeOnExec(fdSocket.get()); } // Otherwise, create and bind to a Unix domain socket. @@ -323,7 +323,7 @@ static void daemonLoop(std::optional forceTrustClientOpt) throw SysError("accepting connection"); } - closeOnExec(remote.get()); + unix::closeOnExec(remote.get()); PeerInfo peer { .pidKnown = false }; TrustedFlag trusted; From 56f0b5304f14212b111ca51c27f1c216caf3acbf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jun 2024 10:50:15 -0400 Subject: [PATCH 53/67] Document the `nix-util` Meson build system more extensively I hope this will make it easier to maintain, and also make it easier for others to assist with porting the rest of the build system to Meson. Co-authored-by: Robert Hensing --- flake.nix | 1 + src/libutil/meson.build | 82 ++++++++++++++++++++++------- src/libutil/package.nix | 9 ++-- src/libutil/unix/file-descriptor.cc | 2 + 4 files changed, 71 insertions(+), 23 deletions(-) diff --git a/flake.nix b/flake.nix index 0e29c6e6f..f3660b8bf 100644 --- a/flake.nix +++ b/flake.nix @@ -243,6 +243,7 @@ changelog-d; default = self.packages.${system}.nix; } // lib.concatMapAttrs + # We need to "flatten" packages we care about to pass `flake check`. (pkgName: {}: { "${pkgName}" = nixpkgsFor.${system}.native.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName}; diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8644c69f4..2259d4e22 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -1,5 +1,5 @@ project('nix-util', 'cpp', - version : run_command('cat', './.version', check : true).stdout().strip(), + version : files('.version'), default_options : [ 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level @@ -8,25 +8,61 @@ project('nix-util', 'cpp', 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], - meson_version : '>= 0.64.0', + meson_version : '>= 1.1', license : 'LGPL-2.1-or-later', ) cxx = meson.get_compiler('cpp') +# These are private dependencies with pkg-config files. What private +# means is that the dependencies are used by the library but they are +# *not* used (e.g. `#include`-ed) in any installed header file, and only +# in regular source code (`*.cc`) or private, uninstalled headers. They +# are thus part of the *implementation* of the library, but not its +# *interface*. +# +# See `man pkg-config` for some details. deps_private = [ ] + +# These are public dependencies with pkg-config files. Public is the +# opposite of private: these dependencies are used in installed header +# files. They are part of the interface (and implementation) of the +# library. +# +# N.B. This concept is mostly unrelated to our own concept of a public +# (stable) API, for consumption outside of the Nix repository. +# `libnixutil` is an unstable C++ library, whose public interface is +# likewise unstable. `libutilc` conversely is a hopefully-soon stable +# C library, whose public interface --- including public but not private +# dependencies --- will also likewise soon be stable. +# +# N.B. For distributions that care about "ABI" stablity and not just +# "API" stability, the private dependencies also matter as they can +# potentially affect the public ABI. deps_public = [ ] + +# These are dependencencies without pkg-config files. Ideally they are +# just private, but they may also be public (e.g. boost). deps_other = [ ] configdata = configuration_data() -# Check for each of these functions, and create a define like `#define HAVE_LUTIMES 1`. +# Check for each of these functions, and create a define like `#define +# HAVE_LUTIMES 1`. The `#define` is unconditional, 0 for not found and 1 +# for found. One therefore uses it with `#if` not `#ifdef`. check_funcs = [ - # Optionally used for changing the mtime of symlinks. + # Optionally used for changing the mtime of symlinks. 'lutimes', + # Optionally used for creating pipes on Unix 'pipe2', + # Optionally used to preallocate files to be large enough before + # writing to them. 'posix_fallocate', + # Optionally used to get more information about processes failing due + # to a signal on Unix. 'strsignal', + # Optionally used to try to close more file descriptors (e.g. before + # forking) on Unix. 'sysconf', ] foreach funcspec : check_funcs @@ -35,8 +71,10 @@ foreach funcspec : check_funcs configdata.set(define_name, define_value) endforeach -# Conditional to work around https://github.com/mesonbuild/meson/issues/13293 -if host_machine.system() != 'windows' and cxx.get_id() == 'gcc' +# This is only conditional to work around +# https://github.com/mesonbuild/meson/issues/13293. It should be +# unconditional. +if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') deps_private += dependency('threads') endif @@ -53,7 +91,8 @@ boost = dependency( 'boost', modules : ['context', 'coroutine'], ) -# Actually public, but wrong type of dep for pkg config +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. deps_other += boost openssl = dependency( @@ -80,8 +119,10 @@ brotli = [ ] deps_private += brotli -# cpuid only makes sense on x86_64 -cpuid_required = host_machine.cpu_family() == 'x86_64' ? get_option('cpuid') : false +cpuid_required = get_option('cpuid') +if host_machine.cpu_family() != 'x86_64' and cpuid_required.enabled() + warning('Force-enabling seccomp on non-x86_64 does not make sense') +endif cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required) configdata.set('HAVE_LIBCPUID', cpuid.found().to_int()) deps_private += cpuid @@ -89,7 +130,7 @@ deps_private += cpuid nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json -config_util_h = configure_file( +config_h = configure_file( configuration : configdata, output : 'config-util.h', ) @@ -157,7 +198,7 @@ sources = files( include_dirs = [include_directories('.')] -headers = [config_util_h] + files( +headers = [config_h] + files( 'abstract-setting-to-json.hh', 'ansicolor.hh', 'archive.hh', @@ -248,7 +289,7 @@ else linker_export_flags = [] endif -libutil = library( +this_library = library( 'nixutil', sources, dependencies : deps_public + deps_private + deps_other, @@ -259,6 +300,11 @@ libutil = library( install_headers(headers, subdir : 'nix', preserve_path : true) +# Part of how we copy boost libraries to a separate installation to +# reduce closure size. These libraries will be copied to our `$out/bin`, +# and these `-l` flags will pick them up there. +# +# https://github.com/NixOS/nixpkgs/issues/45462 libraries_private = ['-lboost_context', '-lboost_coroutine'] if host_machine.system() == 'windows' # `libraries_private` cannot contain ad-hoc dependencies (from @@ -267,22 +313,20 @@ if host_machine.system() == 'windows' endif import('pkgconfig').generate( - libutil, - filebase : 'nix-util', + this_library, + filebase : meson.project_name(), name : 'Nix', description : 'Nix Package Manager', subdirs : ['nix'], extra_cflags : ['-std=c++2a'], requires : deps_public, requires_private : deps_private, - # avoid absolute paths, use vendored ones libraries_private : libraries_private, ) -nix_util = declare_dependency( +meson.override_dependency(meson.project_name(), declare_dependency( include_directories : include_dirs, - link_with : libutil, + link_with : this_library, compile_args : ['-std=c++2a'], dependencies : [], -) -meson.override_dependency('nix-util', nix_util) +)) diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 6575b37a6..a461dccb8 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -83,9 +83,8 @@ mkDerivation (finalAttrs: { '' echo ${version} > .version '' - # Copy libboost_context so we don't get all of Boost in our closure. - - # https://github.com/NixOS/nixpkgs/issues/45462 + # Copy some boost libraries so we don't get all of Boost in our + # closure. https://github.com/NixOS/nixpkgs/issues/45462 + lib.optionalString (!stdenv.hostPlatform.isStatic) ('' mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib @@ -115,7 +114,9 @@ mkDerivation (finalAttrs: { enableParallelBuilding = true; postInstall = - # Remove absolute path to boost libs + # Remove absolute path to boost libs that ends up in `Libs.private` + # by default, and would clash with out `disallowedReferences`. Part + # of the https://github.com/NixOS/nixpkgs/issues/45462 workaround. '' sed -i "$out/lib/pkgconfig/nix-util.pc" -e 's, ${lib.getLib boost}[^ ]*,,g' '' diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index a74f16ce1..bc2b2d614 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -139,7 +139,9 @@ void closeMostFDs(const std::set & exceptions) #endif int maxFD = 0; +#if HAVE_SYSCONF maxFD = sysconf(_SC_OPEN_MAX); +#endif for (int fd = 0; fd < maxFD; ++fd) if (!exceptions.count(fd)) close(fd); /* ignore result */ From 7a5ee5d5973160293fe6b0dcfafc87699a75bd49 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jun 2024 11:42:17 -0400 Subject: [PATCH 54/67] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index f3660b8bf..599c6dfbe 100644 --- a/flake.nix +++ b/flake.nix @@ -243,7 +243,7 @@ changelog-d; default = self.packages.${system}.nix; } // lib.concatMapAttrs - # We need to "flatten" packages we care about to pass `flake check`. + # We need to flatten recursive attribute sets of derivations to pass `flake check`. (pkgName: {}: { "${pkgName}" = nixpkgsFor.${system}.native.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName}; From 0b56c98b1c10c3ea16b8cda9644e5b656b9d7b67 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 13 Jun 2024 18:18:36 +0200 Subject: [PATCH 55/67] C API: Value -> nix_value --- src/libexpr-c/nix_api_expr.h | 3 ++- src/libexpr-c/nix_api_expr_internal.h | 5 +++++ src/libexpr-c/nix_api_value.cc | 11 ++++++++--- src/libexpr-c/nix_api_value.h | 5 ++++- tests/unit/libexpr/nix_api_value.cc | 10 ++++++++++ 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index cb6c00385..140da9e52 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -36,7 +36,8 @@ typedef struct EvalState EvalState; // nix::EvalState * @struct Value * @see value_manip */ -typedef void Value; // nix::Value +typedef struct nix_value nix_value; +[[deprecated("use nix_value instead")]] typedef nix_value Value; // Function prototypes /** diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 7743849fd..5a875ef39 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -20,6 +20,11 @@ struct ListBuilder nix::ListBuilder builder; }; +struct nix_value +{ + nix::Value value; +}; + struct nix_string_return { std::string str; diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 978cf7f43..dd8382133 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -64,6 +64,11 @@ static nix::Value & check_value_out(Value * value) return v; } +static inline nix_value * as_nix_value_ptr(nix::Value * v) +{ + return reinterpret_cast(v); +} + /** * Helper function to convert calls from nix into C API. * @@ -159,7 +164,7 @@ Value * nix_alloc_value(nix_c_context * context, EvalState * state) if (context) context->last_err_code = NIX_OK; try { - Value * res = state->state.allocValue(); + nix_value * res = as_nix_value_ptr(state->state.allocValue()); nix_gc_incref(nullptr, res); return res; } @@ -345,7 +350,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt if (attr) { nix_gc_incref(nullptr, attr->value); state->state.forceValue(*attr->value, nix::noPos); - return attr->value; + return as_nix_value_ptr(attr->value); } nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); return nullptr; @@ -380,7 +385,7 @@ nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * sta *name = ((const std::string &) (state->state.symbols[a.name])).c_str(); nix_gc_incref(nullptr, a.value); state->state.forceValue(*a.value, nix::noPos); - return a.value; + return as_nix_value_ptr(a.value); } NIXC_CATCH_ERRS_NULL } diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 244860707..acc4a1969 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -35,8 +35,11 @@ typedef enum { } ValueType; // forward declarations -typedef void Value; +typedef struct nix_value nix_value; typedef struct EvalState EvalState; + +[[deprecated("use nix_value instead")]] typedef nix_value Value; + // type defs /** @brief Stores an under-construction set of bindings * @ingroup value_manip diff --git a/tests/unit/libexpr/nix_api_value.cc b/tests/unit/libexpr/nix_api_value.cc index c71593c85..b674d8681 100644 --- a/tests/unit/libexpr/nix_api_value.cc +++ b/tests/unit/libexpr/nix_api_value.cc @@ -4,16 +4,26 @@ #include "nix_api_util_internal.h" #include "nix_api_expr.h" #include "nix_api_value.h" +#include "nix_api_expr_internal.h" #include "tests/nix_api_expr.hh" #include "tests/string_callback.hh" #include "gmock/gmock.h" +#include #include #include namespace nixC { +TEST_F(nix_api_expr_test, as_nix_value_ptr) +{ + // nix_alloc_value casts nix::Value to nix_value + // It should be obvious from the decl that that works, but if it doesn't, + // the whole implementation would be utterly broken. + ASSERT_EQ(sizeof(nix::Value), sizeof(nix_value)); +} + TEST_F(nix_api_expr_test, nix_value_get_int_invalid) { ASSERT_EQ(0, nix_get_int(ctx, nullptr)); From c50db4e58c6f18b9e7948de9d87f5af48b07fa9b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 13 Jun 2024 18:21:04 +0200 Subject: [PATCH 56/67] C API: Add nix_value_{inc,dec}ref - Can be implemented more easily by more eval architectures. - Better types in generated bindings remove some uncertainty and doubt. --- src/libexpr-c/nix_api_expr.cc | 9 +++++++++ src/libexpr-c/nix_api_expr.h | 5 +++++ src/libexpr-c/nix_api_value.h | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index b86d745db..28b8922ce 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -181,6 +181,15 @@ nix_err nix_gc_decref(nix_c_context * context, const void *) void nix_gc_now() {} #endif +nix_err nix_value_incref(nix_c_context * context, nix_value *x) +{ + return nix_gc_incref(context, (const void *) x); +} +nix_err nix_value_decref(nix_c_context * context, nix_value *x) +{ + return nix_gc_decref(context, (const void *) x); +} + void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)) { #ifdef HAVE_BOEHMGC diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index 140da9e52..ce00246c6 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -189,6 +189,11 @@ void nix_state_free(EvalState * state); * you're done with a value returned by the evaluator. * @{ */ + +// TODO: Deprecate nix_gc_incref in favor of the type-specific reference counting functions? +// e.g. nix_value_incref. +// It gives implementors more flexibility, and adds safety, so that generated +// bindings can be used without fighting the host type system (where applicable). /** * @brief Increment the garbage collector reference counter for the given object. * diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index acc4a1969..26044bd57 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -147,6 +147,25 @@ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp); */ Value * nix_alloc_value(nix_c_context * context, EvalState * state); +/** + * @brief Increment the garbage collector reference counter for the given `nix_value`. + * + * The Nix language evaluator C API keeps track of alive objects by reference counting. + * When you're done with a refcounted pointer, call nix_value_decref(). + * + * @param[out] context Optional, stores error information + * @param[in] value The object to keep alive + */ +nix_err nix_value_incref(nix_c_context * context, nix_value * value); + +/** + * @brief Decrement the garbage collector reference counter for the given object + * + * @param[out] context Optional, stores error information + * @param[in] value The object to stop referencing + */ +nix_err nix_value_decref(nix_c_context * context, nix_value * value); + /** @addtogroup value_manip Manipulating values * @brief Functions to inspect and change Nix language values, represented by Value. * @{ From 5d8118d9cb6a2c349fd96bee5a9bdd1c4b65fae9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 13 Jun 2024 18:23:21 +0200 Subject: [PATCH 57/67] C API: Docs --- src/libexpr-c/nix_api_expr.h | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index ce00246c6..783176bfa 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -29,12 +29,20 @@ extern "C" { * @see nix_state_create */ typedef struct EvalState EvalState; // nix::EvalState -/** - * @brief Represents a value in the Nix language. + +/** @brief A Nix language value, or thunk that may evaluate to a value. + * + * Values are the primary objects manipulated in the Nix language. + * They are considered to be immutable from a user's perspective, but the process of evaluating a Value changes its + * ValueType if it was a thunk. After a Value has been evaluated, its ValueType does not change. + * + * Evaluation in this context refers to the process of evaluating a single Value object, also called "forcing" the + * Value; see `nix_value_force`. + * + * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules. * - * Owned by the garbage collector. - * @struct Value * @see value_manip + * @see nix_value_incref, nix_value_decref */ typedef struct nix_value nix_value; [[deprecated("use nix_value instead")]] typedef nix_value Value; @@ -128,7 +136,7 @@ nix_err nix_value_call_multi( * The Nix interpreter is lazy, and not-yet-evaluated Values can be * of type NIX_TYPE_THUNK instead of their actual value. * - * This function converts these Values into their final type. + * This function mutates such a Value, so that, if successful, it has its final type. * * @note This function is mainly needed before calling @ref getters, but not for API calls that return a `Value`. * From b94e1d62182114ab5198fd58869a9af29d54cff5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 13 Jun 2024 18:51:58 +0200 Subject: [PATCH 58/67] C API: Value -> nix_value See issue https://github.com/NixOS/nix/issues/10434 --- src/libexpr-c/nix_api_expr.cc | 26 +++--- src/libexpr-c/nix_api_expr.h | 26 +++--- src/libexpr-c/nix_api_value.cc | 93 ++++++++++--------- src/libexpr-c/nix_api_value.h | 71 +++++++------- .../libexpr-support/tests/nix_api_expr.hh | 2 +- tests/unit/libexpr/nix_api_expr.cc | 62 +++++++------ tests/unit/libexpr/nix_api_external.cc | 4 +- tests/unit/libexpr/nix_api_value.cc | 32 +++---- 8 files changed, 161 insertions(+), 155 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 28b8922ce..bdf7a1e63 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -42,56 +42,56 @@ nix_err nix_libexpr_init(nix_c_context * context) } nix_err nix_expr_eval_from_string( - nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value) + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); - state->state.eval(parsedExpr, *(nix::Value *) value); - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value) +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.callFunction(*(nix::Value *) fn, *(nix::Value *) arg, *(nix::Value *) value, nix::noPos); - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value) +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.callFunction(*(nix::Value *) fn, nargs, (nix::Value * *)args, *(nix::Value *) value, nix::noPos); - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value) +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value) +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.forceValueDeep(*(nix::Value *) value); + state->state.forceValueDeep(value->value); } NIXC_CATCH_ERRS } diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index 783176bfa..adf8b65b1 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -33,11 +33,11 @@ typedef struct EvalState EvalState; // nix::EvalState /** @brief A Nix language value, or thunk that may evaluate to a value. * * Values are the primary objects manipulated in the Nix language. - * They are considered to be immutable from a user's perspective, but the process of evaluating a Value changes its - * ValueType if it was a thunk. After a Value has been evaluated, its ValueType does not change. + * They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its + * ValueType if it was a thunk. After a value has been evaluated, its ValueType does not change. * - * Evaluation in this context refers to the process of evaluating a single Value object, also called "forcing" the - * Value; see `nix_value_force`. + * Evaluation in this context refers to the process of evaluating a single value object, also called "forcing" the + * value; see `nix_value_force`. * * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules. * @@ -74,7 +74,7 @@ nix_err nix_libexpr_init(nix_c_context * context); * @return NIX_OK if the evaluation was successful, an error code otherwise. */ nix_err nix_expr_eval_from_string( - nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value); + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value); /** * @brief Calls a Nix function with an argument. @@ -88,7 +88,7 @@ nix_err nix_expr_eval_from_string( * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk. * Note the different argument order. */ -nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value); +nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * fn, nix_value * arg, nix_value * value); /** * @brief Calls a Nix function with multiple arguments. @@ -107,7 +107,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V * @see NIX_VALUE_CALL For a macro that wraps this function for convenience. */ nix_err nix_value_call_multi( - nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value); + nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value); /** * @brief Calls a Nix function with multiple arguments. @@ -125,7 +125,7 @@ nix_err nix_value_call_multi( */ #define NIX_VALUE_CALL(context, state, value, fn, ...) \ do { \ - Value * args_array[] = {__VA_ARGS__}; \ + nix_value * args_array[] = {__VA_ARGS__}; \ size_t nargs = sizeof(args_array) / sizeof(args_array[0]); \ nix_value_call_multi(context, state, fn, nargs, args_array, value); \ } while (0) @@ -133,12 +133,10 @@ nix_err nix_value_call_multi( /** * @brief Forces the evaluation of a Nix value. * - * The Nix interpreter is lazy, and not-yet-evaluated Values can be + * The Nix interpreter is lazy, and not-yet-evaluated values can be * of type NIX_TYPE_THUNK instead of their actual value. * - * This function mutates such a Value, so that, if successful, it has its final type. - * - * @note This function is mainly needed before calling @ref getters, but not for API calls that return a `Value`. + * This function mutates such a `nix_value`, so that, if successful, it has its final type. * * @param[out] context Optional, stores error information * @param[in] state The state of the evaluation. @@ -147,7 +145,7 @@ nix_err nix_value_call_multi( * @return NIX_OK if the force operation was successful, an error code * otherwise. */ -nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value); +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value); /** * @brief Forces the deep evaluation of a Nix value. @@ -163,7 +161,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * valu * @return NIX_OK if the deep force operation was successful, an error code * otherwise. */ -nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value); +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value); /** * @brief Create a new Nix language evaluator state. diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index dd8382133..2f2f99617 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -21,45 +21,45 @@ #endif // Internal helper functions to check [in] and [out] `Value *` parameters -static const nix::Value & check_value_not_null(const Value * value) +static const nix::Value & check_value_not_null(const nix_value * value) { if (!value) { - throw std::runtime_error("Value is null"); + throw std::runtime_error("nix_value is null"); } return *((const nix::Value *) value); } -static nix::Value & check_value_not_null(Value * value) +static nix::Value & check_value_not_null(nix_value * value) { if (!value) { - throw std::runtime_error("Value is null"); + throw std::runtime_error("nix_value is null"); } - return *((nix::Value *) value); + return value->value; } -static const nix::Value & check_value_in(const Value * value) +static const nix::Value & check_value_in(const nix_value * value) { auto & v = check_value_not_null(value); if (!v.isValid()) { - throw std::runtime_error("Uninitialized Value"); + throw std::runtime_error("Uninitialized nix_value"); } return v; } -static nix::Value & check_value_in(Value * value) +static nix::Value & check_value_in(nix_value * value) { auto & v = check_value_not_null(value); if (!v.isValid()) { - throw std::runtime_error("Uninitialized Value"); + throw std::runtime_error("Uninitialized nix_value"); } return v; } -static nix::Value & check_value_out(Value * value) +static nix::Value & check_value_out(nix_value * value) { auto & v = check_value_not_null(value); if (v.isValid()) { - throw std::runtime_error("Value already initialized. Variables are immutable"); + throw std::runtime_error("nix_value already initialized. Variables are immutable"); } return v; } @@ -92,7 +92,7 @@ static void nix_c_primop_wrapper( // or maybe something to make blackholes work better; we don't know). nix::Value vTmp; - f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &vTmp); + f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp); if (ctx.last_err_code != NIX_OK) { /* TODO: Throw different errors depending on the error code */ @@ -159,7 +159,7 @@ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp) NIXC_CATCH_ERRS } -Value * nix_alloc_value(nix_c_context * context, EvalState * state) +nix_value * nix_alloc_value(nix_c_context * context, EvalState * state) { if (context) context->last_err_code = NIX_OK; @@ -171,7 +171,7 @@ Value * nix_alloc_value(nix_c_context * context, EvalState * state) NIXC_CATCH_ERRS_NULL } -ValueType nix_get_type(nix_c_context * context, const Value * value) +ValueType nix_get_type(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -207,7 +207,7 @@ ValueType nix_get_type(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL); } -const char * nix_get_typename(nix_c_context * context, const Value * value) +const char * nix_get_typename(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -219,7 +219,7 @@ const char * nix_get_typename(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_NULL } -bool nix_get_bool(nix_c_context * context, const Value * value) +bool nix_get_bool(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -231,7 +231,8 @@ bool nix_get_bool(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(false); } -nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data) +nix_err +nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -243,7 +244,7 @@ nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_str NIXC_CATCH_ERRS } -const char * nix_get_path_string(nix_c_context * context, const Value * value) +const char * nix_get_path_string(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -262,7 +263,7 @@ const char * nix_get_path_string(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_NULL } -unsigned int nix_get_list_size(nix_c_context * context, const Value * value) +unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -274,7 +275,7 @@ unsigned int nix_get_list_size(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0); } -unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value) +unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -286,7 +287,7 @@ unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0); } -double nix_get_float(nix_c_context * context, const Value * value) +double nix_get_float(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -298,7 +299,7 @@ double nix_get_float(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0.0); } -int64_t nix_get_int(nix_c_context * context, const Value * value) +int64_t nix_get_int(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -310,7 +311,7 @@ int64_t nix_get_int(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0); } -ExternalValue * nix_get_external(nix_c_context * context, Value * value) +ExternalValue * nix_get_external(nix_c_context * context, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -322,7 +323,7 @@ ExternalValue * nix_get_external(nix_c_context * context, Value * value) NIXC_CATCH_ERRS_NULL; } -Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix) +nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix) { if (context) context->last_err_code = NIX_OK; @@ -333,12 +334,12 @@ Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalSta nix_gc_incref(nullptr, p); if (p != nullptr) state->state.forceValue(*p, nix::noPos); - return (Value *) p; + return as_nix_value_ptr(p); } NIXC_CATCH_ERRS_NULL } -Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name) +nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) { if (context) context->last_err_code = NIX_OK; @@ -358,7 +359,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt NIXC_CATCH_ERRS_NULL } -bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name) +bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) { if (context) context->last_err_code = NIX_OK; @@ -374,8 +375,8 @@ bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState NIXC_CATCH_ERRS_RES(false); } -Value * -nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name) +nix_value * nix_get_attr_byidx( + nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name) { if (context) context->last_err_code = NIX_OK; @@ -390,7 +391,8 @@ nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * sta NIXC_CATCH_ERRS_NULL } -const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i) +const char * +nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i) { if (context) context->last_err_code = NIX_OK; @@ -402,7 +404,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * valu NIXC_CATCH_ERRS_NULL } -nix_err nix_init_bool(nix_c_context * context, Value * value, bool b) +nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b) { if (context) context->last_err_code = NIX_OK; @@ -414,7 +416,7 @@ nix_err nix_init_bool(nix_c_context * context, Value * value, bool b) } // todo string context -nix_err nix_init_string(nix_c_context * context, Value * value, const char * str) +nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str) { if (context) context->last_err_code = NIX_OK; @@ -425,7 +427,7 @@ nix_err nix_init_string(nix_c_context * context, Value * value, const char * str NIXC_CATCH_ERRS } -nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str) +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str) { if (context) context->last_err_code = NIX_OK; @@ -436,7 +438,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * val NIXC_CATCH_ERRS } -nix_err nix_init_float(nix_c_context * context, Value * value, double d) +nix_err nix_init_float(nix_c_context * context, nix_value * value, double d) { if (context) context->last_err_code = NIX_OK; @@ -447,7 +449,7 @@ nix_err nix_init_float(nix_c_context * context, Value * value, double d) NIXC_CATCH_ERRS } -nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i) +nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i) { if (context) context->last_err_code = NIX_OK; @@ -458,7 +460,7 @@ nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i) NIXC_CATCH_ERRS } -nix_err nix_init_null(nix_c_context * context, Value * value) +nix_err nix_init_null(nix_c_context * context, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -469,7 +471,7 @@ nix_err nix_init_null(nix_c_context * context, Value * value) NIXC_CATCH_ERRS } -nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg) +nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg) { if (context) context->last_err_code = NIX_OK; @@ -482,7 +484,7 @@ nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value NIXC_CATCH_ERRS } -nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val) +nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val) { if (context) context->last_err_code = NIX_OK; @@ -509,7 +511,8 @@ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, NIXC_CATCH_ERRS_NULL } -nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value) +nix_err +nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -529,7 +532,7 @@ void nix_list_builder_free(ListBuilder * list_builder) #endif } -nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value) +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -540,7 +543,7 @@ nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value NIXC_CATCH_ERRS } -nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p) +nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * p) { if (context) context->last_err_code = NIX_OK; @@ -551,7 +554,7 @@ nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p) NIXC_CATCH_ERRS } -nix_err nix_copy_value(nix_c_context * context, Value * value, const Value * source) +nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source) { if (context) context->last_err_code = NIX_OK; @@ -563,7 +566,7 @@ nix_err nix_copy_value(nix_c_context * context, Value * value, const Value * sou NIXC_CATCH_ERRS } -nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b) +nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b) { if (context) context->last_err_code = NIX_OK; @@ -589,7 +592,7 @@ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * NIXC_CATCH_ERRS_NULL } -nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, Value * value) +nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -610,7 +613,7 @@ void nix_bindings_builder_free(BindingsBuilder * bb) #endif } -nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD) +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD) { if (context) context->last_err_code = NIX_OK; diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 26044bd57..044f68c9e 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -93,7 +93,8 @@ typedef struct nix_realised_string nix_realised_string; * @param[out] ret return value * @see nix_alloc_primop, nix_init_primop */ -typedef void (*PrimOpFun)(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret); +typedef void (*PrimOpFun)( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret); /** @brief Allocate a PrimOp * @@ -145,7 +146,7 @@ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp); * @return value, or null in case of errors * */ -Value * nix_alloc_value(nix_c_context * context, EvalState * state); +nix_value * nix_alloc_value(nix_c_context * context, EvalState * state); /** * @brief Increment the garbage collector reference counter for the given `nix_value`. @@ -167,7 +168,7 @@ nix_err nix_value_incref(nix_c_context * context, nix_value * value); nix_err nix_value_decref(nix_c_context * context, nix_value * value); /** @addtogroup value_manip Manipulating values - * @brief Functions to inspect and change Nix language values, represented by Value. + * @brief Functions to inspect and change Nix language values, represented by nix_value. * @{ */ /** @anchor getters @@ -179,7 +180,7 @@ nix_err nix_value_decref(nix_c_context * context, nix_value * value); * @param[in] value Nix value to inspect * @return type of nix value */ -ValueType nix_get_type(nix_c_context * context, const Value * value); +ValueType nix_get_type(nix_c_context * context, const nix_value * value); /** @brief Get type name of value as defined in the evaluator * @param[out] context Optional, stores error information @@ -187,14 +188,14 @@ ValueType nix_get_type(nix_c_context * context, const Value * value); * @return type name, owned string * @todo way to free the result */ -const char * nix_get_typename(nix_c_context * context, const Value * value); +const char * nix_get_typename(nix_c_context * context, const nix_value * value); /** @brief Get boolean value * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return true or false, error info via context */ -bool nix_get_bool(nix_c_context * context, const Value * value); +bool nix_get_bool(nix_c_context * context, const nix_value * value); /** @brief Get the raw string * @@ -208,7 +209,7 @@ bool nix_get_bool(nix_c_context * context, const Value * value); * @return error code, NIX_OK on success. */ nix_err -nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data); +nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data); /** @brief Get path as string * @param[out] context Optional, stores error information @@ -216,42 +217,42 @@ nix_get_string(nix_c_context * context, const Value * value, nix_get_string_call * @return string * @return NULL in case of error. */ -const char * nix_get_path_string(nix_c_context * context, const Value * value); +const char * nix_get_path_string(nix_c_context * context, const nix_value * value); /** @brief Get the length of a list * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return length of list, error info via context */ -unsigned int nix_get_list_size(nix_c_context * context, const Value * value); +unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value); /** @brief Get the element count of an attrset * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return attrset element count, error info via context */ -unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value); +unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value); /** @brief Get float value in 64 bits * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return float contents, error info via context */ -double nix_get_float(nix_c_context * context, const Value * value); +double nix_get_float(nix_c_context * context, const nix_value * value); /** @brief Get int value * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return int contents, error info via context */ -int64_t nix_get_int(nix_c_context * context, const Value * value); +int64_t nix_get_int(nix_c_context * context, const nix_value * value); /** @brief Get external reference * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return reference to external, NULL in case of error */ -ExternalValue * nix_get_external(nix_c_context * context, Value *); +ExternalValue * nix_get_external(nix_c_context * context, nix_value *); /** @brief Get the ix'th element of a list * @@ -262,7 +263,7 @@ ExternalValue * nix_get_external(nix_c_context * context, Value *); * @param[in] ix list element to get * @return value, NULL in case of errors */ -Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix); +nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); /** @brief Get an attr by name * @@ -273,7 +274,7 @@ Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalSta * @param[in] name attribute name * @return value, NULL in case of errors */ -Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name); +nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Check if an attribute name exists on a value * @param[out] context Optional, stores error information @@ -282,7 +283,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt * @param[in] name attribute name * @return value, error info via context */ -bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name); +bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Get an attribute by index in the sorted bindings * @@ -296,8 +297,8 @@ bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * @param[out] name will store a pointer to the attribute name * @return value, NULL in case of errors */ -Value * -nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name); +nix_value * nix_get_attr_byidx( + nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name); /** @brief Get an attribute name by index in the sorted bindings * @@ -310,7 +311,8 @@ nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * sta * @param[in] i attribute index * @return name, NULL in case of errors */ -const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i); +const char * +nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i); /**@}*/ /** @name Initializers @@ -327,7 +329,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * valu * @param[in] b the boolean value * @return error code, NIX_OK on success. */ -nix_err nix_init_bool(nix_c_context * context, Value * value, bool b); +nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b); /** @brief Set a string * @param[out] context Optional, stores error information @@ -335,7 +337,7 @@ nix_err nix_init_bool(nix_c_context * context, Value * value, bool b); * @param[in] str the string, copied * @return error code, NIX_OK on success. */ -nix_err nix_init_string(nix_c_context * context, Value * value, const char * str); +nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str); /** @brief Set a path * @param[out] context Optional, stores error information @@ -343,7 +345,7 @@ nix_err nix_init_string(nix_c_context * context, Value * value, const char * str * @param[in] str the path string, copied * @return error code, NIX_OK on success. */ -nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str); +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str); /** @brief Set a float * @param[out] context Optional, stores error information @@ -351,7 +353,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * val * @param[in] d the float, 64-bits * @return error code, NIX_OK on success. */ -nix_err nix_init_float(nix_c_context * context, Value * value, double d); +nix_err nix_init_float(nix_c_context * context, nix_value * value, double d); /** @brief Set an int * @param[out] context Optional, stores error information @@ -360,13 +362,13 @@ nix_err nix_init_float(nix_c_context * context, Value * value, double d); * @return error code, NIX_OK on success. */ -nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i); +nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i); /** @brief Set null * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @return error code, NIX_OK on success. */ -nix_err nix_init_null(nix_c_context * context, Value * value); +nix_err nix_init_null(nix_c_context * context, nix_value * value); /** @brief Set the value to a thunk that will perform a function application when needed. * @@ -382,7 +384,7 @@ nix_err nix_init_null(nix_c_context * context, Value * value); * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value. * Note the different argument order. */ -nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg); +nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg); /** @brief Set an external value * @param[out] context Optional, stores error information @@ -390,7 +392,7 @@ nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * @param[in] val the external value to set. Will be GC-referenced by the value. * @return error code, NIX_OK on success. */ -nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val); +nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val); /** @brief Create a list from a list builder * @param[out] context Optional, stores error information @@ -398,7 +400,7 @@ nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * @param[out] value Nix value to modify * @return error code, NIX_OK on success. */ -nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value); +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value); /** @brief Create a list builder * @param[out] context Optional, stores error information @@ -415,7 +417,8 @@ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, * @param[in] value value to insert * @return error code, NIX_OK on success. */ -nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value); +nix_err +nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value); /** @brief Free a list builder * @@ -430,7 +433,7 @@ void nix_list_builder_free(ListBuilder * list_builder); * @param[in] b bindings builder to use. Make sure to unref this afterwards. * @return error code, NIX_OK on success. */ -nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b); +nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b); /** @brief Set primop * @param[out] context Optional, stores error information @@ -439,14 +442,14 @@ nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * * @see nix_alloc_primop * @return error code, NIX_OK on success. */ -nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * op); +nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op); /** @brief Copy from another value * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] source value to copy from * @return error code, NIX_OK on success. */ -nix_err nix_copy_value(nix_c_context * context, Value * value, const Value * source); +nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source); /**@}*/ /** @brief Create a bindings builder @@ -466,7 +469,7 @@ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * * @return error code, NIX_OK on success. */ nix_err -nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, Value * value); +nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, nix_value * value); /** @brief Free a bindings builder * @@ -493,7 +496,7 @@ void nix_bindings_builder_free(BindingsBuilder * builder); You should set this to false when building for your application's purpose. * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free */ -nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD); +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD); /** @brief Start of the string * @param[in] realised_string diff --git a/tests/unit/libexpr-support/tests/nix_api_expr.hh b/tests/unit/libexpr-support/tests/nix_api_expr.hh index d1840d034..6ddca0d14 100644 --- a/tests/unit/libexpr-support/tests/nix_api_expr.hh +++ b/tests/unit/libexpr-support/tests/nix_api_expr.hh @@ -25,7 +25,7 @@ protected: } EvalState * state; - Value * value; + nix_value * value; }; } diff --git a/tests/unit/libexpr/nix_api_expr.cc b/tests/unit/libexpr/nix_api_expr.cc index 92a6a1175..8b97d6923 100644 --- a/tests/unit/libexpr/nix_api_expr.cc +++ b/tests/unit/libexpr/nix_api_expr.cc @@ -39,12 +39,12 @@ TEST_F(nix_api_expr_test, nix_expr_eval_drv) ASSERT_EQ(NIX_TYPE_ATTRS, nix_get_type(nullptr, value)); EvalState * stateFn = nix_state_create(nullptr, nullptr, store); - Value * valueFn = nix_alloc_value(nullptr, state); + nix_value * valueFn = nix_alloc_value(nullptr, state); nix_expr_eval_from_string(nullptr, stateFn, "builtins.toString", ".", valueFn); ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(nullptr, valueFn)); EvalState * stateResult = nix_state_create(nullptr, nullptr, store); - Value * valueResult = nix_alloc_value(nullptr, stateResult); + nix_value * valueResult = nix_alloc_value(nullptr, stateResult); nix_value_call(ctx, stateResult, valueFn, value, valueResult); ASSERT_EQ(NIX_TYPE_STRING, nix_get_type(nullptr, valueResult)); @@ -70,7 +70,7 @@ TEST_F(nix_api_expr_test, nix_build_drv) })"; nix_expr_eval_from_string(nullptr, state, expr, ".", value); - Value * drvPathValue = nix_get_attr_byname(nullptr, value, state, "drvPath"); + nix_value * drvPathValue = nix_get_attr_byname(nullptr, value, state, "drvPath"); std::string drvPath; nix_get_string(nullptr, drvPathValue, OBSERVE_STRING(drvPath)); @@ -84,7 +84,7 @@ TEST_F(nix_api_expr_test, nix_build_drv) StorePath * drvStorePath = nix_store_parse_path(ctx, store, drvPath.c_str()); ASSERT_EQ(true, nix_store_is_valid_path(ctx, store, drvStorePath)); - Value * outPathValue = nix_get_attr_byname(ctx, value, state, "outPath"); + nix_value * outPathValue = nix_get_attr_byname(ctx, value, state, "outPath"); std::string outPath; nix_get_string(ctx, outPathValue, OBSERVE_STRING(outPath)); @@ -193,7 +193,8 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context) const char * SAMPLE_USER_DATA = "whatever"; -static void primop_square(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret) +static void +primop_square(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) { assert(context); assert(state); @@ -207,17 +208,17 @@ TEST_F(nix_api_expr_test, nix_expr_primop) PrimOp * primop = nix_alloc_primop(ctx, primop_square, 1, "square", nullptr, "square an integer", (void *) SAMPLE_USER_DATA); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * three = nix_alloc_value(ctx, state); + nix_value * three = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, three, 3); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_value_call(ctx, state, primopValue, three, result); assert_ctx_ok(); @@ -226,7 +227,8 @@ TEST_F(nix_api_expr_test, nix_expr_primop) ASSERT_EQ(9, r); } -static void primop_repeat(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret) +static void +primop_repeat(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) { assert(context); assert(state); @@ -255,27 +257,27 @@ TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_multiple_calls) PrimOp * primop = nix_alloc_primop(ctx, primop_repeat, 2, "repeat", nullptr, "repeat a string", (void *) SAMPLE_USER_DATA); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * hello = nix_alloc_value(ctx, state); + nix_value * hello = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_string(ctx, hello, "hello"); assert_ctx_ok(); - Value * three = nix_alloc_value(ctx, state); + nix_value * three = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, three, 3); assert_ctx_ok(); - Value * partial = nix_alloc_value(ctx, state); + nix_value * partial = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_value_call(ctx, state, primopValue, hello, partial); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_value_call(ctx, state, partial, three, result); assert_ctx_ok(); @@ -290,22 +292,22 @@ TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_single_call) PrimOp * primop = nix_alloc_primop(ctx, primop_repeat, 2, "repeat", nullptr, "repeat a string", (void *) SAMPLE_USER_DATA); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * hello = nix_alloc_value(ctx, state); + nix_value * hello = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_string(ctx, hello, "hello"); assert_ctx_ok(); - Value * three = nix_alloc_value(ctx, state); + nix_value * three = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, three, 3); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); NIX_VALUE_CALL(ctx, state, result, primopValue, hello, three); assert_ctx_ok(); @@ -318,7 +320,7 @@ TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_single_call) } static void -primop_bad_no_return(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret) +primop_bad_no_return(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) { } @@ -327,17 +329,17 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_no_return) PrimOp * primop = nix_alloc_primop(ctx, primop_bad_no_return, 1, "badNoReturn", nullptr, "a broken primop", nullptr); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * three = nix_alloc_value(ctx, state); + nix_value * three = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, three, 3); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_value_call(ctx, state, primopValue, three, result); ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); @@ -348,8 +350,8 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_no_return) ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badNoReturn"))); } -static void -primop_bad_return_thunk(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret) +static void primop_bad_return_thunk( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) { nix_init_apply(context, ret, args[0], args[1]); } @@ -358,22 +360,22 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk) PrimOp * primop = nix_alloc_primop(ctx, primop_bad_return_thunk, 2, "badReturnThunk", nullptr, "a broken primop", nullptr); assert_ctx_ok(); - Value * primopValue = nix_alloc_value(ctx, state); + nix_value * primopValue = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_primop(ctx, primopValue, primop); assert_ctx_ok(); - Value * toString = nix_alloc_value(ctx, state); + nix_value * toString = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_expr_eval_from_string(ctx, state, "builtins.toString", ".", toString); assert_ctx_ok(); - Value * four = nix_alloc_value(ctx, state); + nix_value * four = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_init_int(ctx, four, 4); assert_ctx_ok(); - Value * result = nix_alloc_value(ctx, state); + nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); NIX_VALUE_CALL(ctx, state, result, primopValue, toString, four); @@ -387,11 +389,11 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk) TEST_F(nix_api_expr_test, nix_value_call_multi_no_args) { - Value * n = nix_alloc_value(ctx, state); + nix_value * n = nix_alloc_value(ctx, state); nix_init_int(ctx, n, 3); assert_ctx_ok(); - Value * r = nix_alloc_value(ctx, state); + nix_value * r = nix_alloc_value(ctx, state); nix_value_call_multi(ctx, state, n, 0, nullptr, r); assert_ctx_ok(); diff --git a/tests/unit/libexpr/nix_api_external.cc b/tests/unit/libexpr/nix_api_external.cc index 2391f8317..81ff285a4 100644 --- a/tests/unit/libexpr/nix_api_external.cc +++ b/tests/unit/libexpr/nix_api_external.cc @@ -49,10 +49,10 @@ TEST_F(nix_api_expr_test, nix_expr_eval_external) nix_init_external(ctx, value, val); EvalState * stateResult = nix_state_create(nullptr, nullptr, store); - Value * valueResult = nix_alloc_value(nullptr, stateResult); + nix_value * valueResult = nix_alloc_value(nullptr, stateResult); EvalState * stateFn = nix_state_create(nullptr, nullptr, store); - Value * valueFn = nix_alloc_value(nullptr, stateFn); + nix_value * valueFn = nix_alloc_value(nullptr, stateFn); nix_expr_eval_from_string(nullptr, state, "builtins.typeOf", ".", valueFn); diff --git a/tests/unit/libexpr/nix_api_value.cc b/tests/unit/libexpr/nix_api_value.cc index b674d8681..7fc8b4f64 100644 --- a/tests/unit/libexpr/nix_api_value.cc +++ b/tests/unit/libexpr/nix_api_value.cc @@ -148,8 +148,8 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list) int size = 10; ListBuilder * builder = nix_make_list_builder(ctx, state, size); - Value * intValue = nix_alloc_value(ctx, state); - Value * intValue2 = nix_alloc_value(ctx, state); + nix_value * intValue = nix_alloc_value(ctx, state); + nix_value * intValue2 = nix_alloc_value(ctx, state); // `init` and `insert` can be called in any order nix_init_int(ctx, intValue, 42); @@ -204,10 +204,10 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, size); - Value * intValue = nix_alloc_value(ctx, state); + nix_value * intValue = nix_alloc_value(ctx, state); nix_init_int(ctx, intValue, 42); - Value * stringValue = nix_alloc_value(ctx, state); + nix_value * stringValue = nix_alloc_value(ctx, state); nix_init_string(ctx, stringValue, "foo"); nix_bindings_builder_insert(ctx, builder, "a", intValue); @@ -217,7 +217,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) ASSERT_EQ(2, nix_get_attrs_size(ctx, value)); - Value * out_value = nix_get_attr_byname(ctx, value, state, "a"); + nix_value * out_value = nix_get_attr_byname(ctx, value, state, "a"); ASSERT_EQ(42, nix_get_int(ctx, out_value)); nix_gc_decref(ctx, out_value); @@ -261,10 +261,10 @@ TEST_F(nix_api_expr_test, nix_value_init) // two = 2; // f = a: a * a; - Value * two = nix_alloc_value(ctx, state); + nix_value * two = nix_alloc_value(ctx, state); nix_init_int(ctx, two, 2); - Value * f = nix_alloc_value(ctx, state); + nix_value * f = nix_alloc_value(ctx, state); nix_expr_eval_from_string( ctx, state, @@ -278,7 +278,7 @@ TEST_F(nix_api_expr_test, nix_value_init) // r = f two; - Value * r = nix_alloc_value(ctx, state); + nix_value * r = nix_alloc_value(ctx, state); nix_init_apply(ctx, r, f, two); assert_ctx_ok(); @@ -307,11 +307,11 @@ TEST_F(nix_api_expr_test, nix_value_init) TEST_F(nix_api_expr_test, nix_value_init_apply_error) { - Value * some_string = nix_alloc_value(ctx, state); + nix_value * some_string = nix_alloc_value(ctx, state); nix_init_string(ctx, some_string, "some string"); assert_ctx_ok(); - Value * v = nix_alloc_value(ctx, state); + nix_value * v = nix_alloc_value(ctx, state); nix_init_apply(ctx, v, some_string, some_string); assert_ctx_ok(); @@ -336,7 +336,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) // r = f e // r should not throw an exception, because e is not evaluated - Value * f = nix_alloc_value(ctx, state); + nix_value * f = nix_alloc_value(ctx, state); nix_expr_eval_from_string( ctx, state, @@ -347,9 +347,9 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) f); assert_ctx_ok(); - Value * e = nix_alloc_value(ctx, state); + nix_value * e = nix_alloc_value(ctx, state); { - Value * g = nix_alloc_value(ctx, state); + nix_value * g = nix_alloc_value(ctx, state); nix_expr_eval_from_string( ctx, state, @@ -365,7 +365,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) nix_gc_decref(ctx, g); } - Value * r = nix_alloc_value(ctx, state); + nix_value * r = nix_alloc_value(ctx, state); nix_init_apply(ctx, r, f, e); assert_ctx_ok(); @@ -377,7 +377,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) ASSERT_EQ(1, n); // nix_get_attr_byname isn't lazy (it could have been) so it will throw the exception - Value * foo = nix_get_attr_byname(ctx, r, state, "foo"); + nix_value * foo = nix_get_attr_byname(ctx, r, state, "foo"); ASSERT_EQ(nullptr, foo); ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("error message for test case nix_value_init_apply_lazy_arg")); @@ -388,7 +388,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) TEST_F(nix_api_expr_test, nix_copy_value) { - Value * source = nix_alloc_value(ctx, state); + nix_value * source = nix_alloc_value(ctx, state); nix_init_int(ctx, source, 42); nix_copy_value(ctx, value, source); From 03883f0d1d3a35e95a5f30c911503c53bfc5693a Mon Sep 17 00:00:00 2001 From: Hamir Mahal Date: Thu, 13 Jun 2024 14:13:21 -0700 Subject: [PATCH 59/67] fix: copy in `install-multi-user.sh` (#10902) --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -754,7 +754,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME From ea8e49bea5c2fdc93b87f52cbb58001e30e5f94d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jun 2024 17:46:10 -0400 Subject: [PATCH 60/67] Force the cpuid option for libutil rather than relying on detection This is more robust, and match's Nixpkgs policy to force enable flags statically by default (a common distro thing). --- src/libutil/package.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/package.nix b/src/libutil/package.nix index a461dccb8..dd93e5663 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -102,6 +102,10 @@ mkDerivation (finalAttrs: { '' ); + mesonFlags = [ + (lib.mesonEnable "cpuid" stdenv.hostPlatform.isx86_64) + ]; + env = { # Needed for Meson to find Boost. # https://github.com/NixOS/nixpkgs/issues/86131. From 81004a05c60d579c935bd300146f4a0c918bdbeb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Jun 2024 09:28:27 -0400 Subject: [PATCH 61/67] Build `nix-store` with Meson MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Special thanks to everyone that has worked on a Meson port so far, @p01arst0rm and @Qyriad in particular. Co-Authored-By: p01arst0rm Co-Authored-By: Artemis Tosini Co-Authored-By: Artemis Tosini Co-Authored-By: Felix Uhl Co-Authored-By: Jade Lovelace Co-Authored-By: Lunaphied Co-Authored-By: Maximilian Bosch Co-Authored-By: Pierre Bourdon Co-Authored-By: Qyriad Co-Authored-By: Rebecca Turner Co-Authored-By: Winter Co-Authored-By: eldritch horrors Co-Authored-By: jade Co-Authored-By: julia Co-Authored-By: rebecca “wiggles” turner Co-Authored-By: wiggles dog Co-Authored-By: fricklerhandwerk Co-authored-by: Eli Schwartz --- flake.nix | 15 +- maintainers/hydra.nix | 6 +- meson.build | 1 + src/libstore/.version | 1 + src/libstore/build/derivation-goal.cc | 2 +- src/libstore/linux/meson.build | 10 + src/libstore/meson.build | 452 ++++++++++++++++++++++++++ src/libstore/meson.options | 25 ++ src/libstore/package.nix | 142 ++++++++ src/libstore/unix/meson.build | 19 ++ src/libstore/windows/meson.build | 11 + 11 files changed, 681 insertions(+), 3 deletions(-) create mode 120000 src/libstore/.version create mode 100644 src/libstore/linux/meson.build create mode 100644 src/libstore/meson.build create mode 100644 src/libstore/meson.options create mode 100644 src/libstore/package.nix create mode 100644 src/libstore/unix/meson.build create mode 100644 src/libstore/windows/meson.build diff --git a/flake.nix b/flake.nix index 599c6dfbe..377247cb8 100644 --- a/flake.nix +++ b/flake.nix @@ -169,6 +169,17 @@ ; }; + nix-store = final.callPackage ./src/libstore/package.nix { + inherit + fileset + stdenv + officialRelease + versionSuffix + ; + libseccomp = final.libseccomp-nix; + busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; + }; + nix = final.callPackage ./package.nix { inherit @@ -260,6 +271,7 @@ { "nix" = { }; "nix-util" = { }; + "nix-store" = { }; } // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = @@ -312,10 +324,11 @@ "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; }; - inherit (pkgs.nix-util) mesonFlags; + mesonFlags = pkgs.nix-util.mesonFlags ++ pkgs.nix-store.mesonFlags; nativeBuildInputs = attrs.nativeBuildInputs or [] ++ pkgs.nix-util.nativeBuildInputs + ++ pkgs.nix-store.nativeBuildInputs ++ [ modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" diff --git a/maintainers/hydra.nix b/maintainers/hydra.nix index 2541b061e..d53647549 100644 --- a/maintainers/hydra.nix +++ b/maintainers/hydra.nix @@ -33,7 +33,11 @@ let doBuild = false; }; - forAllPackages = lib.genAttrs [ "nix" "nix-util" ]; + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-store" + ]; in { # Binary package for various platforms. diff --git a/meson.build b/meson.build index fc6441877..10883d832 100644 --- a/meson.build +++ b/meson.build @@ -7,3 +7,4 @@ project('nix-dev-shell', 'cpp', ) subproject('libutil') +subproject('libstore') diff --git a/src/libstore/.version b/src/libstore/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libstore/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index e72ef4cf9..146a060f3 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -26,7 +26,7 @@ #include #ifndef _WIN32 // TODO abstract over proc exit status -# include +# include #endif #include diff --git a/src/libstore/linux/meson.build b/src/libstore/linux/meson.build new file mode 100644 index 000000000..0c494b5d6 --- /dev/null +++ b/src/libstore/linux/meson.build @@ -0,0 +1,10 @@ +sources += files( + 'personality.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'fchmodat2-compat.hh', + 'personality.hh', +) diff --git a/src/libstore/meson.build b/src/libstore/meson.build new file mode 100644 index 000000000..c9c6f66f1 --- /dev/null +++ b/src/libstore/meson.build @@ -0,0 +1,452 @@ +project('nix-store', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +# See note in ../nix-util/meson.build +deps_private = [ ] + +# See note in ../nix-util/meson.build +deps_public = [ ] + +# See note in ../nix-util/meson.build +deps_other = [ ] + +configdata = configuration_data() + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +configdata.set_quoted('SYSTEM', host_machine.system()) + +nix_util = dependency('nix-util') +if nix_util.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util +else + deps_public += nix_util +endif + +run_command('ln', '-s', + meson.project_build_root() / '__nothing_link_target', + meson.project_build_root() / '__nothing_symlink', + check : true, +) +can_link_symlink = run_command('ln', + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : false, +).returncode() == 0 +run_command('rm', '-f', + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : true, +) +summary('can hardlink to symlink', can_link_symlink, bool_yn : true) +configdata.set('CAN_LINK_SYMLINK', can_link_symlink.to_int()) + +# Check for each of these functions, and create a define like `#define HAVE_LCHOWN 1`. +# +# Only need to do functions that deps (like `libnixutil`) didn't already +# check for. +check_funcs = [ + # Optionally used for canonicalising files from the build + 'lchown', +] +foreach funcspec : check_funcs + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) +endforeach + +has_acl_support = cxx.has_header('sys/xattr.h') \ + and cxx.has_function('llistxattr') \ + and cxx.has_function('lremovexattr') +configdata.set('HAVE_ACL_SUPPORT', has_acl_support.to_int()) + +# This is only conditional to work around +# https://github.com/mesonbuild/meson/issues/13293. It should be +# unconditional. +if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') + deps_private += dependency('threads') +endif + +boost = dependency( + 'boost', + modules : ['container'], +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +curl = dependency('libcurl', 'curl') +deps_private += curl + +# seccomp only makes sense on Linux +is_linux = host_machine.system() == 'linux' +seccomp_required = get_option('seccomp-sandboxing') +if not is_linux and seccomp_required.enabled() + warning('Force-enabling seccomp on non-Linux does not make sense') +endif +seccomp = dependency('libseccomp', 'seccomp', required : seccomp_required, version : '>=2.5.5') +if is_linux and not seccomp.found() + warning('Sandbox security is reduced because libseccomp has not been found! Please provide libseccomp if it supports your CPU architecture.') +endif +configdata.set('HAVE_SECCOMP', seccomp.found().to_int()) +deps_private += seccomp + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') +deps_private += sqlite + + +enable_embedded_sandbox_shell = get_option('embedded-sandbox-shell') +if enable_embedded_sandbox_shell + # This one goes in config.h + # The path to busybox is passed as a -D flag when compiling this_library. + # Idk why, ask the old buildsystem. + configdata.set('HAVE_EMBEDDED_SANDBOX_SHELL', 1) +endif + +generated_headers = [] +foreach header : [ 'schema.sql', 'ca-specific-schema.sql' ] + generated_headers += custom_target( + command : [ 'bash', '-c', '{ echo \'R"__NIX_STR(\' && cat @INPUT@ && echo \')__NIX_STR"\'; } > "$1"', '_ignored_argv0', '@OUTPUT@' ], + input : header, + output : '@PLAINNAME@.gen.hh', + install : true, + install_dir : get_option('includedir') / 'nix' + ) +endforeach + +if enable_embedded_sandbox_shell + hexdump = find_program('hexdump', native : true) + embedded_sandbox_shell_gen = custom_target( + 'embedded-sandbox-shell.gen.hh', + command : [ + hexdump, + '-v', + '-e', + '1/1 "0x%x," "\n"' + ], + input : busybox.full_path(), + output : 'embedded-sandbox-shell.gen.hh', + capture : true, + feed : true, + ) + generated_headers += embedded_sandbox_shell_gen +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-store.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.h', + '-include', 'config-store.h', + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + # FIXME: remove when we get meson 1.4.0 which will default this to on for us: + # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions + '-D_GLIBCXX_ASSERTIONS=1', + language : 'cpp', +) + +sources = files( + 'binary-cache-store.cc', + 'build-result.cc', + 'build/derivation-goal.cc', + 'build/drv-output-substitution-goal.cc', + 'build/entry-points.cc', + 'build/goal.cc', + 'build/substitution-goal.cc', + 'build/worker.cc', + 'builtins/buildenv.cc', + 'builtins/fetchurl.cc', + 'builtins/unpack-channel.cc', + 'common-protocol.cc', + 'content-address.cc', + 'daemon.cc', + 'derivations.cc', + 'derived-path-map.cc', + 'derived-path.cc', + 'downstream-placeholder.cc', + 'dummy-store.cc', + 'export-import.cc', + 'filetransfer.cc', + 'gc.cc', + 'globals.cc', + 'http-binary-cache-store.cc', + 'indirect-root-store.cc', + 'keys.cc', + 'legacy-ssh-store.cc', + 'local-binary-cache-store.cc', + 'local-fs-store.cc', + 'local-overlay-store.cc', + 'local-store.cc', + 'log-store.cc', + 'machines.cc', + 'make-content-addressed.cc', + 'misc.cc', + 'names.cc', + 'nar-accessor.cc', + 'nar-info-disk-cache.cc', + 'nar-info.cc', + 'optimise-store.cc', + 'outputs-spec.cc', + 'parsed-derivations.cc', + 'path-info.cc', + 'path-references.cc', + 'path-with-outputs.cc', + 'path.cc', + 'pathlocks.cc', + 'posix-fs-canonicalise.cc', + 'profiles.cc', + 'realisation.cc', + 'remote-fs-accessor.cc', + 'remote-store.cc', + 's3-binary-cache-store.cc', + 'serve-protocol-connection.cc', + 'serve-protocol.cc', + 'sqlite.cc', + 'ssh-store-config.cc', + 'ssh-store.cc', + 'ssh.cc', + 'store-api.cc', + 'store-reference.cc', + 'uds-remote-store.cc', + 'worker-protocol-connection.cc', + 'worker-protocol.cc', +) + +include_dirs = [ + include_directories('.'), + include_directories('build'), +] + +headers = [config_h] +files( + 'binary-cache-store.hh', + 'build-result.hh', + 'build/derivation-goal.hh', + 'build/drv-output-substitution-goal.hh', + 'build/goal.hh', + 'build/substitution-goal.hh', + 'build/worker.hh', + 'builtins.hh', + 'builtins/buildenv.hh', + 'common-protocol-impl.hh', + 'common-protocol.hh', + 'content-address.hh', + 'daemon.hh', + 'derivations.hh', + 'derived-path-map.hh', + 'derived-path.hh', + 'downstream-placeholder.hh', + 'filetransfer.hh', + 'gc-store.hh', + 'globals.hh', + 'indirect-root-store.hh', + 'keys.hh', + 'legacy-ssh-store.hh', + 'length-prefixed-protocol-helper.hh', + 'local-fs-store.hh', + 'local-overlay-store.hh', + 'local-store.hh', + 'log-store.hh', + 'machines.hh', + 'make-content-addressed.hh', + 'names.hh', + 'nar-accessor.hh', + 'nar-info-disk-cache.hh', + 'nar-info.hh', + 'outputs-spec.hh', + 'parsed-derivations.hh', + 'path-info.hh', + 'path-references.hh', + 'path-regex.hh', + 'path-with-outputs.hh', + 'path.hh', + 'pathlocks.hh', + 'posix-fs-canonicalise.hh', + 'profiles.hh', + 'realisation.hh', + 'remote-fs-accessor.hh', + 'remote-store-connection.hh', + 'remote-store.hh', + 's3-binary-cache-store.hh', + 's3.hh', + 'serve-protocol-connection.hh', + 'serve-protocol-impl.hh', + 'serve-protocol.hh', + 'sqlite.hh', + 'ssh-store-config.hh', + 'ssh.hh', + 'store-api.hh', + 'store-cast.hh', + 'store-dir-config.hh', + 'store-reference.hh', + 'uds-remote-store.hh', + 'worker-protocol-connection.hh', + 'worker-protocol-impl.hh', + 'worker-protocol.hh', +) + +if host_machine.system() == 'linux' + subdir('linux') +endif + +if host_machine.system() == 'windows' + subdir('windows') +else + subdir('unix') +endif + +fs = import('fs') + +prefix = get_option('prefix') +# For each of these paths, assume that it is relative to the prefix unless +# it is already an absolute path (which is the default for store-dir, state-dir, and log-dir). +path_opts = [ + # Meson built-ins. + 'datadir', + 'bindir', + 'mandir', + 'libdir', + 'includedir', + 'libexecdir', + # Homecooked Nix directories. + 'store-dir', + 'state-dir', + 'log-dir', +] +# For your grepping pleasure, this loop sets the following variables that aren't mentioned +# literally above: +# store_dir +# state_dir +# log_dir +# profile_dir +foreach optname : path_opts + varname = optname.replace('-', '_') + path = get_option(optname) + if fs.is_absolute(path) + set_variable(varname, path) + else + set_variable(varname, prefix / path) + endif +endforeach + +# sysconfdir doesn't get anything installed to directly, and is only used to +# tell Nix where to look for nix.conf, so it doesn't get appended to prefix. +sysconfdir = get_option('sysconfdir') +if not fs.is_absolute(sysconfdir) + sysconfdir = '/' / sysconfdir +endif + +lsof = find_program('lsof', required : false) + +# Aside from prefix itself, each of these was made into an absolute path +# by joining it with prefix, unless it was already an absolute path +# (which is the default for store-dir, state-dir, and log-dir). +cpp_str_defines = { + 'NIX_PREFIX': prefix, + 'NIX_STORE_DIR': store_dir, + 'NIX_DATA_DIR': datadir, + 'NIX_STATE_DIR': state_dir / 'nix', + 'NIX_LOG_DIR': log_dir, + 'NIX_CONF_DIR': sysconfdir / 'nix', + 'NIX_BIN_DIR': bindir, + 'NIX_MAN_DIR': mandir, +} + +if lsof.found() + lsof_path = lsof.full_path() +else + # Just look up on the PATH + lsof_path = 'lsof' +endif +cpp_str_defines += { + 'LSOF': lsof_path +} + +#if busybox.found() + cpp_str_defines += { +# 'SANDBOX_SHELL': busybox.full_path() + } +#endif + +cpp_args = [] + +foreach name, value : cpp_str_defines + cpp_args += [ + '-D' + name + '=' + '"' + value + '"' + ] +endforeach + +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # See note in `../nix-util/meson.build` + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif + +this_library = library( + 'nixstore', + generated_headers, + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + cpp_args : cpp_args, + link_args: linker_export_flags, + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +requires = deps_public +if nix_util.type_name() == 'internal' + # `requires` cannot contain declared dependencies (from the + # subproject), so we need to do this manually + requires = [ 'nix-util' ] + requires +endif + +import('pkgconfig').generate( + this_library, + filebase : meson.project_name(), + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : requires, + requires_private : deps_private, + libraries_private : ['-lboost_container'], +) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_library, + compile_args : ['-std=c++2a'], + dependencies : [nix_util], +)) diff --git a/src/libstore/meson.options b/src/libstore/meson.options new file mode 100644 index 000000000..723a8e020 --- /dev/null +++ b/src/libstore/meson.options @@ -0,0 +1,25 @@ +# vim: filetype=meson + +option('embedded-sandbox-shell', type : 'boolean', value : false, + description : 'include the sandbox shell in the Nix binary', +) + +option('seccomp-sandboxing', type : 'feature', + description : 'build support for seccomp sandboxing (recommended unless your arch doesn\'t support libseccomp, only relevant on Linux)', +) + +option('sandbox-shell', type : 'string', value : 'busybox', + description : 'path to a statically-linked shell to use as /bin/sh in sandboxes (usually busybox)', +) + +option('store-dir', type : 'string', value : '/nix/store', + description : 'path of the Nix store', +) + +option('state-dir', type : 'string', value : '/nix/var', + description : 'path to store state in for Nix', +) + +option('log-dir', type : 'string', value : '/nix/var/log/nix', + description : 'path to store logs in for Nix', +) diff --git a/src/libstore/package.nix b/src/libstore/package.nix new file mode 100644 index 000000000..e54dfe597 --- /dev/null +++ b/src/libstore/package.nix @@ -0,0 +1,142 @@ +{ lib +, stdenv +, releaseTools +, fileset + +, meson +, ninja +, pkg-config + +, nix-util +, boost +, curl +, aws-sdk-cpp +, libseccomp +, nlohmann_json +, man +, sqlite + +, busybox-sandbox-shell ? null + +# Configuration Options + +, versionSuffix ? "" +, officialRelease ? false + +# Check test coverage of Nix. Probably want to use with at least +# one of `doCheck` or `doInstallCheck` enabled. +, withCoverageChecks ? false + +# Avoid setting things that would interfere with a functioning devShell +, forDevShell ? false +}: + +let + version = lib.fileContents ./.version + versionSuffix; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: { + pname = "nix-store"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions [ + ./meson.build + ./meson.options + ./linux/meson.build + ./unix/meson.build + ./windows/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "sb") ./.) + (fileset.fileFilter (file: file.hasExt "md") ./.) + (fileset.fileFilter (file: file.hasExt "sql") ./.) + ]; + }; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + boost + curl + sqlite + ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) + (aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }) + ; + + propagatedBuildInputs = [ + nix-util + nlohmann_json + ]; + + disallowedReferences = [ boost ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix + '' + echo ${version} > .version + ''; + + mesonFlags = [ + (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux) + (lib.mesonBool "embedded-sandbox-shell" stdenv.hostPlatform.isStatic) + ] ++ lib.optionals stdenv.hostPlatform.isLinux [ + (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox") + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + postInstall = + # Remove absolute path to boost libs that ends up in `Libs.private` + # by default, and would clash with out `disallowedReferences`. Part + # of the https://github.com/NixOS/nixpkgs/issues/45462 workaround. + '' + sed -i "$out/lib/pkgconfig/nix-store.pc" -e 's, ${lib.getLib boost}[^ ]*,,g' + ''; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = [ "fortify" ]; +}) diff --git a/src/libstore/unix/meson.build b/src/libstore/unix/meson.build new file mode 100644 index 000000000..d9d190131 --- /dev/null +++ b/src/libstore/unix/meson.build @@ -0,0 +1,19 @@ +sources += files( + 'build/child.cc', + 'build/hook-instance.cc', + 'build/local-derivation-goal.cc', + 'pathlocks.cc', + 'user-lock.cc', +) + +include_dirs += include_directories( + '.', + 'build', +) + +headers += files( + 'build/child.hh', + 'build/hook-instance.hh', + 'build/local-derivation-goal.hh', + 'user-lock.hh', +) diff --git a/src/libstore/windows/meson.build b/src/libstore/windows/meson.build new file mode 100644 index 000000000..b81c5b2af --- /dev/null +++ b/src/libstore/windows/meson.build @@ -0,0 +1,11 @@ +sources += files( + 'pathlocks.cc', +) + +include_dirs += include_directories( + '.', + #'build', +) + +headers += files( +) From 5a9e1c0d20e2332c79fb0fd7570315a5d93041f2 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Sat, 15 Jun 2024 14:02:17 +0200 Subject: [PATCH 62/67] Restrict supported tarball formats to actual Tarballs The documentation is clear about the supported formats (with at least `builtins.fetchTarball`). The way the code was written previously it supported all the formats that libarchive supported. That is a surprisingly large amount of formats that are likely not on the radar of the Nix developers and users. Before people end up relying on this (or if they do) it is better to break it now before it becomes a widespread "feature". Zip file support has been retained as (at least to my knowledge) historically that has been used to fetch nixpkgs in some shell expressions *many* years back. Fixes https://github.com/NixOS/nix/issues/10917 --- src/libutil/tarfile.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 6bb2bd2f3..f0e24e937 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -79,7 +79,8 @@ TarArchive::TarArchive(Source & source, bool raw, std::optional com } if (!raw) { - archive_read_support_format_all(archive); + archive_read_support_format_tar(archive); + archive_read_support_format_zip(archive); } else { archive_read_support_format_raw(archive); archive_read_support_format_empty(archive); @@ -96,7 +97,8 @@ TarArchive::TarArchive(const Path & path) , buffer(defaultBufferSize) { archive_read_support_filter_all(archive); - archive_read_support_format_all(archive); + archive_read_support_format_tar(archive); + archive_read_support_format_zip(archive); archive_read_set_option(archive, NULL, "mac-ext", NULL); check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); } From 2894c1b38e961fe2cc22df2a8247773cd012dcac Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Jun 2024 16:34:49 +0200 Subject: [PATCH 63/67] WIP add testresults output --- package.nix | 8 +++++++- tests/unit/libexpr/local.mk | 2 +- tests/unit/libfetchers/local.mk | 2 +- tests/unit/libstore/local.mk | 2 +- tests/unit/libutil/local.mk | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/package.nix b/package.nix index 677ee73c7..5986edeb4 100644 --- a/package.nix +++ b/package.nix @@ -208,7 +208,9 @@ in { # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs || enableExternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -317,6 +319,10 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; + preCheck = '' + mkdir $testresults + ''; + installTargets = lib.optional doBuild "install" ++ lib.optional enableInternalAPIDocs "internal-api-html" ++ lib.optional enableExternalAPIDocs "external-api-html"; diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk index 09a7dfca1..1617e2823 100644 --- a/tests/unit/libexpr/local.mk +++ b/tests/unit/libexpr/local.mk @@ -4,7 +4,7 @@ programs += libexpr-tests libexpr-tests_NAME := libnixexpr-tests -libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libexpr-tests.xml libexpr-tests_DIR := $(d) diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk index d576d28f3..286a59030 100644 --- a/tests/unit/libfetchers/local.mk +++ b/tests/unit/libfetchers/local.mk @@ -4,7 +4,7 @@ programs += libfetchers-tests libfetchers-tests_NAME = libnixfetchers-tests -libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libfetchers-tests.xml libfetchers-tests_DIR := $(d) diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk index 0af1d2622..8d3d6b0af 100644 --- a/tests/unit/libstore/local.mk +++ b/tests/unit/libstore/local.mk @@ -4,7 +4,7 @@ programs += libstore-tests libstore-tests_NAME = libnixstore-tests -libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libstore-tests.xml libstore-tests_DIR := $(d) diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk index b9bddc24d..404f35cf1 100644 --- a/tests/unit/libutil/local.mk +++ b/tests/unit/libutil/local.mk @@ -4,7 +4,7 @@ programs += libutil-tests libutil-tests_NAME = libnixutil-tests -libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libutil-tests.xml libutil-tests_DIR := $(d) From de639ceafe75ae25293f49f170ccad0c3e4a133f Mon Sep 17 00:00:00 2001 From: Jared Baur Date: Sat, 15 Jun 2024 09:06:35 +0000 Subject: [PATCH 64/67] Don't chown when local-store is read-only If the local-store is using the read-only flag, the underlying filesystem might be read-only, thus an attempt to `chown` would always fail. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b44879cc9..676a035fa 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -233,7 +233,7 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); - else { + else if (!readOnly) { struct stat st; if (stat(realStoreDir.get().c_str(), &st)) throw SysError("getting attributes of path '%1%'", realStoreDir); From b0cfac8f93b3dc22df6bff14b12edc9d00a1d439 Mon Sep 17 00:00:00 2001 From: PoweredByPie Date: Mon, 17 Jun 2024 00:03:50 -0700 Subject: [PATCH 65/67] Fix compile error on windows --- src/libutil/file-descriptor.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index 492b67d74..be61375f6 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -157,7 +157,7 @@ void closeOnExec(Descriptor fd); #endif #if defined(_WIN32) && _WIN32_WINNT >= 0x0600 -namespace windows ( +namespace windows { Path handleToPath(Descriptor handle); std::wstring handleToFileName(Descriptor handle); From 5e806673c3b7dda4a6dcf4fc45a0aa292751e6cb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Jun 2024 08:29:02 -0400 Subject: [PATCH 66/67] Make `hydraJobs.build` include the constituent packages We were only doing that for the more exotic builds, just forgot. --- maintainers/hydra.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maintainers/hydra.nix b/maintainers/hydra.nix index d53647549..8ccf2f951 100644 --- a/maintainers/hydra.nix +++ b/maintainers/hydra.nix @@ -41,7 +41,8 @@ let in { # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); + build = forAllPackages (pkgName: + forAllSystems (system: self.packages.${system}.${pkgName})); shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); From c9cdc2423acb84714f2434333dbadfd1acaa9d26 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Jun 2024 09:04:41 -0400 Subject: [PATCH 67/67] Temporarily remove the Meson builds from `packages` in the flake This will avoid some out-of-memory issues in GitHub actions that result from num jobs > 1 and num cores = 4. Once we only have the Meson build system, this problem should go away, and we can reenable these jobs. --- flake.nix | 7 +++++-- maintainers/hydra.nix | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 377247cb8..9f494cb15 100644 --- a/flake.nix +++ b/flake.nix @@ -270,8 +270,11 @@ (lib.genAttrs stdenvs (_: { }))) { "nix" = { }; - "nix-util" = { }; - "nix-store" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; } // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = diff --git a/maintainers/hydra.nix b/maintainers/hydra.nix index 8ccf2f951..cc0dadac9 100644 --- a/maintainers/hydra.nix +++ b/maintainers/hydra.nix @@ -42,7 +42,7 @@ in { # Binary package for various platforms. build = forAllPackages (pkgName: - forAllSystems (system: self.packages.${system}.${pkgName})); + forAllSystems (system: nixpkgsFor.${system}.native.${pkgName})); shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);