From 111aeb9e92263b72478414dfb84c62be21ee64a2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 Mar 2023 23:00:14 -0400 Subject: [PATCH 001/211] Testing overlayfs stores --- tests/init.sh | 2 +- tests/local.mk | 3 +- tests/overlay-local-store.sh | 3 ++ tests/overlay-local-store/inner.sh | 51 ++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 tests/overlay-local-store.sh create mode 100755 tests/overlay-local-store/inner.sh diff --git a/tests/init.sh b/tests/init.sh index c420e8c9f..2ec5fa1dd 100755 --- a/tests/init.sh +++ b/tests/init.sh @@ -3,7 +3,7 @@ source common/vars-and-functions.sh test -n "$TEST_ROOT" if test -d "$TEST_ROOT"; then - chmod -R u+w "$TEST_ROOT" + chmod -R u+rw "$TEST_ROOT" # We would delete any daemon socket, so let's stop the daemon first. killDaemon rm -rf "$TEST_ROOT" diff --git a/tests/local.mk b/tests/local.mk index 328f27e90..05894727c 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -124,7 +124,8 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - toString-path.sh + toString-path.sh \ + overlay-local-store.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/overlay-local-store.sh b/tests/overlay-local-store.sh new file mode 100644 index 000000000..6c0a5f96d --- /dev/null +++ b/tests/overlay-local-store.sh @@ -0,0 +1,3 @@ +source common.sh + +exec unshare --mount --map-root-user overlay-local-store/inner.sh diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh new file mode 100755 index 000000000..fb690a85f --- /dev/null +++ b/tests/overlay-local-store/inner.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +export NIX_CONFIG='build-users-group = ' + +# Creating testing directories + +storeA="$TEST_ROOT/store_a" +storeB="$TEST_ROOT/store_b?real=$TEST_ROOT/merged-store" +storeBTop="$TEST_ROOT/store_b" + +mkdir -p "$TEST_ROOT"/{store_a,store_b,merged-store,workdir} + +# Mounting Overlay Store + +nix-store --store "$TEST_ROOT/store_a" --add dummy +nix-store --store "$TEST_ROOT/store_b" --add dummy + +mount -t overlay overlay \ + -o lowerdir="$TEST_ROOT/store_a/nix/store" \ + -o upperdir="$TEST_ROOT/store_b/nix/store" \ + -o workdir="$TEST_ROOT/workdir" \ + "$TEST_ROOT/merged-store" || skipTest "overlayfs is not supported" + +path_a=$(nix-build '' -A signal-desktop --store "$storeA") + +# Checking for Path in store_a +stat "$TEST_ROOT/store_a/$path_a" + +# Checking for Path in store_b +expect 1 stat "$TEST_ROOT/store_b/$path_a" + +# Checking for Path in merged-store +ls "$TEST_ROOT/merged-store/$(echo "$path_a" | sed 's|/nix/store/||g')" + + +# Verifying path in store_a +nix-store --verify-path --store "$storeA" "$path_a" + +# Verifiying path in merged-store (Should fail) +expect 1 nix-store --verify-path --store "$storeB" "$path_a" + +# Verifying path in store_b (Should fail) +expect 1 nix-store --verify-path --store "$storeBTop" "$path_a" + +path_b=$(nix-build '' -A signal-desktop --store "$storeB") From f0a176e2f1061475546ade22b81343e4b4967568 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 10:20:06 -0400 Subject: [PATCH 002/211] Init local overlay store --- src/libstore/local-overlay-store.cc | 20 ++++++++ src/libstore/local-overlay-store.hh | 74 +++++++++++++++++++++++++++++ src/libstore/store-api.cc | 3 ++ 3 files changed, 97 insertions(+) create mode 100644 src/libstore/local-overlay-store.cc create mode 100644 src/libstore/local-overlay-store.hh diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc new file mode 100644 index 000000000..4cb50aef0 --- /dev/null +++ b/src/libstore/local-overlay-store.cc @@ -0,0 +1,20 @@ +#include "local-overlay-store.hh" + +namespace nix { + +LocalOverlayStore::LocalOverlayStore(const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) + , LocalOverlayStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , LocalStore(params) + , lowerStore(openStore(lowerStoreUri).dynamic_pointer_cast()) +{ +} + + +static RegisterStoreImplementation regLocalOverlayStore; + +} diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh new file mode 100644 index 000000000..3129bb7a0 --- /dev/null +++ b/src/libstore/local-overlay-store.hh @@ -0,0 +1,74 @@ +#include "local-store.hh" + +namespace nix { + +/** + * Configuration for `LocalOverlayStore`. + */ +struct LocalOverlayStoreConfig : virtual LocalStoreConfig +{ + // FIXME why doesn't this work? + // using LocalStoreConfig::LocalStoreConfig; + + LocalOverlayStoreConfig(const StringMap & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) + { } + + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + )"}; + + const std::string name() override { return "Experimental Local Overlay Store"; } + + std::string doc() override + { + return + "" + // FIXME write docs + //#include "local-overlay-store.md" + ; + } +}; + +/** + * Variation of local store using overlayfs for the store dir. + */ +class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore +{ + /** + * The store beneath us. + * + * Our store dir should be an overlay fs where the lower layer + * is that store's store dir, and the upper layer is some + * scratch storage just for us. + */ + ref lowerStore; + +public: + LocalOverlayStore(const Params & params); + + LocalOverlayStore(std::string scheme, std::string path, const Params & params) + : LocalOverlayStore(params) + { + throw UnimplementedError("LocalOverlayStore"); + } + + static std::set uriSchemes() + { return {}; } + + std::string getUri() override + { + return "local-overlay"; + } + +private: + // Overridden methods… +}; + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8fc3ed46a..c9ea03a1a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -10,6 +10,7 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +#include "local-overlay-store.hh" #include #include @@ -1391,6 +1392,8 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para return std::make_shared(params); } else if (uri == "local") { return std::make_shared(params); + } else if (uri == "local-overlay") { + return std::make_shared(params); } else if (isNonUriPath(uri)) { Store::Params params2 = params; params2["root"] = absPath(uri); From f08754a97a22a97fe48da65517f8fb95292ab251 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 14:47:39 -0400 Subject: [PATCH 003/211] Progress on tests --- tests/hermetic.nix | 56 ++++++++++++++++++++++++++++++ tests/overlay-local-store.sh | 4 +++ tests/overlay-local-store/inner.sh | 16 +++++++-- 3 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 tests/hermetic.nix diff --git a/tests/hermetic.nix b/tests/hermetic.nix new file mode 100644 index 000000000..7ccd0fd72 --- /dev/null +++ b/tests/hermetic.nix @@ -0,0 +1,56 @@ +{ busybox }: + +with import ./config.nix; + +let + contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1"; + caArgs = if contentAddressedByDefault then { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } else {}; + + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + } // removeAttrs args ["builder" "meta" "passthru"] + // caArgs) + // { meta = args.meta or {}; passthru = args.passthru or {}; }; + + input1 = mkDerivation { + shell = busybox; + name = "build-remote-input-1"; + buildCommand = "echo hi-input1; echo FOO > $out"; + }; + + input2 = mkDerivation { + shell = busybox; + name = "build-remote-input-2"; + buildCommand = "echo hi; echo BAR > $out"; + }; + + input3 = mkDerivation { + shell = busybox; + name = "build-remote-input-3"; + buildCommand = '' + echo hi-input3 + read x < ${input2} + echo $x BAZ > $out + ''; + }; + +in + + mkDerivation { + shell = busybox; + name = "build-remote"; + passthru = { inherit input1 input2 input3; }; + buildCommand = + '' + read x < ${input1} + read y < ${input3} + echo "$x $y" > $out + ''; + } diff --git a/tests/overlay-local-store.sh b/tests/overlay-local-store.sh index 6c0a5f96d..c98931118 100644 --- a/tests/overlay-local-store.sh +++ b/tests/overlay-local-store.sh @@ -1,3 +1,7 @@ source common.sh +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" +if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi + exec unshare --mount --map-root-user overlay-local-store/inner.sh diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index fb690a85f..5079dd0bd 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -11,13 +11,16 @@ export NIX_CONFIG='build-users-group = ' # Creating testing directories storeA="$TEST_ROOT/store_a" -storeB="$TEST_ROOT/store_b?real=$TEST_ROOT/merged-store" +storeB="local-overlay?root=$TEST_ROOT/store_b&lower-store=$TEST_ROOT/merged-store" storeBTop="$TEST_ROOT/store_b" mkdir -p "$TEST_ROOT"/{store_a,store_b,merged-store,workdir} # Mounting Overlay Store +## Restore normal, because we are using these chroot stores +#NIX_STORE_DIR=/nix/store + nix-store --store "$TEST_ROOT/store_a" --add dummy nix-store --store "$TEST_ROOT/store_b" --add dummy @@ -27,7 +30,14 @@ mount -t overlay overlay \ -o workdir="$TEST_ROOT/workdir" \ "$TEST_ROOT/merged-store" || skipTest "overlayfs is not supported" -path_a=$(nix-build '' -A signal-desktop --store "$storeA") +# Add in lower +NIX_REMOTE=$storeA source add.sh + +# Add in layered +NIX_REMOTE=$storeB source add.sh + +#busyboxExpr="\"\${$(dirname "$busybox")}/$(basename "$busybox")\"" +path_a=$(nix-build ./hermetic.nix --arg busybox "$busybox" --store "$storeA") # Checking for Path in store_a stat "$TEST_ROOT/store_a/$path_a" @@ -48,4 +58,4 @@ expect 1 nix-store --verify-path --store "$storeB" "$path_a" # Verifying path in store_b (Should fail) expect 1 nix-store --verify-path --store "$storeBTop" "$path_a" -path_b=$(nix-build '' -A signal-desktop --store "$storeB") +path_b=$(nix-build ./hermetic.nix --arg busybox $busybox --store "$storeB") From d80fc2ac1b39b4d81425d1aa614d4d06e5cd8640 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 16:00:47 -0400 Subject: [PATCH 004/211] First round of testing, with todos --- tests/hermetic.nix | 12 ++--- tests/overlay-local-store.sh | 3 ++ tests/overlay-local-store/inner.sh | 85 +++++++++++++++++++----------- 3 files changed, 64 insertions(+), 36 deletions(-) diff --git a/tests/hermetic.nix b/tests/hermetic.nix index 7ccd0fd72..4c9d7a51f 100644 --- a/tests/hermetic.nix +++ b/tests/hermetic.nix @@ -1,4 +1,4 @@ -{ busybox }: +{ busybox, seed }: with import ./config.nix; @@ -21,19 +21,19 @@ let input1 = mkDerivation { shell = busybox; - name = "build-remote-input-1"; - buildCommand = "echo hi-input1; echo FOO > $out"; + name = "hermetic-input-1"; + buildCommand = "echo hi-input1 seed=${toString seed}; echo FOO > $out"; }; input2 = mkDerivation { shell = busybox; - name = "build-remote-input-2"; + name = "hermetic-input-2"; buildCommand = "echo hi; echo BAR > $out"; }; input3 = mkDerivation { shell = busybox; - name = "build-remote-input-3"; + name = "hermetic-input-3"; buildCommand = '' echo hi-input3 read x < ${input2} @@ -45,7 +45,7 @@ in mkDerivation { shell = busybox; - name = "build-remote"; + name = "hermetic"; passthru = { inherit input1 input2 input3; }; buildCommand = '' diff --git a/tests/overlay-local-store.sh b/tests/overlay-local-store.sh index c98931118..242c16e20 100644 --- a/tests/overlay-local-store.sh +++ b/tests/overlay-local-store.sh @@ -3,5 +3,8 @@ source common.sh requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi +needLocalStore "The test uses --store always so we would just be bypassing the daemon" + +echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf exec unshare --mount --map-root-user overlay-local-store/inner.sh diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 5079dd0bd..524c801bd 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -10,52 +10,77 @@ export NIX_CONFIG='build-users-group = ' # Creating testing directories -storeA="$TEST_ROOT/store_a" -storeB="local-overlay?root=$TEST_ROOT/store_b&lower-store=$TEST_ROOT/merged-store" -storeBTop="$TEST_ROOT/store_b" +storeA="$TEST_ROOT/store-a" +storeBTop="$TEST_ROOT/store-b" +storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$TEST_ROOT/store-a" -mkdir -p "$TEST_ROOT"/{store_a,store_b,merged-store,workdir} +mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} # Mounting Overlay Store -## Restore normal, because we are using these chroot stores -#NIX_STORE_DIR=/nix/store +# Init lower store with some stuff +nix-store --store "$storeA" --add dummy -nix-store --store "$TEST_ROOT/store_a" --add dummy -nix-store --store "$TEST_ROOT/store_b" --add dummy +# Build something in lower store +path=$(nix-build ./hermetic.nix --arg busybox "$busybox" --arg seed 1 --store "$storeA") mount -t overlay overlay \ - -o lowerdir="$TEST_ROOT/store_a/nix/store" \ - -o upperdir="$TEST_ROOT/store_b/nix/store" \ + -o lowerdir="$storeA/nix/store" \ + -o upperdir="$storeBTop" \ -o workdir="$TEST_ROOT/workdir" \ - "$TEST_ROOT/merged-store" || skipTest "overlayfs is not supported" + "$TEST_ROOT/merged-store/nix/store" \ + || skipTest "overlayfs is not supported" -# Add in lower -NIX_REMOTE=$storeA source add.sh +cleanupOverlay () { + umount "$TEST_ROOT/merged-store/nix/store" + rm -r $TEST_ROOT/workdir +} +trap cleanupOverlay EXIT -# Add in layered -NIX_REMOTE=$storeB source add.sh +toRealPath () { + storeDir=$1; shift + storePath=$1; shift + echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") +} -#busyboxExpr="\"\${$(dirname "$busybox")}/$(basename "$busybox")\"" -path_a=$(nix-build ./hermetic.nix --arg busybox "$busybox" --store "$storeA") +### Check status -# Checking for Path in store_a -stat "$TEST_ROOT/store_a/$path_a" +# Checking for path in lower layer +stat $(toRealPath "$storeA/nix/store" "$path") -# Checking for Path in store_b -expect 1 stat "$TEST_ROOT/store_b/$path_a" +# Checking for path in upper layer (should fail) +expect 1 stat $(toRealPath "$storeBTop" "$path") -# Checking for Path in merged-store -ls "$TEST_ROOT/merged-store/$(echo "$path_a" | sed 's|/nix/store/||g')" +# Checking for path in overlay store matching lower layer +diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-store/nix/store" "$path") +# Verifying path in lower layer +nix-store --verify-path --store "$storeA" "$path" -# Verifying path in store_a -nix-store --verify-path --store "$storeA" "$path_a" +# Verifying path in merged-store +# FIXME should succeed +expect 1 nix-store --verify-path --store "$storeB" "$path" -# Verifiying path in merged-store (Should fail) -expect 1 nix-store --verify-path --store "$storeB" "$path_a" +### Do a redundant add -# Verifying path in store_b (Should fail) -expect 1 nix-store --verify-path --store "$storeBTop" "$path_a" +path=$(nix-store --store "$storeB" --add dummy) -path_b=$(nix-build ./hermetic.nix --arg busybox $busybox --store "$storeB") +# lower store should have it from before +stat $(toRealPath "$storeA/nix/store" "$path") + +# upper store should not have redundant copy +# FIXME should fail +stat $(toRealPath "$storeA/nix/store" "$path") + +### Do a build in overlay store + +path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB") + +# Checking for path in lower layer (should fail) +expect 1 stat $(toRealPath "$storeA/nix/store" "$path") + +# Checking for path in upper layer +stat $(toRealPath "$storeBTop" "$path") + +# Verifying path in overlay store +nix-store --verify-path --store "$storeB" "$path" From 0193c2abcd28cb4f5de73601360a1ba13de633d9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 16:03:32 -0400 Subject: [PATCH 005/211] Improve tests slightly --- tests/overlay-local-store/inner.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 524c801bd..ccf24b7b0 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -63,12 +63,15 @@ expect 1 nix-store --verify-path --store "$storeB" "$path" ### Do a redundant add +# upper layer should not have it +expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") + path=$(nix-store --store "$storeB" --add dummy) # lower store should have it from before stat $(toRealPath "$storeA/nix/store" "$path") -# upper store should not have redundant copy +# upper layer should still not have it (no redundant copy) # FIXME should fail stat $(toRealPath "$storeA/nix/store" "$path") From 31e98ed0a0b43596466f4980a1d3e49bc8e7a928 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 16:48:55 -0400 Subject: [PATCH 006/211] Specialize `LocalOverlayStore::registerDrvOutput` --- src/libstore/local-overlay-store.cc | 11 +++++++++++ src/libstore/local-overlay-store.hh | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 4cb50aef0..2e3564df9 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -15,6 +15,17 @@ LocalOverlayStore::LocalOverlayStore(const Params & params) } +void LocalOverlayStore::registerDrvOutput(const Realisation & info) +{ + // First do queryRealisation on lower layer to populate DB + auto res = lowerStore->queryRealisation(info.id); + if (res) + LocalStore::registerDrvOutput(*res); + + LocalStore::registerDrvOutput(info); +} + + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 3129bb7a0..e5329f5d9 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -69,6 +69,8 @@ public: private: // Overridden methods… + + void registerDrvOutput(const Realisation & info) override; }; } From 5406256d78244cf838eb917c35895035cbe1e402 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 17:30:17 -0400 Subject: [PATCH 007/211] Specialize `LocalOverlayStore::queryPathInfoUncached` --- src/libstore/local-overlay-store.cc | 26 ++++++++++++++++++++++++++ src/libstore/local-overlay-store.hh | 3 +++ 2 files changed, 29 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 2e3564df9..545e8d28f 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,4 +1,5 @@ #include "local-overlay-store.hh" +#include "callback.hh" namespace nix { @@ -26,6 +27,31 @@ void LocalOverlayStore::registerDrvOutput(const Realisation & info) } +void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + + auto callbackPtr = std::make_shared(std::move(callback)); + + // If we don't have it, check lower store + LocalStore::queryPathInfoUncached(path, + {[this, path, callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get()); + } catch (...) { + lowerStore->queryPathInfo(path, + {[path, callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get().get_ptr()); + } catch (...) { + callbackPtr->rethrow(); + } + }}); + } + }}); +} + + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index e5329f5d9..12b705efb 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -71,6 +71,9 @@ private: // Overridden methods… void registerDrvOutput(const Realisation & info) override; + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; }; } From 59a8099038179d2901a69b122d33d14544cd6e0d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 17:37:40 -0400 Subject: [PATCH 008/211] Fix `LocalOverlayStore::queryPathInfoUncached`, FIXME in test --- src/libstore/local-overlay-store.cc | 23 +++++++++++++---------- tests/overlay-local-store/inner.sh | 3 +-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 545e8d28f..bd48c12e4 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -33,21 +33,24 @@ void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, auto callbackPtr = std::make_shared(std::move(callback)); - // If we don't have it, check lower store LocalStore::queryPathInfoUncached(path, {[this, path, callbackPtr](std::future> fut) { try { - (*callbackPtr)(fut.get()); + auto info = fut.get(); + if (info) + return (*callbackPtr)(std::move(info)); } catch (...) { - lowerStore->queryPathInfo(path, - {[path, callbackPtr](std::future> fut) { - try { - (*callbackPtr)(fut.get().get_ptr()); - } catch (...) { - callbackPtr->rethrow(); - } - }}); + return callbackPtr->rethrow(); } + // If we don't have it, check lower store + lowerStore->queryPathInfo(path, + {[path, callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get().get_ptr()); + } catch (...) { + return callbackPtr->rethrow(); + } + }}); }}); } diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index ccf24b7b0..69c6f6d0c 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -58,8 +58,7 @@ diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-s nix-store --verify-path --store "$storeA" "$path" # Verifying path in merged-store -# FIXME should succeed -expect 1 nix-store --verify-path --store "$storeB" "$path" +nix-store --verify-path --store "$storeB" "$path" ### Do a redundant add From b3d320c5946deb77bc76b86d4b4655ebfd502471 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 18:50:16 -0400 Subject: [PATCH 009/211] Convert more methods Fixed one test, broke another --- src/libstore/local-overlay-store.cc | 80 ++++++++++++++++++++++++++++- src/libstore/local-overlay-store.hh | 29 +++++++++++ tests/overlay-local-store/inner.sh | 29 ++++++----- 3 files changed, 123 insertions(+), 15 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index bd48c12e4..a98d2e8fd 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -3,6 +3,12 @@ namespace nix { +Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { + auto res = upperLayer + "/" + path.to_string(); + warn("upper path: %s", res); + return res; +} + LocalOverlayStore::LocalOverlayStore(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) @@ -30,7 +36,6 @@ void LocalOverlayStore::registerDrvOutput(const Realisation & info) void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { - auto callbackPtr = std::make_shared(std::move(callback)); LocalStore::queryPathInfoUncached(path, @@ -55,6 +60,79 @@ void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, } +void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput, + Callback> callback) noexcept +{ + auto callbackPtr = std::make_shared(std::move(callback)); + + LocalStore::queryRealisationUncached(drvOutput, + {[this, drvOutput, callbackPtr](std::future> fut) { + try { + auto info = fut.get(); + if (info) + return (*callbackPtr)(std::move(info)); + } catch (...) { + return callbackPtr->rethrow(); + } + // If we don't have it, check lower store + lowerStore->queryRealisation(drvOutput, + {[callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get()); + } catch (...) { + return callbackPtr->rethrow(); + } + }}); + }}); +} + + +bool LocalOverlayStore::isValidPathUncached(const StorePath & path) +{ + auto res = LocalStore::isValidPathUncached(path); + if (res) return res; + return lowerStore->isValidPath(path); +} + + +void LocalOverlayStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + LocalStore::addToStore(info, source, repair, checkSigs); + if (lowerStore->isValidPath(info.path)) { + // dedup stores + deletePath(toUpperPath(info.path)); + } +} + + +StorePath LocalOverlayStore::addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) +{ + auto path = LocalStore::addToStoreFromDump(dump, name, method, hashAlgo, repair, references); + if (lowerStore->isValidPath(path)) { + // dedup stores + deletePath(toUpperPath(path)); + } + return path; +} + + +StorePath LocalOverlayStore::addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) +{ + auto path = LocalStore::addTextToStore(name, s, references, repair); + if (lowerStore->isValidPath(path)) { + // dedup stores + deletePath(toUpperPath(path)); + } + return path; +} + + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 12b705efb..c087b3893 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -22,6 +22,12 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )"}; + + const Setting upperLayer{(StoreConfig*) this, "", "upper-layer", + R"( + Must be used as OverlayFS upper layer for this store's store dir. )"}; const std::string name() override { return "Experimental Local Overlay Store"; } @@ -34,6 +40,12 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig //#include "local-overlay-store.md" ; } + + /** + * Given a store path, get its location (if it is exists) in the + * upper layer of the overlayfs. + */ + Path toUpperPath(const StorePath & path); }; /** @@ -74,6 +86,23 @@ private: void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept override; + + bool isValidPathUncached(const StorePath & path) override; + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) override; + + StorePath addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override; + + void queryRealisationUncached(const DrvOutput&, + Callback> callback) noexcept override; }; } diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 69c6f6d0c..d9d3b76cf 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -12,7 +12,7 @@ export NIX_CONFIG='build-users-group = ' storeA="$TEST_ROOT/store-a" storeBTop="$TEST_ROOT/store-b" -storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$TEST_ROOT/store-a" +storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} @@ -71,18 +71,19 @@ path=$(nix-store --store "$storeB" --add dummy) stat $(toRealPath "$storeA/nix/store" "$path") # upper layer should still not have it (no redundant copy) -# FIXME should fail -stat $(toRealPath "$storeA/nix/store" "$path") +expect 1 stat $(toRealPath "$storeB/nix/store" "$path") -### Do a build in overlay store +## Ooops something went wrong -path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB") - -# Checking for path in lower layer (should fail) -expect 1 stat $(toRealPath "$storeA/nix/store" "$path") - -# Checking for path in upper layer -stat $(toRealPath "$storeBTop" "$path") - -# Verifying path in overlay store -nix-store --verify-path --store "$storeB" "$path" +## ### Do a build in overlay store +## +## path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB") +## +## # Checking for path in lower layer (should fail) +## expect 1 stat $(toRealPath "$storeA/nix/store" "$path") +## +## # Checking for path in upper layer +## stat $(toRealPath "$storeBTop" "$path") +## +## # Verifying path in overlay store +## nix-store --verify-path --store "$storeB" "$path" From ddaf2750b5af28d56b99f79d3ac20ae11d3464d3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 10:22:38 -0400 Subject: [PATCH 010/211] Specialize more methods, fix tests --- src/libstore/local-overlay-store.cc | 30 +++++++++++++++++++++++++---- src/libstore/local-overlay-store.hh | 2 ++ src/libstore/local-store.hh | 2 +- tests/overlay-local-store/inner.sh | 26 ++++++++++++------------- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index a98d2e8fd..ad1947698 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -4,9 +4,7 @@ namespace nix { Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { - auto res = upperLayer + "/" + path.to_string(); - warn("upper path: %s", res); - return res; + return upperLayer + "/" + path.to_string(); } LocalOverlayStore::LocalOverlayStore(const Params & params) @@ -91,7 +89,31 @@ bool LocalOverlayStore::isValidPathUncached(const StorePath & path) { auto res = LocalStore::isValidPathUncached(path); if (res) return res; - return lowerStore->isValidPath(path); + res = lowerStore->isValidPath(path); + if (res) { + // Get path info from lower store so upper DB genuinely has it. + LocalStore::registerValidPath(*lowerStore->queryPathInfo(path)); + } + return res; +} + + +void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) +{ + // First, get any from lower store so we merge + { + StorePathSet notInUpper; + for (auto & [p, _] : infos) + if (!LocalStore::isValidPathUncached(p)) // avoid divergence + notInUpper.insert(p); + auto pathsInLower = lowerStore->queryValidPaths(notInUpper); + ValidPathInfos inLower; + for (auto & p : pathsInLower) + inLower.insert_or_assign(p, *lowerStore->queryPathInfo(p)); + LocalStore::registerValidPaths(inLower); + } + // Then do original request + LocalStore::registerValidPaths(infos); } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index c087b3893..daebe8547 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -89,6 +89,8 @@ private: bool isValidPathUncached(const StorePath & path) override; + void registerValidPaths(const ValidPathInfos & infos) override; + void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 55add18dd..b61ec3590 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -232,7 +232,7 @@ public: */ void registerValidPath(const ValidPathInfo & info); - void registerValidPaths(const ValidPathInfos & infos); + virtual void registerValidPaths(const ValidPathInfos & infos); unsigned int getProtocol() override; diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index d9d3b76cf..dc8ec8c2c 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -22,7 +22,7 @@ mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} nix-store --store "$storeA" --add dummy # Build something in lower store -path=$(nix-build ./hermetic.nix --arg busybox "$busybox" --arg seed 1 --store "$storeA") +path=$(nix-build ./hermetic.nix --arg busybox "$busybox" --arg seed 1 --store "$storeA" --no-out-link) mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ @@ -73,17 +73,15 @@ stat $(toRealPath "$storeA/nix/store" "$path") # upper layer should still not have it (no redundant copy) expect 1 stat $(toRealPath "$storeB/nix/store" "$path") -## Ooops something went wrong +### Do a build in overlay store -## ### Do a build in overlay store -## -## path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB") -## -## # Checking for path in lower layer (should fail) -## expect 1 stat $(toRealPath "$storeA/nix/store" "$path") -## -## # Checking for path in upper layer -## stat $(toRealPath "$storeBTop" "$path") -## -## # Verifying path in overlay store -## nix-store --verify-path --store "$storeB" "$path" +path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) + +# Checking for path in lower layer (should fail) +expect 1 stat $(toRealPath "$storeA/nix/store" "$path") + +# Checking for path in upper layer +stat $(toRealPath "$storeBTop" "$path") + +# Verifying path in overlay store +nix-store --verify-path --store "$storeB" "$path" From e7c3399ed2e62ecf8a0fb5294f19ffab2061cc04 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 10:40:10 -0400 Subject: [PATCH 011/211] Specialize `LocalOverlayStore::queryPathFromHashPart` With test --- src/libstore/local-overlay-store.cc | 10 ++++++++++ src/libstore/local-overlay-store.hh | 2 ++ tests/overlay-local-store/inner.sh | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ad1947698..cb409c2bd 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -98,6 +98,16 @@ bool LocalOverlayStore::isValidPathUncached(const StorePath & path) } +std::optional LocalOverlayStore::queryPathFromHashPart(const std::string & hashPart) +{ + auto res = LocalStore::queryPathFromHashPart(hashPart); + if (res) + return res; + else + return lowerStore->queryPathFromHashPart(hashPart); +} + + void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) { // First, get any from lower store so we merge diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index daebe8547..88da4ba4a 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -89,6 +89,8 @@ private: bool isValidPathUncached(const StorePath & path) override; + std::optional queryPathFromHashPart(const std::string & hashPart) override; + void registerValidPaths(const ValidPathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index dc8ec8c2c..279f005a9 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -60,6 +60,14 @@ nix-store --verify-path --store "$storeA" "$path" # Verifying path in merged-store nix-store --verify-path --store "$storeB" "$path" +hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') + +# Lower store can find from hash part +[[ $(nix store --store $storeA path-from-hash-part $hashPart) == $path ]] + +# merged store can find from hash part +[[ $(nix store --store $storeB path-from-hash-part $hashPart) == $path ]] + ### Do a redundant add # upper layer should not have it From 5059be53b1553f99b95f31227998de69ecf74d90 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 16:42:28 -0400 Subject: [PATCH 012/211] Fix recursive ingestion from lower store --- src/libstore/local-overlay-store.cc | 7 ++++++- tests/overlay-local-store/inner.sh | 15 +++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index cb409c2bd..431b82c90 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -92,7 +92,12 @@ bool LocalOverlayStore::isValidPathUncached(const StorePath & path) res = lowerStore->isValidPath(path); if (res) { // Get path info from lower store so upper DB genuinely has it. - LocalStore::registerValidPath(*lowerStore->queryPathInfo(path)); + auto p = lowerStore->queryPathInfo(path); + // recur on references, syncing entire closure. + for (auto & r : p->references) + if (r != path) + isValidPath(r); + LocalStore::registerValidPath(*p); } return res; } diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 279f005a9..1cb29ceb7 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -22,7 +22,8 @@ mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} nix-store --store "$storeA" --add dummy # Build something in lower store -path=$(nix-build ./hermetic.nix --arg busybox "$busybox" --arg seed 1 --store "$storeA" --no-out-link) +drvPath=$(nix-instantiate --store $storeA ./hermetic.nix --arg busybox "$busybox" --arg seed 1) +path=$(nix-store --store "$storeA" --realise $drvPath) mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ @@ -38,9 +39,9 @@ cleanupOverlay () { trap cleanupOverlay EXIT toRealPath () { - storeDir=$1; shift - storePath=$1; shift - echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") + storeDir=$1; shift + storePath=$1; shift + echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") } ### Check status @@ -60,6 +61,12 @@ nix-store --verify-path --store "$storeA" "$path" # Verifying path in merged-store nix-store --verify-path --store "$storeB" "$path" +[[ \ + $(nix-store --store $storeA --query --outputs $drvPath) \ + == \ + $(nix-store --store $storeB --query --outputs $drvPath) \ + ]] + hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') # Lower store can find from hash part From 8339c170d781825acd55f30af4d76c5c14d50511 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 16:49:44 -0400 Subject: [PATCH 013/211] More tests --- tests/overlay-local-store/inner.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 1cb29ceb7..d13d8b9ab 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -55,18 +55,26 @@ expect 1 stat $(toRealPath "$storeBTop" "$path") # Checking for path in overlay store matching lower layer diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-store/nix/store" "$path") -# Verifying path in lower layer -nix-store --verify-path --store "$storeA" "$path" - -# Verifying path in merged-store -nix-store --verify-path --store "$storeB" "$path" +# Checking derivers query agreement +[[ \ + $(nix-store --store $storeA --query --deriver $path) \ + == \ + $(nix-store --store $storeB --query --deriver $path) \ + ]] +# Checking outputs query agreement [[ \ $(nix-store --store $storeA --query --outputs $drvPath) \ == \ $(nix-store --store $storeB --query --outputs $drvPath) \ ]] +# Verifying path in lower layer +nix-store --verify-path --store "$storeA" "$path" + +# Verifying path in merged-store +nix-store --verify-path --store "$storeB" "$path" + hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') # Lower store can find from hash part From 4173743a3c1f0418c298d12cc71ffa26f6bc848d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 May 2023 17:20:58 -0400 Subject: [PATCH 014/211] Implement more queries --- src/libstore/local-overlay-store.cc | 16 ++++++++++++++++ src/libstore/local-overlay-store.hh | 4 ++++ tests/overlay-local-store/inner.sh | 15 +++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 431b82c90..c6b721b48 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -103,6 +103,22 @@ bool LocalOverlayStore::isValidPathUncached(const StorePath & path) } +void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & referrers) +{ + LocalStore::queryReferrers(path, referrers); + lowerStore->queryReferrers(path, referrers); +} + + +StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) +{ + auto res = LocalStore::queryValidDerivers(path); + for (auto p : lowerStore->queryValidDerivers(path)) + res.insert(p); + return res; +} + + std::optional LocalOverlayStore::queryPathFromHashPart(const std::string & hashPart) { auto res = LocalStore::queryPathFromHashPart(hashPart); diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 88da4ba4a..9bc513b3e 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -89,6 +89,10 @@ private: bool isValidPathUncached(const StorePath & path) override; + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; + + StorePathSet queryValidDerivers(const StorePath & path) override; + std::optional queryPathFromHashPart(const std::string & hashPart) override; void registerValidPaths(const ValidPathInfos & infos) override; diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index d13d8b9ab..7ab587b87 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -55,6 +55,21 @@ expect 1 stat $(toRealPath "$storeBTop" "$path") # Checking for path in overlay store matching lower layer diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-store/nix/store" "$path") +# Checking requisites query agreement +[[ \ + $(nix-store --store $storeA --query --requisites $drvPath) \ + == \ + $(nix-store --store $storeB --query --requisites $drvPath) \ + ]] + +# Checking referrers query agreement +busyboxStore=$(nix store --store $storeA add-path $busybox) +[[ \ + $(nix-store --store $storeA --query --referrers $busyboxStore) \ + == \ + $(nix-store --store $storeB --query --referrers $busyboxStore) \ + ]] + # Checking derivers query agreement [[ \ $(nix-store --store $storeA --query --deriver $path) \ From b5591ece4c4afae6a0ea71396a72eea529b39b79 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 15 May 2023 10:20:16 +0100 Subject: [PATCH 015/211] Check that overlay store directory is mounted correctly. Nix does not manage the overlayfs mount point itself, but the correct functioning of the overlay store does depend on this mount point being set up correctly. Rather than just assume this is the case, check that the lowerdir and upperdir options are what we expect them to be. This check is on by default, but can be disabled if needed. --- src/libstore/local-overlay-store.cc | 27 +++++++++++++++++++++++++++ src/libstore/local-overlay-store.hh | 3 +++ 2 files changed, 30 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index c6b721b48..a40f55fdb 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,5 +1,6 @@ #include "local-overlay-store.hh" #include "callback.hh" +#include namespace nix { @@ -17,6 +18,32 @@ LocalOverlayStore::LocalOverlayStore(const Params & params) , LocalStore(params) , lowerStore(openStore(lowerStoreUri).dynamic_pointer_cast()) { + if (checkMount.get()) { + std::smatch match; + std::string mountInfo; + auto mounts = readFile("/proc/self/mounts"); + auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); + + // Mount points can be stacked, so there might be multiple matching entries. + // Loop until the last match, which will be the current state of the mount point. + while (std::regex_search(mounts, match, regex)) { + mountInfo = match.str(); + mounts = match.suffix(); + } + + auto checkOption = [&](std::string option, std::string value) { + return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); + }; + + auto expectedLowerDir = lowerStore->realStoreDir.get(); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) { + debug("expected lowerdir: %s", expectedLowerDir); + debug("expected upperdir: %s", upperLayer); + debug("actual mount: %s", mountInfo); + throw Error("overlay filesystem '%s' mounted incorrectly", + realStoreDir.get()); + } + } } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 9bc513b3e..8280ed273 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -30,6 +30,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig Must be used as OverlayFS upper layer for this store's store dir. )"}; + Setting checkMount{(StoreConfig*) this, true, "check-mount", + "Check that the overlay filesystem is correctly mounted."}; + const std::string name() override { return "Experimental Local Overlay Store"; } std::string doc() override From b0989cb10b6e1a59070e8c999520c0fbfd25657d Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 15 May 2023 10:28:43 +0100 Subject: [PATCH 016/211] Support percent encoded URIs for lower store. --- src/libstore/local-overlay-store.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index a40f55fdb..ab7af6aea 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,5 +1,6 @@ #include "local-overlay-store.hh" #include "callback.hh" +#include "url.hh" #include namespace nix { @@ -16,7 +17,7 @@ LocalOverlayStore::LocalOverlayStore(const Params & params) , Store(params) , LocalFSStore(params) , LocalStore(params) - , lowerStore(openStore(lowerStoreUri).dynamic_pointer_cast()) + , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast()) { if (checkMount.get()) { std::smatch match; From 0df37edb1c1ae5af43496f009221f2d4fe23a629 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 15 May 2023 10:29:06 +0100 Subject: [PATCH 017/211] Make upper-layer a PathSetting instead of a Setting. --- src/libstore/local-overlay-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 8280ed273..525a54745 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -25,7 +25,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig Must be used as OverlayFS lower layer for this store's store dir. )"}; - const Setting upperLayer{(StoreConfig*) this, "", "upper-layer", + const PathSetting upperLayer{(StoreConfig*) this, false, "", "upper-layer", R"( Must be used as OverlayFS upper layer for this store's store dir. )"}; From b7e5aaf90d50d06e057cceefaa831ea5e4a671bd Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Mon, 15 May 2023 13:40:01 -0500 Subject: [PATCH 018/211] Add test for checking that we reject bad local overlay store uris --- tests/overlay-local-store/inner.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index 7ab587b87..c8d49eb19 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -14,6 +14,16 @@ storeA="$TEST_ROOT/store-a" storeBTop="$TEST_ROOT/store-b" storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" +mkdir -p $TEST_ROOT/bad_test +badTestRoot=$TEST_ROOT/bad_test +storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" +storeBadLower="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" + +declare -a storesBad=( + "$storeBadRoot" "$storeBadLower" "$storeBadUpper" +) + mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} # Mounting Overlay Store @@ -105,6 +115,11 @@ expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") path=$(nix-store --store "$storeB" --add dummy) +for i in "${storesBad[@]}"; do + echo $i + expectStderr 1 nix-store --store "$i" --add dummy | grepQuiet "overlay filesystem .* mounted incorrectly" +done + # lower store should have it from before stat $(toRealPath "$storeA/nix/store" "$path") From 0979a374c555a34f1ee622c3f205b1702a59ccc6 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Mon, 15 May 2023 14:34:57 -0500 Subject: [PATCH 019/211] Begin to split up overlay-local-store tests The bad-uris tests are now in their own file. "Outer" is a bad name, but it will be split up next. --- tests/common.sh | 2 +- tests/local.mk | 3 +- tests/overlay-local-store.sh | 10 ----- tests/overlay-local-store/bad-uris.sh | 25 ++++++++++++ tests/overlay-local-store/common.sh | 52 ++++++++++++++++++++++++ tests/overlay-local-store/inner.sh | 57 +++------------------------ tests/overlay-local-store/outer.sh | 5 +++ 7 files changed, 90 insertions(+), 64 deletions(-) delete mode 100644 tests/overlay-local-store.sh create mode 100644 tests/overlay-local-store/bad-uris.sh create mode 100644 tests/overlay-local-store/common.sh create mode 100755 tests/overlay-local-store/outer.sh diff --git a/tests/common.sh b/tests/common.sh index 8941671d6..7b0922c9f 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -4,7 +4,7 @@ if [[ -z "${COMMON_SH_SOURCED-}" ]]; then COMMON_SH_SOURCED=1 -source "$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/common/vars-and-functions.sh" +source "$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")/common/vars-and-functions.sh" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then startDaemon fi diff --git a/tests/local.mk b/tests/local.mk index 61fb1a85b..ea5550c75 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -133,7 +133,8 @@ nix_tests = \ impure-derivations.sh \ path-from-hash-part.sh \ toString-path.sh \ - overlay-local-store.sh + overlay-local-store/outer.sh \ + overlay-local-store/bad-uris.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/overlay-local-store.sh b/tests/overlay-local-store.sh deleted file mode 100644 index 242c16e20..000000000 --- a/tests/overlay-local-store.sh +++ /dev/null @@ -1,10 +0,0 @@ -source common.sh - -requireSandboxSupport -[[ $busybox =~ busybox ]] || skipTest "no busybox" -if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi -needLocalStore "The test uses --store always so we would just be bypassing the daemon" - -echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf - -exec unshare --mount --map-root-user overlay-local-store/inner.sh diff --git a/tests/overlay-local-store/bad-uris.sh b/tests/overlay-local-store/bad-uris.sh new file mode 100644 index 000000000..d4261bd97 --- /dev/null +++ b/tests/overlay-local-store/bad-uris.sh @@ -0,0 +1,25 @@ +source common.sh + +requireEnvironment +setupConfig +storeDirs + +mkdir -p $TEST_ROOT/bad_test +badTestRoot=$TEST_ROOT/bad_test +storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" +storeBadLower="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" + +declare -a storesBad=( + "$storeBadRoot" "$storeBadLower" "$storeBadUpper" +) + +for i in "${storesBad[@]}"; do + echo $i + unshare --mount --map-root-user bash <> "$NIX_CONF_DIR"/nix.conf + echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf +} + +storeDirs () { + storeA="$TEST_ROOT/store-a" + storeBTop="$TEST_ROOT/store-b" + storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + # Creating testing directories + mkdir -p "$TEST_ROOT"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} +} + +# Mounting Overlay Store +mountOverlayfs () { + mount -t overlay overlay \ + -o lowerdir="$storeA/nix/store" \ + -o upperdir="$storeBTop" \ + -o workdir="$TEST_ROOT/workdir" \ + "$TEST_ROOT/merged-store/nix/store" \ + || skipTest "overlayfs is not supported" + + cleanupOverlay () { + umount "$TEST_ROOT/merged-store/nix/store" + rm -r $TEST_ROOT/workdir + } + trap cleanupOverlay EXIT +} + +toRealPath () { + storeDir=$1; shift + storePath=$1; shift + echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") +} + +initLowerStore () { + # Init lower store with some stuff + nix-store --store "$storeA" --add ../dummy + + # Build something in lower store + drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg busybox "$busybox" --arg seed 1) + path=$(nix-store --store "$storeA" --realise $drvPath) +} diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/inner.sh index c8d49eb19..adbe29557 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/inner.sh @@ -6,53 +6,11 @@ set -x source common.sh -export NIX_CONFIG='build-users-group = ' +storeDirs -# Creating testing directories +initLowerStore -storeA="$TEST_ROOT/store-a" -storeBTop="$TEST_ROOT/store-b" -storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" - -mkdir -p $TEST_ROOT/bad_test -badTestRoot=$TEST_ROOT/bad_test -storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" -storeBadLower="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" -storeBadUpper="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" - -declare -a storesBad=( - "$storeBadRoot" "$storeBadLower" "$storeBadUpper" -) - -mkdir -p "$TEST_ROOT"/{store-a,store-b,merged-store/nix/store,workdir} - -# Mounting Overlay Store - -# Init lower store with some stuff -nix-store --store "$storeA" --add dummy - -# Build something in lower store -drvPath=$(nix-instantiate --store $storeA ./hermetic.nix --arg busybox "$busybox" --arg seed 1) -path=$(nix-store --store "$storeA" --realise $drvPath) - -mount -t overlay overlay \ - -o lowerdir="$storeA/nix/store" \ - -o upperdir="$storeBTop" \ - -o workdir="$TEST_ROOT/workdir" \ - "$TEST_ROOT/merged-store/nix/store" \ - || skipTest "overlayfs is not supported" - -cleanupOverlay () { - umount "$TEST_ROOT/merged-store/nix/store" - rm -r $TEST_ROOT/workdir -} -trap cleanupOverlay EXIT - -toRealPath () { - storeDir=$1; shift - storePath=$1; shift - echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") -} +mountOverlayfs ### Check status @@ -113,12 +71,7 @@ hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') # upper layer should not have it expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") -path=$(nix-store --store "$storeB" --add dummy) - -for i in "${storesBad[@]}"; do - echo $i - expectStderr 1 nix-store --store "$i" --add dummy | grepQuiet "overlay filesystem .* mounted incorrectly" -done +path=$(nix-store --store "$storeB" --add ../dummy) # lower store should have it from before stat $(toRealPath "$storeA/nix/store" "$path") @@ -128,7 +81,7 @@ expect 1 stat $(toRealPath "$storeB/nix/store" "$path") ### Do a build in overlay store -path=$(nix-build ./hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) +path=$(nix-build ../hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) # Checking for path in lower layer (should fail) expect 1 stat $(toRealPath "$storeA/nix/store" "$path") diff --git a/tests/overlay-local-store/outer.sh b/tests/overlay-local-store/outer.sh new file mode 100755 index 000000000..ba55f1d06 --- /dev/null +++ b/tests/overlay-local-store/outer.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +exec unshare --mount --map-root-user ./inner.sh From b1fba1c2a13ca8a9e295ee0a572df207d3591db1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 16:44:36 -0400 Subject: [PATCH 020/211] Fix PS4 for heredocs --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index a9e6c802f..dc7ce13cc 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]}:$LINENO) ' +export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export NIX_STORE_DIR From 97deb00cbc5697d86a04ec7491b5b0da26ee5357 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 18:13:11 -0400 Subject: [PATCH 021/211] Create notion of "test group", use for local overlay store --- mk/lib.mk | 9 +++++++++ mk/tests.mk | 5 +---- tests/local.mk | 10 +++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/mk/lib.mk b/mk/lib.mk index 34fa624d8..a6c2b32e4 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,6 +10,7 @@ bin-scripts := noinst-scripts := man-pages := install-tests := +install-tests-groups := ifdef HOST_OS HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) @@ -122,6 +123,14 @@ $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) $(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +$(foreach test, $(install-tests), $(eval installcheck: $(test).test)) +$(foreach test-group, $(install-tests-groups), \ + $(eval installcheck: $(test-group).test-group) \ + $(eval .PHONY: $(test-group).test-group) \ + $(foreach test, $($(test-group)-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval $(test-group).test-group: $(test).test))) + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/tests.mk b/mk/tests.mk index 3ebbd86e3..dca3c5da2 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -2,10 +2,7 @@ test-deps = -define run-install-test - - installcheck: $1.test - +define run-install-test $1 .PHONY: $1.test $1.test: $1 $(test-deps) @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null diff --git a/tests/local.mk b/tests/local.mk index 778d087b1..2dea58e7b 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -135,9 +135,13 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - toString-path.sh \ - overlay-local-store/outer.sh \ - overlay-local-store/bad-uris.sh + toString-path.sh + +overlay-local-store-tests := \ + $(d)/overlay-local-store/outer.sh \ + $(d)/overlay-local-store/bad-uris.sh + +install-tests-groups += overlay-local-store ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh From 5d18120ba837039718b3d08d627c7c231f379a40 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 23:00:18 -0400 Subject: [PATCH 022/211] Split tests some more Good for parallelism and easier reading. --- tests/local.mk | 4 ++- tests/overlay-local-store/build-inner.sh | 26 +++++++++++++++++++ .../{outer.sh => build.sh} | 2 +- .../{inner.sh => check-post-init-inner.sh} | 26 ------------------- tests/overlay-local-store/check-post-init.sh | 5 ++++ tests/overlay-local-store/common.sh | 26 +++++++++++-------- .../redundant-add-inner.sh | 26 +++++++++++++++++++ tests/overlay-local-store/redundant-add.sh | 5 ++++ 8 files changed, 81 insertions(+), 39 deletions(-) create mode 100755 tests/overlay-local-store/build-inner.sh rename tests/overlay-local-store/{outer.sh => build.sh} (50%) rename tests/overlay-local-store/{inner.sh => check-post-init-inner.sh} (68%) create mode 100755 tests/overlay-local-store/check-post-init.sh create mode 100755 tests/overlay-local-store/redundant-add-inner.sh create mode 100755 tests/overlay-local-store/redundant-add.sh diff --git a/tests/local.mk b/tests/local.mk index 2dea58e7b..dd168ce3f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -138,7 +138,9 @@ nix_tests = \ toString-path.sh overlay-local-store-tests := \ - $(d)/overlay-local-store/outer.sh \ + $(d)/overlay-local-store/check-post-init.sh \ + $(d)/overlay-local-store/redundant-add.sh \ + $(d)/overlay-local-store/build.sh \ $(d)/overlay-local-store/bad-uris.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/build-inner.sh b/tests/overlay-local-store/build-inner.sh new file mode 100755 index 000000000..969de282d --- /dev/null +++ b/tests/overlay-local-store/build-inner.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +storeDirs + +initLowerStore + +mountOverlayfs + +### Do a build in overlay store + +path=$(nix-build ../hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) + +# Checking for path in lower layer (should fail) +expect 1 stat $(toRealPath "$storeA/nix/store" "$path") + +# Checking for path in upper layer +stat $(toRealPath "$storeBTop" "$path") + +# Verifying path in overlay store +nix-store --verify-path --store "$storeB" "$path" diff --git a/tests/overlay-local-store/outer.sh b/tests/overlay-local-store/build.sh similarity index 50% rename from tests/overlay-local-store/outer.sh rename to tests/overlay-local-store/build.sh index ba55f1d06..758585400 100755 --- a/tests/overlay-local-store/outer.sh +++ b/tests/overlay-local-store/build.sh @@ -2,4 +2,4 @@ source common.sh requireEnvironment setupConfig -exec unshare --mount --map-root-user ./inner.sh +execUnshare ./build-inner.sh diff --git a/tests/overlay-local-store/inner.sh b/tests/overlay-local-store/check-post-init-inner.sh similarity index 68% rename from tests/overlay-local-store/inner.sh rename to tests/overlay-local-store/check-post-init-inner.sh index adbe29557..421b64934 100755 --- a/tests/overlay-local-store/inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -65,29 +65,3 @@ hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') # merged store can find from hash part [[ $(nix store --store $storeB path-from-hash-part $hashPart) == $path ]] - -### Do a redundant add - -# upper layer should not have it -expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") - -path=$(nix-store --store "$storeB" --add ../dummy) - -# lower store should have it from before -stat $(toRealPath "$storeA/nix/store" "$path") - -# upper layer should still not have it (no redundant copy) -expect 1 stat $(toRealPath "$storeB/nix/store" "$path") - -### Do a build in overlay store - -path=$(nix-build ../hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link) - -# Checking for path in lower layer (should fail) -expect 1 stat $(toRealPath "$storeA/nix/store" "$path") - -# Checking for path in upper layer -stat $(toRealPath "$storeBTop" "$path") - -# Verifying path in overlay store -nix-store --verify-path --store "$storeB" "$path" diff --git a/tests/overlay-local-store/check-post-init.sh b/tests/overlay-local-store/check-post-init.sh new file mode 100755 index 000000000..985bf978e --- /dev/null +++ b/tests/overlay-local-store/check-post-init.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./check-post-init-inner.sh diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 6d9b69b4c..149102000 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -1,23 +1,23 @@ source ../common.sh requireEnvironment () { - requireSandboxSupport - [[ $busybox =~ busybox ]] || skipTest "no busybox" - if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi - needLocalStore "The test uses --store always so we would just be bypassing the daemon" + requireSandboxSupport + [[ $busybox =~ busybox ]] || skipTest "no busybox" + if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi + needLocalStore "The test uses --store always so we would just be bypassing the daemon" } setupConfig () { - echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf - echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf + echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf + echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf } storeDirs () { - storeA="$TEST_ROOT/store-a" - storeBTop="$TEST_ROOT/store-b" - storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" - # Creating testing directories - mkdir -p "$TEST_ROOT"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} + storeA="$TEST_ROOT/store-a" + storeBTop="$TEST_ROOT/store-b" + storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + # Creating testing directories + mkdir -p "$TEST_ROOT"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } # Mounting Overlay Store @@ -50,3 +50,7 @@ initLowerStore () { drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg busybox "$busybox" --arg seed 1) path=$(nix-store --store "$storeA" --realise $drvPath) } + +execUnshare () { + exec unshare --mount --map-root-user "$@" +} diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/overlay-local-store/redundant-add-inner.sh new file mode 100755 index 000000000..921069c25 --- /dev/null +++ b/tests/overlay-local-store/redundant-add-inner.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +storeDirs + +initLowerStore + +mountOverlayfs + +### Do a redundant add + +# upper layer should not have it +expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") + +path=$(nix-store --store "$storeB" --add ../dummy) + +# lower store should have it from before +stat $(toRealPath "$storeA/nix/store" "$path") + +# upper layer should still not have it (no redundant copy) +expect 1 stat $(toRealPath "$storeB/nix/store" "$path") diff --git a/tests/overlay-local-store/redundant-add.sh b/tests/overlay-local-store/redundant-add.sh new file mode 100755 index 000000000..fbd4799e7 --- /dev/null +++ b/tests/overlay-local-store/redundant-add.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./redundant-add-inner.sh From 0ec7f2fb3f935f39585a6f1daa21a7520e1a4b40 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 23:10:53 -0400 Subject: [PATCH 023/211] Create `local.mk` for local-overlay-store tests --- Makefile | 1 + tests/local.mk | 8 -------- tests/overlay-local-store/local.mk | 7 +++++++ 3 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 tests/overlay-local-store/local.mk diff --git a/Makefile b/Makefile index d6b49473a..3ce5cc5e3 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ makefiles += \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ tests/local.mk \ + tests/overlay-local-store/local.mk \ tests/plugins/local.mk else makefiles += \ diff --git a/tests/local.mk b/tests/local.mk index dd168ce3f..9cb81e1f0 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -137,14 +137,6 @@ nix_tests = \ path-from-hash-part.sh \ toString-path.sh -overlay-local-store-tests := \ - $(d)/overlay-local-store/check-post-init.sh \ - $(d)/overlay-local-store/redundant-add.sh \ - $(d)/overlay-local-store/build.sh \ - $(d)/overlay-local-store/bad-uris.sh - -install-tests-groups += overlay-local-store - ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk new file mode 100644 index 000000000..b94238a67 --- /dev/null +++ b/tests/overlay-local-store/local.mk @@ -0,0 +1,7 @@ +overlay-local-store-tests := \ + $(d)/check-post-init.sh \ + $(d)/redundant-add.sh \ + $(d)/build.sh \ + $(d)/bad-uris.sh + +install-tests-groups += overlay-local-store From 4d69bd034ae3435b20d1b5d38a819247a3031c25 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:10:25 +0100 Subject: [PATCH 024/211] More detailed explanation of check-mount setting. --- src/libstore/local-overlay-store.hh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 525a54745..6f798c460 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -31,7 +31,15 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig )"}; Setting checkMount{(StoreConfig*) this, true, "check-mount", - "Check that the overlay filesystem is correctly mounted."}; + R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )"}; const std::string name() override { return "Experimental Local Overlay Store"; } From de359da09a0a3a0b68cbcb20c25adc3908046296 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:14:58 +0100 Subject: [PATCH 025/211] Add read-only setting to LocalStoreConfig. --- src/libstore/local-store.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 55add18dd..1557c6242 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -46,6 +46,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig "require-sigs", "Whether store paths copied into this store should have a trusted signature."}; + Setting readOnly{(StoreConfig*) this, + false, + "read-only", + "TODO"}; + const std::string name() override { return "Local Store"; } std::string doc() override; From 79583c2d38b75d227886679bca35a5b38a6e2119 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:29:16 +0100 Subject: [PATCH 026/211] Do not attempt to chmod per-user dir when read-only. --- src/libstore/local-store.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7fb312c37..d219f3e95 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -204,8 +204,10 @@ LocalStore::LocalStore(const Params & params) for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); + if (!readOnly) { + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } } /* Optionally, create directories and set permissions for a From 50bbdc65c8adccaf2d34fd90ee9f3897c347c667 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:53:31 +0100 Subject: [PATCH 027/211] Do not attempt to acquire big-lock when read-only. --- src/libstore/local-store.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d219f3e95..94176c62f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -271,10 +271,12 @@ LocalStore::LocalStore(const Params & params) /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - Path globalLockPath = dbDir + "/big-lock"; - globalLock = openLockFile(globalLockPath.c_str(), true); + if (!readOnly) { + Path globalLockPath = dbDir + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + } - if (!lockFile(globalLock.get(), ltRead, false)) { + if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -305,7 +307,7 @@ LocalStore::LocalStore(const Params & params) "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 1.11 first."); - if (!lockFile(globalLock.get(), ltWrite, false)) { + if (!readOnly && !lockFile(globalLock.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store..."); lockFile(globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(globalLock.get(), ltWrite, true); @@ -340,7 +342,8 @@ LocalStore::LocalStore(const Params & params) writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); - lockFile(globalLock.get(), ltRead, true); + if (!readOnly) + lockFile(globalLock.get(), ltRead, true); } else openDB(*state, false); From c22936ca6ad68630f2982a62245b91b11c4fb699 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:56:29 +0100 Subject: [PATCH 028/211] Do not attempt to migrate to CA schema when read-only. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 94176c62f..5d9e96441 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -348,7 +348,7 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + if (!readOnly && experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } From 7f443e04283b027dff8589eb54473313be877577 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:57:13 +0100 Subject: [PATCH 029/211] Do not check for write access to database when read-only. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5d9e96441..b0f6709aa 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -480,7 +480,7 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (access(dbDir.c_str(), R_OK | W_OK)) + if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ From afed9ccfadc24a6a58da781cfe7eb5d1a4e4f7c3 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 13:57:56 +0100 Subject: [PATCH 030/211] Add enum for intended sqlite database open modes. --- src/libstore/sqlite.hh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 6e14852cb..8f4bc49a8 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -11,6 +11,24 @@ struct sqlite3_stmt; namespace nix { +enum class SQLiteOpenMode { + /** + * Open the database in read-write mode. + * If the database does not exist, it will be created. + */ + Normal, + /** + * Open the database in read-write mode. + * Fails with an error if the database does not exist. + */ + NoCreate, + /** + * Open the database in read-only mode. + * Fails with an error if the database does not exist. + */ + ReadOnly +}; + /** * RAII wrapper to close a SQLite database automatically. */ From 78fdd6f24e79f50f7d27a2576685398e67928f61 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 14:13:03 +0100 Subject: [PATCH 031/211] Open sqlite database according to new modes. --- src/libstore/local-store.cc | 9 ++++++++- src/libstore/sqlite.cc | 7 ++++--- src/libstore/sqlite.hh | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b0f6709aa..1c5d8fc9e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -480,13 +480,20 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { + if (create && readOnly) { + throw Error("unable to create database while in read-only mode"); + } + if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - state.db = SQLite(dbPath, create); + auto openMode = readOnly ? SQLiteOpenMode::ReadOnly + : create ? SQLiteOpenMode::Normal + : SQLiteOpenMode::NoCreate; + state.db = SQLite(dbPath, openMode); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index df334c23c..8159744b7 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -50,14 +50,15 @@ static void traceSQL(void * x, const char * sql) notice("SQL<[%1%]>", sql); }; -SQLite::SQLite(const Path & path, bool create) +SQLite::SQLite(const Path & path, SQLiteOpenMode mode) { // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - int flags = SQLITE_OPEN_READWRITE; - if (create) flags |= SQLITE_OPEN_CREATE; + int flags = mode == SQLiteOpenMode::ReadOnly + ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs); if (ret != SQLITE_OK) { const char * err = sqlite3_errstr(ret); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 8f4bc49a8..33f0eeed8 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -36,7 +36,7 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path, bool create = true); + SQLite(const Path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } From aa376f4ab17142516b19de6aa0b990b6d097b2f2 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 15:29:29 +0100 Subject: [PATCH 032/211] Need to open database using immutable parameter. This requires switching on SQLITE_OPEN_URI because there is no open flag to make the database immutable. Without immutable, sqlite will still attempt to create journal and wal files, even when the database is opened read-only. https://www.sqlite.org/c3ref/open.html The immutable parameter is a boolean query parameter that indicates that the database file is stored on read-only media. When immutable is set, SQLite assumes that the database file cannot be changed, even by a process with higher privilege, and so the database is opened read-only and all locking and change detection is disabled. --- src/libstore/sqlite.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 8159744b7..4166cc2cd 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,6 +1,7 @@ #include "sqlite.hh" #include "globals.hh" #include "util.hh" +#include "url.hh" #include @@ -52,14 +53,16 @@ static void traceSQL(void * x, const char * sql) SQLite::SQLite(const Path & path, SQLiteOpenMode mode) { + bool readOnly = mode == SQLiteOpenMode::ReadOnly; + // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - int flags = mode == SQLiteOpenMode::ReadOnly - ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + int flags = readOnly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; - int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs); + auto uri = "file:" + percentEncode(path) + "?immutable=" + (readOnly ? "1" : "0"); + int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs); if (ret != SQLITE_OK) { const char * err = sqlite3_errstr(ret); throw Error("cannot open SQLite database '%s': %s", path, err); From b1a7b26eef46106deec0a954d400e5a1aac47945 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 16 May 2023 15:48:40 +0100 Subject: [PATCH 033/211] Rename ReadOnly to Immutable and clarify its purpose. --- src/libstore/local-store.cc | 2 +- src/libstore/sqlite.cc | 7 +++---- src/libstore/sqlite.hh | 7 +++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1c5d8fc9e..59685400f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -490,7 +490,7 @@ void LocalStore::openDB(State & state, bool create) /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - auto openMode = readOnly ? SQLiteOpenMode::ReadOnly + auto openMode = readOnly ? SQLiteOpenMode::Immutable : create ? SQLiteOpenMode::Normal : SQLiteOpenMode::NoCreate; state.db = SQLite(dbPath, openMode); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 4166cc2cd..7c8decb74 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -53,15 +53,14 @@ static void traceSQL(void * x, const char * sql) SQLite::SQLite(const Path & path, SQLiteOpenMode mode) { - bool readOnly = mode == SQLiteOpenMode::ReadOnly; - // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - int flags = readOnly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + bool immutable = mode == SQLiteOpenMode::Immutable; + int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; - auto uri = "file:" + percentEncode(path) + "?immutable=" + (readOnly ? "1" : "0"); + auto uri = "file:" + percentEncode(path) + "?immutable=" + (immutable ? "1" : "0"); int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs); if (ret != SQLITE_OK) { const char * err = sqlite3_errstr(ret); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 33f0eeed8..b0e84f8ed 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -23,10 +23,13 @@ enum class SQLiteOpenMode { */ NoCreate, /** - * Open the database in read-only mode. + * Open the database in immutable mode. + * In addition to the database being read-only, + * no wal or journal files will be created by sqlite. + * Use this mode if the database is on a read-only filesystem. * Fails with an error if the database does not exist. */ - ReadOnly + Immutable }; /** From 5966b76c974a8f2eddac8c610b754a2dbcf8bb9b Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 17 May 2023 08:49:51 +0100 Subject: [PATCH 034/211] Document the new read-only local store setting. --- src/libstore/local-store.hh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1557c6242..7fcf645e7 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -49,7 +49,18 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig Setting readOnly{(StoreConfig*) this, false, "read-only", - "TODO"}; + R"( + Allow this store to be opened when its database is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even + for querying (when write access is not needed). This causes it to fail if the + database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the + **imutable** parameter set. Do not use this unless the filesystem is read-only. + Using it when the filesystem is writable can cause incorrect query results or + corruption errors if the database is changed by another process. + )"}; const std::string name() override { return "Local Store"; } From 85a24530524e5a7410fd00714a4a8f4633b4f758 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Wed, 17 May 2023 13:48:59 -0500 Subject: [PATCH 035/211] Add tests for read-only local store Make sure we don't go down the path of making temproots when doing operations on a read-only store --- src/libstore/gc.cc | 4 ++++ tests/local.mk | 3 ++- tests/read-only-store.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/read-only-store.sh diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0038ec802..09f6ddb9e 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -110,6 +110,10 @@ void LocalStore::createTempRootsFile() void LocalStore::addTempRoot(const StorePath & path) { + if (readOnly) { + debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); + return; + } createTempRootsFile(); /* Open/create the global GC lock file. */ diff --git a/tests/local.mk b/tests/local.mk index 9e340e2e2..576529567 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -134,7 +134,8 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - toString-path.sh + toString-path.sh \ + read-only-store.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh new file mode 100644 index 000000000..20ded9634 --- /dev/null +++ b/tests/read-only-store.sh @@ -0,0 +1,29 @@ +source common.sh + +clearStore + +## Testing read-only mode without forcing the underlying store to actually be read-only + +# Make sure the command fails when the store doesn't already have a database +expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "unable to create database while in read-only mode" + +# Make sure the store actually has a current-database +nix-store --add dummy + +# Try again and make sure we fail when adding a item not already in the store +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "attempt to write a readonly database" + +# Make sure we can get an already-present store-path in the database +nix-store --store local?read-only=true --add dummy + +## Ensure store is actually read-only +chmod -R -w $TEST_ROOT/store +chmod -R -w $TEST_ROOT/var + +# Make sure we fail on add operations on the read-only store +# This is only for adding files that are not *already* in the store +expectStderr 1 nix-store --add eval.nix | grepQuiet "error: opening lock file '$(readlink -e $TEST_ROOT)/var/nix/db/big-lock'" +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "Permission denied" + +# Should succeed +nix-store --store local?read-only=true --add dummy From 9290af763a7eb93e1967d540c7505c4514b2c546 Mon Sep 17 00:00:00 2001 From: Dylan Green <67574902+cidkidnix@users.noreply.github.com> Date: Wed, 17 May 2023 14:52:13 -0500 Subject: [PATCH 036/211] Update tests/read-only-store.sh Co-authored-by: John Ericson --- tests/read-only-store.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index 20ded9634..b440fe356 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -16,7 +16,9 @@ expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet # Make sure we can get an already-present store-path in the database nix-store --store local?read-only=true --add dummy -## Ensure store is actually read-only +## Testing read-only mode with an underlying store that is actually read-only + +# Ensure store is actually read-only chmod -R -w $TEST_ROOT/store chmod -R -w $TEST_ROOT/var From 60c014972171edf271b173ab03e13866c2839035 Mon Sep 17 00:00:00 2001 From: Dylan Green <67574902+cidkidnix@users.noreply.github.com> Date: Wed, 17 May 2023 14:56:48 -0500 Subject: [PATCH 037/211] Apply suggestions from code review Co-authored-by: John Ericson --- src/libstore/gc.cc | 1 + tests/read-only-store.sh | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 09f6ddb9e..3c9544017 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -114,6 +114,7 @@ void LocalStore::addTempRoot(const StorePath & path) debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); return; } + createTempRootsFile(); /* Open/create the global GC lock file. */ diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index b440fe356..72d855d42 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -2,19 +2,26 @@ source common.sh clearStore +happy () { + # We can do a read-only query just fine with a read-only store + nix --store local?read-only=true path-info $dummyPath + + # We can "write" an already-present store-path a read-only store, because no IO is actually required + nix-store --store local?read-only=true --add dummy +} ## Testing read-only mode without forcing the underlying store to actually be read-only # Make sure the command fails when the store doesn't already have a database expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "unable to create database while in read-only mode" -# Make sure the store actually has a current-database -nix-store --add dummy +# Make sure the store actually has a current-database, with at least one store object +dummyPath=$(nix-store --add dummy) # Try again and make sure we fail when adding a item not already in the store expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "attempt to write a readonly database" -# Make sure we can get an already-present store-path in the database -nix-store --store local?read-only=true --add dummy +# Test a few operations that should work with the read-only store in its current state +happy ## Testing read-only mode with an underlying store that is actually read-only @@ -27,5 +34,5 @@ chmod -R -w $TEST_ROOT/var expectStderr 1 nix-store --add eval.nix | grepQuiet "error: opening lock file '$(readlink -e $TEST_ROOT)/var/nix/db/big-lock'" expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "Permission denied" -# Should succeed -nix-store --store local?read-only=true --add dummy +# Test the same operations from before should again succeed +happy From fe174d72a2b8ef7a4a6415bf284b1e282e2052cf Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 18 May 2023 13:44:39 +0100 Subject: [PATCH 038/211] Fix spelling of 'immutable' in documentation. --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7fcf645e7..37e31210f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -57,7 +57,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig database is on a read-only filesystem. Enable read-only mode to disable locking and open the SQLite database with the - **imutable** parameter set. Do not use this unless the filesystem is read-only. + **immutable** parameter set. Do not use this unless the filesystem is read-only. Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. )"}; From d55e38b98a281b6915cd4b57b597059d4be9b0ac Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 18 May 2023 13:45:31 +0100 Subject: [PATCH 039/211] Check earlier whether schema migration is required. --- src/libstore/local-store.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 59685400f..99fb9f434 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -284,6 +284,14 @@ LocalStore::LocalStore(const Params & params) /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); + if (readOnly && curSchema < nixSchemaVersion) { + debug("current schema version: %d", curSchema); + debug("supported schema version: %d", nixSchemaVersion); + throw Error(curSchema == 0 ? + "database does not exist, and cannot be created in read-only mode" : + "database schema needs migrating, but this cannot be done in read-only mode"); + } + if (curSchema > nixSchemaVersion) throw Error("current Nix store schema is version %1%, but I only support %2%", curSchema, nixSchemaVersion); @@ -307,7 +315,7 @@ LocalStore::LocalStore(const Params & params) "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 1.11 first."); - if (!readOnly && !lockFile(globalLock.get(), ltWrite, false)) { + if (!lockFile(globalLock.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store..."); lockFile(globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(globalLock.get(), ltWrite, true); @@ -342,8 +350,7 @@ LocalStore::LocalStore(const Params & params) writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); - if (!readOnly) - lockFile(globalLock.get(), ltRead, true); + lockFile(globalLock.get(), ltRead, true); } else openDB(*state, false); From 8ffeb1c4e58e8ed0032b097d3d1d9625726f5fec Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 18 May 2023 13:47:31 +0100 Subject: [PATCH 040/211] Throw error instead of silently skipping CA migration. --- src/libstore/local-store.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 99fb9f434..c25aa0e72 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -355,8 +355,12 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); - if (!readOnly && experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + if (!readOnly) { + migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + } else { + throw Error("need to migrate to CA schema, but this cannot be done in read-only mode"); + } } /* Prepare SQL statements. */ From 0c36fe6c8cb4af5558c3a6a75240ef3b01510084 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 18 May 2023 14:36:24 +0100 Subject: [PATCH 041/211] Update test to match new error message. --- tests/read-only-store.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index 72d855d42..408ffa63f 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -12,7 +12,7 @@ happy () { ## Testing read-only mode without forcing the underlying store to actually be read-only # Make sure the command fails when the store doesn't already have a database -expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "unable to create database while in read-only mode" +expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "database does not exist, and cannot be created in read-only mode" # Make sure the store actually has a current-database, with at least one store object dummyPath=$(nix-store --add dummy) From 72518000860b56f5cd08a3b3a4fdfc61c021e2e2 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 22 May 2023 11:38:37 +0100 Subject: [PATCH 042/211] Put read-only setting behind an experimental flag. --- src/libstore/local-store.cc | 4 ++++ src/libutil/experimental-features.cc | 18 +++++++++++++++++- src/libutil/experimental-features.hh | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c25aa0e72..4188d413f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -202,6 +202,10 @@ LocalStore::LocalStore(const Params & params) createSymlink(profilesDir, gcRootsDir + "/profiles"); } + if (readOnly) { + experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); + } + for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); if (!readOnly) { diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ad0ec0427..29f5b84db 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -209,6 +209,22 @@ constexpr std::array xpFeatureDetails = {{ files. )", }, + { + .tag = Xp::ReadOnlyLocalStore, + .name = "read-only-local-store", + .description = R"( + Allow the use of the `read-only` parameter in local store URIs. + + Set this parameter to `true` to allow stores with databases on read-only + filesystems to be opened for querying; ordinarily Nix will refuse to do this. + + Enabling this setting disables the locking required for safe concurrent + access, so you should be certain that the database will not be changed. + While the filesystem the database resides on might be read-only to this + process, consider whether another user, process, or system, might have + write access to it. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 409100592..19d425b6d 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -30,6 +30,7 @@ enum struct ExperimentalFeature DiscardReferences, DaemonTrustOverride, DynamicDerivations, + ReadOnlyLocalStore, }; /** From d6ea3b6a19b7d48a7fb4cc03b0eaf080217fa84f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 22 May 2023 12:14:07 +0100 Subject: [PATCH 043/211] Need to enable read-only-local-store flag for test. --- tests/read-only-store.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index 408ffa63f..2f89b849f 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -1,5 +1,7 @@ source common.sh +enableFeatures "read-only-local-store" + clearStore happy () { From c47f744e05d8c80bd65083b0fc0fbfef2ff0c08f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 1 Jun 2023 14:14:13 +0100 Subject: [PATCH 044/211] Also skip makeStoreWritable when read-only=true. --- src/libstore/local-store.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4188d413f..17385cdfb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -190,7 +190,11 @@ LocalStore::LocalStore(const Params & params) /* Create missing state directories if they don't already exist. */ createDirs(realStoreDir); - makeStoreWritable(); + if (readOnly) { + experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); + } else { + makeStoreWritable(); + } createDirs(linksDir); Path profilesDir = stateDir + "/profiles"; createDirs(profilesDir); @@ -202,10 +206,6 @@ LocalStore::LocalStore(const Params & params) createSymlink(profilesDir, gcRootsDir + "/profiles"); } - if (readOnly) { - experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); - } - for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); if (!readOnly) { From 98edbb968617b2478f52ab527bc4b1a75933b8c6 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 24 May 2023 11:24:07 +0100 Subject: [PATCH 045/211] Factor out GC path deletion so it can be overridden. --- src/libstore/gc-store.hh | 7 +++++++ src/libstore/gc.cc | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 2c26c65c4..da1551056 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -97,6 +97,13 @@ struct GcStore : public virtual Store * Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + + /** + * Called by `collectGarbage` to recursively delete a path. + * The default implementation simply calls `deletePath`, but it can be + * overridden by stores that wish to provide their own deletion behaviour. + */ + virtual void deleteGCPath(const Path & path, uint64_t & bytesFreed); }; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3c9544017..959f4d3b1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -654,7 +654,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) results.paths.insert(path); uint64_t bytesFreed; - deletePath(realPath, bytesFreed); + deleteGCPath(realPath, bytesFreed); + results.bytesFreed += bytesFreed; if (results.bytesFreed > options.maxFreed) { @@ -872,7 +873,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (unlink(path.c_str()) == -1) throw SysError("deleting '%1%'", path); - /* Do not accound for deleted file here. Rely on deletePath() + /* Do not account for deleted file here. Rely on deletePath() accounting. */ } @@ -890,6 +891,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } +void GcStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +{ + deletePath(path, bytesFreed); +} + + void LocalStore::autoGC(bool sync) { static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); From a48acfd68424960c9a7a5d4eeb1af2a7ea91835d Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 24 May 2023 11:26:33 +0100 Subject: [PATCH 046/211] Skip deletion of lower paths for overlay store GC. --- src/libstore/local-overlay-store.cc | 14 +++++++++++++- src/libstore/local-overlay-store.hh | 2 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ab7af6aea..f42b37324 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -182,7 +182,7 @@ void LocalOverlayStore::addToStore(const ValidPathInfo & info, Source & source, LocalStore::addToStore(info, source, repair, checkSigs); if (lowerStore->isValidPath(info.path)) { // dedup stores - deletePath(toUpperPath(info.path)); + deletePath(toUpperPath(info.path)); // TODO: Investigate whether this can trigger 'stale file handle' errors. } } @@ -214,6 +214,18 @@ StorePath LocalOverlayStore::addTextToStore( } +void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +{ + auto mergedDir = realStoreDir.get() + "/"; + if (path.substr(0, mergedDir.length()) != mergedDir) { + warn("local-overlay: unexpected gc path '%s' ", path); + return; + } + if (pathExists(toUpperPath({path.substr(mergedDir.length())}))) { + GcStore::deleteGCPath(path, bytesFreed); + } +} + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 6f798c460..701024bfb 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -122,6 +122,8 @@ private: void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; + + void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; }; } From 8a9baa0a3091bd2ebd8d6027e0130c8b6dec76c5 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 6 Jun 2023 12:06:32 +0100 Subject: [PATCH 047/211] More sensible to have deleteGCPath in LocalStore. --- src/libstore/gc-store.hh | 7 ------- src/libstore/gc.cc | 6 ------ src/libstore/local-overlay-store.cc | 2 +- src/libstore/local-store.cc | 6 ++++++ src/libstore/local-store.hh | 7 +++++++ 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index da1551056..2c26c65c4 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -97,13 +97,6 @@ struct GcStore : public virtual Store * Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; - - /** - * Called by `collectGarbage` to recursively delete a path. - * The default implementation simply calls `deletePath`, but it can be - * overridden by stores that wish to provide their own deletion behaviour. - */ - virtual void deleteGCPath(const Path & path, uint64_t & bytesFreed); }; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 959f4d3b1..80875efed 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -891,12 +891,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } -void GcStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) -{ - deletePath(path, bytesFreed); -} - - void LocalStore::autoGC(bool sync) { static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index f42b37324..061478c52 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -222,7 +222,7 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) return; } if (pathExists(toUpperPath({path.substr(mergedDir.length())}))) { - GcStore::deleteGCPath(path, bytesFreed); + LocalStore::deleteGCPath(path, bytesFreed); } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17385cdfb..172fa1ef0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -457,6 +457,12 @@ AutoCloseFD LocalStore::openGCLock() } +void LocalStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +{ + deletePath(path, bytesFreed); +} + + LocalStore::~LocalStore() { std::shared_future future; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 46a660c68..fcddffa82 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -222,6 +222,13 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; + /** + * Called by `collectGarbage` to recursively delete a path. + * The default implementation simply calls `deletePath`, but it can be + * overridden by stores that wish to provide their own deletion behaviour. + */ + virtual void deleteGCPath(const Path & path, uint64_t & bytesFreed); + /** * Optimise the disk space usage of the Nix store by hard-linking * files with the same contents. From ee1241da8649fb310f8b5ac28d27f3c08117cdf9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 6 Jun 2023 12:12:25 +0100 Subject: [PATCH 048/211] Remove unnecessary overrides of add methods. --- src/libstore/local-overlay-store.cc | 38 ----------------------------- src/libstore/local-overlay-store.hh | 12 --------- 2 files changed, 50 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 061478c52..ddccb8c5a 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -176,44 +176,6 @@ void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) } -void LocalOverlayStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) -{ - LocalStore::addToStore(info, source, repair, checkSigs); - if (lowerStore->isValidPath(info.path)) { - // dedup stores - deletePath(toUpperPath(info.path)); // TODO: Investigate whether this can trigger 'stale file handle' errors. - } -} - - -StorePath LocalOverlayStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) -{ - auto path = LocalStore::addToStoreFromDump(dump, name, method, hashAlgo, repair, references); - if (lowerStore->isValidPath(path)) { - // dedup stores - deletePath(toUpperPath(path)); - } - return path; -} - - -StorePath LocalOverlayStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) -{ - auto path = LocalStore::addTextToStore(name, s, references, repair); - if (lowerStore->isValidPath(path)) { - // dedup stores - deletePath(toUpperPath(path)); - } - return path; -} - - void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) { auto mergedDir = realStoreDir.get() + "/"; diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 701024bfb..e6b877b04 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -108,18 +108,6 @@ private: void registerValidPaths(const ValidPathInfos & infos) override; - void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) override; - - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; - - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override; - void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; From 7ed0ab2dabececd84578ddf70ecad0e528d67a28 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 12:48:06 +0100 Subject: [PATCH 049/211] Check _NIX_TEST_NO_SANDBOX when setting _canUseSandbox. --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index dc7ce13cc..ad0623871 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -141,7 +141,7 @@ restartDaemon() { startDaemon } -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then +if [[ -z "${_NIX_TEST_NO_SANDBOX:-}" ]] && [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then _canUseSandbox=1 fi From fad0dd4afb66577f9c17b5a315d120ac0f5acd94 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 12:58:59 +0100 Subject: [PATCH 050/211] Skip build-remote-trustless unless sandbox is supported. --- tests/build-remote-trustless-should-fail-0.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh index fad1def59..b14101f83 100644 --- a/tests/build-remote-trustless-should-fail-0.sh +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -2,6 +2,7 @@ source common.sh enableFeatures "daemon-trust-override" +requireSandboxSupport restartDaemon [[ $busybox =~ busybox ]] || skipTest "no busybox" From 264b644c534418f3a27b7e6013afa4aee7bdf89c Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 13:22:17 +0100 Subject: [PATCH 051/211] More detail on why read-only mode disables locking. --- src/libutil/experimental-features.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b758a0262..1fbc99b34 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -223,11 +223,14 @@ constexpr std::array xpFeatureDetails = {{ Set this parameter to `true` to allow stores with databases on read-only filesystems to be opened for querying; ordinarily Nix will refuse to do this. - Enabling this setting disables the locking required for safe concurrent - access, so you should be certain that the database will not be changed. - While the filesystem the database resides on might be read-only to this - process, consider whether another user, process, or system, might have - write access to it. + This is because SQLite requires write access to the database file to perform + the file locking operations necessary for safe concurrent access. When `read-only` + is set to `true`, the database will be opened in immutable mode. + + Under this mode, SQLite will not do any locking at all, so you should be certain + that the database will not be changed. While the filesystem the database resides + on might be read-only to this process, consider whether another user, process, + or system, might have write access to it. )", }, }}; From 7cdaa0b8a626004b19b87b1eec74d1adb5f20263 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:25:15 +0100 Subject: [PATCH 052/211] Update tests/read-only-store.sh Co-authored-by: Valentin Gagarin --- tests/read-only-store.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh index 2f89b849f..d63920c19 100644 --- a/tests/read-only-store.sh +++ b/tests/read-only-store.sh @@ -2,6 +2,8 @@ source common.sh enableFeatures "read-only-local-store" +needLocalStore "cannot open store read-only when daemon has already opened it writeable" + clearStore happy () { From 78e2f931d048f9c41e5432cd8e49bf6bc0c3baae Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:32:16 +0100 Subject: [PATCH 053/211] Update src/libstore/local-store.cc Co-authored-by: Valentin Gagarin --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17385cdfb..364be5dd5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -363,7 +363,7 @@ LocalStore::LocalStore(const Params & params) if (!readOnly) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } else { - throw Error("need to migrate to CA schema, but this cannot be done in read-only mode"); + throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode"); } } From 984b01924a58dc80b0e5da7722ccfa108cbca036 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:32:35 +0100 Subject: [PATCH 054/211] Update src/libstore/local-store.cc Co-authored-by: Valentin Gagarin --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 364be5dd5..e69460e6c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -496,7 +496,7 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { if (create && readOnly) { - throw Error("unable to create database while in read-only mode"); + throw Error("cannot create database while in read-only mode"); } if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) From a7b1b92d817348669fc2cd34c085dae57ceed5b8 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:32:56 +0100 Subject: [PATCH 055/211] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 31a6f4fd9..35c4febef 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -50,7 +50,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig false, "read-only", R"( - Allow this store to be opened when its database is on a read-only filesystem. + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed). This causes it to fail if the From 4642b60afe2a79b00ae51ca1ec1fc35eb5a224c5 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Thu, 15 Jun 2023 13:33:26 +0100 Subject: [PATCH 056/211] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 35c4febef..a1ab3bb4d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -57,7 +57,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig database is on a read-only filesystem. Enable read-only mode to disable locking and open the SQLite database with the - **immutable** parameter set. Do not use this unless the filesystem is read-only. + [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. Do not use this unless the filesystem is read-only. Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. )"}; From f2fe9822c14bc86e5da8f8cb97cb16a961ecc46f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 13:31:37 +0100 Subject: [PATCH 057/211] Comment explaining what schema version 0 means. --- src/libstore/local-store.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a1ab3bb4d..dd563348b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -285,6 +285,10 @@ public: private: + /** + * Retrieve the current version of the database schema. + * If the database does not exist yet, the version returned will be 0. + */ int getSchema(); void openDB(State & state, bool create); From f5d83a80293426b2f869af206b93b5cdfeb34ec9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 13:36:28 +0100 Subject: [PATCH 058/211] One line per sentence in markdown docs. --- src/libstore/local-store.hh | 14 +++++++------- src/libutil/experimental-features.cc | 14 +++++--------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index dd563348b..0e7b1334f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -52,14 +52,14 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig R"( Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - Normally Nix will attempt to open the store database in read-write mode, even - for querying (when write access is not needed). This causes it to fail if the - database is on a read-only filesystem. + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed). + This causes it to fail if the database is on a read-only filesystem. - Enable read-only mode to disable locking and open the SQLite database with the - [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. Do not use this unless the filesystem is read-only. - Using it when the filesystem is writable can cause incorrect query results or - corruption errors if the database is changed by another process. + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + **Warning** + Do not use this unless the filesystem is read-only. + Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. )"}; const std::string name() override { return "Local Store"; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 1fbc99b34..9705c5b38 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -220,17 +220,13 @@ constexpr std::array xpFeatureDetails = {{ .description = R"( Allow the use of the `read-only` parameter in local store URIs. - Set this parameter to `true` to allow stores with databases on read-only - filesystems to be opened for querying; ordinarily Nix will refuse to do this. + Set this parameter to `true` to allow stores with databases on read-only filesystems to be opened for querying; ordinarily Nix will refuse to do this. - This is because SQLite requires write access to the database file to perform - the file locking operations necessary for safe concurrent access. When `read-only` - is set to `true`, the database will be opened in immutable mode. + This is because SQLite requires write access to the database file to perform the file locking operations necessary for safe concurrent access. + When `read-only` is set to `true`, the database will be opened in immutable mode. - Under this mode, SQLite will not do any locking at all, so you should be certain - that the database will not be changed. While the filesystem the database resides - on might be read-only to this process, consider whether another user, process, - or system, might have write access to it. + Under this mode, SQLite will not do any locking at all, so you should be certain that the database will not be changed. + While the filesystem the database resides on might be read-only to this process, consider whether another user, process, or system, might have write access to it. )", }, }}; From feb8d552aeca36adde5a9c25b92fd9eb205a761e Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Mon, 19 Jun 2023 13:22:41 +0100 Subject: [PATCH 059/211] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 0e7b1334f..bd4794eb8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -52,8 +52,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig R"( Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed). - This causes it to fail if the database is on a read-only filesystem. + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. From ef40448b1cb33165af5522f72d5d352d66035671 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 19 Jun 2023 13:47:21 +0100 Subject: [PATCH 060/211] Remove redundant description on experimental flag. --- src/libstore/local-store.hh | 1 + src/libutil/experimental-features.cc | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index bd4794eb8..1af64c77b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -59,6 +59,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig **Warning** Do not use this unless the filesystem is read-only. Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. )"}; const std::string name() override { return "Local Store"; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 7c2b29a07..3a04083ed 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -226,14 +226,6 @@ constexpr std::array xpFeatureDetails = {{ .name = "read-only-local-store", .description = R"( Allow the use of the `read-only` parameter in local store URIs. - - Set this parameter to `true` to allow stores with databases on read-only filesystems to be opened for querying; ordinarily Nix will refuse to do this. - - This is because SQLite requires write access to the database file to perform the file locking operations necessary for safe concurrent access. - When `read-only` is set to `true`, the database will be opened in immutable mode. - - Under this mode, SQLite will not do any locking at all, so you should be certain that the database will not be changed. - While the filesystem the database resides on might be read-only to this process, consider whether another user, process, or system, might have write access to it. )", }, }}; From b09baa3bc31226cac5ccdbd7ba9d72a2ed389207 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 19 Jun 2023 13:57:10 +0100 Subject: [PATCH 061/211] Link to LocalStore section of nix3-help-stores section. --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 3a04083ed..7c4112d32 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -225,7 +225,7 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReadOnlyLocalStore, .name = "read-only-local-store", .description = R"( - Allow the use of the `read-only` parameter in local store URIs. + Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. )", }, }}; From ba492a98baa0fa37b0914e3f7d00f2859cef5bcd Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:07:31 +0100 Subject: [PATCH 062/211] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1af64c77b..8a3b0b43f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -56,10 +56,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. - **Warning** - Do not use this unless the filesystem is read-only. - Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. )"}; const std::string name() override { return "Local Store"; } From 4e72b8483ede9242ddde99eb8011b18098172444 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:01:43 +0100 Subject: [PATCH 063/211] Update src/libstore/sqlite.hh Co-authored-by: John Ericson --- src/libstore/sqlite.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index b0e84f8ed..0c08267f7 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -29,7 +29,7 @@ enum class SQLiteOpenMode { * Use this mode if the database is on a read-only filesystem. * Fails with an error if the database does not exist. */ - Immutable + Immutable, }; /** From 2add2309397bd576712c1b90d364fb90525f4797 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Jul 2023 21:53:06 -0400 Subject: [PATCH 064/211] Fix build --- src/libstore/local-overlay-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index e6b877b04..6f995ba39 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -25,7 +25,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig Must be used as OverlayFS lower layer for this store's store dir. )"}; - const PathSetting upperLayer{(StoreConfig*) this, false, "", "upper-layer", + const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", R"( Must be used as OverlayFS upper layer for this store's store dir. )"}; From 735a672e1f3bbad8e32b211ce33fa4a308ae355a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Jul 2023 22:24:51 -0400 Subject: [PATCH 065/211] Introduce notion of a test group, use for CA tests Grouping our tests should make it easier to understand the intent than one long poorly-arranged list. It also is convenient for running just the tests for a specific component when working on that component. We need at least one test group so this isn't dead code; I decided to collect the tests for the `ca-derivations` and `dynamic-derivations` experimental features in groups. Do ```bash make ca.test-group -jN ``` and ```bash make dyn-drv.test-group -jN ``` to try running just them. I originally did this as part of #8397 for being able to just the local overlay store alone. I am PRing it separately now so we can separate general infra from new features. --- Makefile | 2 ++ doc/manual/src/contributing/testing.md | 28 +++++++++++++++++++++++++ mk/lib.mk | 12 ++++++++++- mk/tests.mk | 8 +++++-- tests/ca/local.mk | 27 ++++++++++++++++++++++++ tests/dyn-drv/local.mk | 11 ++++++++++ tests/local.mk | 29 +++----------------------- 7 files changed, 88 insertions(+), 29 deletions(-) create mode 100644 tests/ca/local.mk create mode 100644 tests/dyn-drv/local.mk diff --git a/Makefile b/Makefile index c6220482a..31b54b93d 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,8 @@ makefiles += \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ tests/local.mk \ + tests/ca/local.mk \ + tests/dyn-drv/local.mk \ tests/test-libstoreconsumer/local.mk \ tests/plugins/local.mk else diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index e5f80a928..da46a638d 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. Each test is a bash script. +### Running the whole test suite + The whole test suite can be run with: ```shell-session @@ -23,6 +25,32 @@ ran test tests/bar.sh... [PASS] ... ``` +### Grouping tests + +Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. +For example, `tests/ca/local.mk` defines a "ca" test group for tests relating to content-addressed derivation outputs. + +That test group can be run like this: + +```shell-session +$ make ca.test-group -j50 +ran test tests/ca/nix-run.sh... [PASS] +ran test tests/ca/import-derivation.sh... [PASS] +... +``` + +The testgroup is defined in Make like this: +```makefile +$(test-group-name)-tests := \ + $(d)/test0.sh \ + $(d)/test1.sh \ + ... + +install-tests-groups += $(test-group-name) +``` + +### Running individual tests + Individual tests can be run with `make`: ```shell-session diff --git a/mk/lib.mk b/mk/lib.mk index 34fa624d8..cddaf9eb1 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,6 +10,7 @@ bin-scripts := noinst-scripts := man-pages := install-tests := +install-tests-groups := ifdef HOST_OS HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) @@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +$(foreach test, $(install-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval installcheck: $(test).test)) +$(foreach test-group, $(install-tests-groups), \ + $(eval $(call run-install-test-group,$(test-group))) \ + $(eval installcheck: $(test-group).test-group) \ + $(foreach test, $($(test-group)-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval $(test-group).test-group: $(test).test))) + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/tests.mk b/mk/tests.mk index 3ebbd86e3..ec8128bdf 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -4,8 +4,6 @@ test-deps = define run-install-test - installcheck: $1.test - .PHONY: $1.test $1.test: $1 $(test-deps) @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null @@ -16,6 +14,12 @@ define run-install-test endef +define run-install-test-group + + .PHONY: $1.test-group + +endef + .PHONY: check installcheck print-top-help += \ diff --git a/tests/ca/local.mk b/tests/ca/local.mk new file mode 100644 index 000000000..d15312708 --- /dev/null +++ b/tests/ca/local.mk @@ -0,0 +1,27 @@ +ca-tests := \ + $(d)/build-with-garbage-path.sh \ + $(d)/build.sh \ + $(d)/concurrent-builds.sh \ + $(d)/derivation-json.sh \ + $(d)/duplicate-realisation-in-closure.sh \ + $(d)/gc.sh \ + $(d)/import-derivation.sh \ + $(d)/new-build-cmd.sh \ + $(d)/nix-copy.sh \ + $(d)/nix-run.sh \ + $(d)/nix-shell.sh \ + $(d)/post-hook.sh \ + $(d)/recursive.sh \ + $(d)/repl.sh \ + $(d)/selfref-gc.sh \ + $(d)/signatures.sh \ + $(d)/substitute.sh \ + $(d)/why-depends.sh + +install-tests-groups += ca + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/ca/config.nix diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk new file mode 100644 index 000000000..f065a5627 --- /dev/null +++ b/tests/dyn-drv/local.mk @@ -0,0 +1,11 @@ +dyn-drv-tests := \ + $(d)/text-hashed-output.sh \ + $(d)/recursive-mod-json.sh + +install-tests-groups += dyn-drv + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/dyn-drv/config.nix diff --git a/tests/local.mk b/tests/local.mk index 88848926b..8cf79fcb3 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -14,7 +14,6 @@ nix_tests = \ flakes/absolute-paths.sh \ flakes/build-paths.sh \ flakes/flake-in-submodule.sh \ - ca/gc.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ @@ -27,8 +26,6 @@ nix_tests = \ user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ - ca/build.sh \ - ca/new-build-cmd.sh \ nix-build.sh \ gc-concurrent.sh \ repair.sh \ @@ -46,24 +43,17 @@ nix_tests = \ referrers.sh \ optimise-store.sh \ substitute-with-invalid-ca.sh \ - ca/concurrent-builds.sh \ signing.sh \ - ca/build-with-garbage-path.sh \ hash.sh \ gc-non-blocking.sh \ check.sh \ - ca/substitute.sh \ nix-shell.sh \ - ca/signatures.sh \ - ca/nix-shell.sh \ - ca/nix-copy.sh \ check-refs.sh \ build-remote-input-addressed.sh \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ flakes/search-root.sh \ - ca/duplicate-realisation-in-closure.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -79,10 +69,7 @@ nix_tests = \ nar-access.sh \ pure-eval.sh \ eval.sh \ - ca/post-hook.sh \ repl.sh \ - ca/repl.sh \ - ca/recursive.sh \ binary-cache-build-remote.sh \ search.sh \ logging.sh \ @@ -107,13 +94,8 @@ nix_tests = \ fmt.sh \ eval-store.sh \ why-depends.sh \ - ca/why-depends.sh \ derivation-json.sh \ - ca/derivation-json.sh \ import-derivation.sh \ - ca/import-derivation.sh \ - dyn-drv/text-hashed-output.sh \ - dyn-drv/recursive-mod-json.sh \ nix_path.sh \ case-hack.sh \ placeholders.sh \ @@ -122,8 +104,7 @@ nix_tests = \ build.sh \ build-delete.sh \ output-normalization.sh \ - ca/nix-run.sh \ - selfref-gc.sh ca/selfref-gc.sh \ + selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ @@ -147,16 +128,12 @@ install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) clean-files += \ $(d)/common/vars-and-functions.sh \ - $(d)/config.nix \ - $(d)/ca/config.nix \ - $(d)/dyn-drv/config.nix + $(d)/config.nix test-deps += \ tests/common/vars-and-functions.sh \ tests/config.nix \ - tests/ca/config.nix \ - tests/test-libstoreconsumer/test-libstoreconsumer \ - tests/dyn-drv/config.nix + tests/test-libstoreconsumer/test-libstoreconsumer ifeq ($(BUILD_SHARED_LIBS), 1) test-deps += tests/plugins/libplugintest.$(SO_EXT) From 83cfa82e52e26a287a5bdb60863751096ab02c63 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Thu, 13 Jul 2023 14:39:46 -0500 Subject: [PATCH 066/211] Add unset to NIX_STORE_DIR for local-overlay tests --- tests/overlay-local-store/build-inner.sh | 4 ++++ tests/overlay-local-store/check-post-init-inner.sh | 4 ++++ tests/overlay-local-store/redundant-add-inner.sh | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/tests/overlay-local-store/build-inner.sh b/tests/overlay-local-store/build-inner.sh index 969de282d..beb40b2fc 100755 --- a/tests/overlay-local-store/build-inner.sh +++ b/tests/overlay-local-store/build-inner.sh @@ -6,6 +6,10 @@ set -x source common.sh +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + storeDirs initLowerStore diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/overlay-local-store/check-post-init-inner.sh index 421b64934..37ed8c113 100755 --- a/tests/overlay-local-store/check-post-init-inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -6,6 +6,10 @@ set -x source common.sh +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + storeDirs initLowerStore diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/overlay-local-store/redundant-add-inner.sh index 921069c25..97969b40e 100755 --- a/tests/overlay-local-store/redundant-add-inner.sh +++ b/tests/overlay-local-store/redundant-add-inner.sh @@ -6,6 +6,10 @@ set -x source common.sh +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + storeDirs initLowerStore From 37598a13e877980d4654c632d10921c41c9ad976 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 09:26:48 +0100 Subject: [PATCH 067/211] Revert "Check _NIX_TEST_NO_SANDBOX when setting _canUseSandbox." This reverts commit 7ed0ab2dabececd84578ddf70ecad0e528d67a28. --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index ad0623871..dc7ce13cc 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -141,7 +141,7 @@ restartDaemon() { startDaemon } -if [[ -z "${_NIX_TEST_NO_SANDBOX:-}" ]] && [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then _canUseSandbox=1 fi From f66b65a30ad1f14dd757874c12255eef42368b04 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 09:29:27 +0100 Subject: [PATCH 068/211] Revert "Skip build-remote-trustless unless sandbox is supported." This reverts commit fad0dd4afb66577f9c17b5a315d120ac0f5acd94. --- tests/build-remote-trustless-should-fail-0.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh index b14101f83..fad1def59 100644 --- a/tests/build-remote-trustless-should-fail-0.sh +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -2,7 +2,6 @@ source common.sh enableFeatures "daemon-trust-override" -requireSandboxSupport restartDaemon [[ $busybox =~ busybox ]] || skipTest "no busybox" From a33ee5c84305b85f5bcd0ea1be43b44cf601693f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 10:49:44 +0100 Subject: [PATCH 069/211] Paths added to lower store are accessible via overlay. --- tests/overlay-local-store/add-lower-inner.sh | 31 +++++++++++++++++++ tests/overlay-local-store/add-lower.sh | 5 +++ tests/overlay-local-store/common.sh | 11 ++++++- tests/overlay-local-store/local.mk | 3 +- .../redundant-add-inner.sh | 5 +-- 5 files changed, 51 insertions(+), 4 deletions(-) create mode 100755 tests/overlay-local-store/add-lower-inner.sh create mode 100755 tests/overlay-local-store/add-lower.sh diff --git a/tests/overlay-local-store/add-lower-inner.sh b/tests/overlay-local-store/add-lower-inner.sh new file mode 100755 index 000000000..40a1f397c --- /dev/null +++ b/tests/overlay-local-store/add-lower-inner.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +# Add something to the overlay store +overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store") +stat "$TEST_ROOT/merged-store/$overlayPath" + +# Now add something to the lower store +lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store") +stat "$TEST_ROOT/store-a/$lowerPath" + +# Remount overlayfs to ensure synchronization +mount -o remount "$TEST_ROOT/merged-store/nix/store" + +# Path should be accessible via overlay store +stat "$TEST_ROOT/merged-store/$lowerPath" diff --git a/tests/overlay-local-store/add-lower.sh b/tests/overlay-local-store/add-lower.sh new file mode 100755 index 000000000..f0ac46a91 --- /dev/null +++ b/tests/overlay-local-store/add-lower.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./add-lower-inner.sh diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 149102000..7850c068b 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -8,7 +8,7 @@ requireEnvironment () { } setupConfig () { - echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf + echo "require-drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf } @@ -54,3 +54,12 @@ initLowerStore () { execUnshare () { exec unshare --mount --map-root-user "$@" } + +addTextToStore() { + storeDir=$1; shift + filename=$1; shift + content=$1; shift + filePath="$TEST_HOME/$filename" + echo "$content" > "$filePath" + nix-store --store "$storeDir" --add "$filePath" +} diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index b94238a67..5d14e2561 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -2,6 +2,7 @@ overlay-local-store-tests := \ $(d)/check-post-init.sh \ $(d)/redundant-add.sh \ $(d)/build.sh \ - $(d)/bad-uris.sh + $(d)/bad-uris.sh \ + $(d)/add-lower.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/overlay-local-store/redundant-add-inner.sh index 97969b40e..cfdae68b4 100755 --- a/tests/overlay-local-store/redundant-add-inner.sh +++ b/tests/overlay-local-store/redundant-add-inner.sh @@ -7,7 +7,7 @@ set -x source common.sh # Avoid store dir being inside sandbox build-dir -unset NIX_STORE_DIR +unset NIX_STORE_DIR # TODO: This causes toRealPath to fail (it expects this var to be set) unset NIX_STATE_DIR storeDirs @@ -27,4 +27,5 @@ path=$(nix-store --store "$storeB" --add ../dummy) stat $(toRealPath "$storeA/nix/store" "$path") # upper layer should still not have it (no redundant copy) -expect 1 stat $(toRealPath "$storeB/nix/store" "$path") +expect 1 stat $(toRealPath "$storeB/nix/store" "$path") # TODO: Check this is failing for the right reason. + # $storeB is a store URI not a directory path From 0ccf6382af4c34dafacd2a417ba658e62d81adbf Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 12:30:33 +0100 Subject: [PATCH 070/211] Add test for verifying overlay store. --- tests/overlay-local-store/local.mk | 3 ++- tests/overlay-local-store/verify-inner.sh | 32 +++++++++++++++++++++++ tests/overlay-local-store/verify.sh | 5 ++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100755 tests/overlay-local-store/verify-inner.sh create mode 100755 tests/overlay-local-store/verify.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 5d14e2561..5e8c76b4e 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -3,6 +3,7 @@ overlay-local-store-tests := \ $(d)/redundant-add.sh \ $(d)/build.sh \ $(d)/bad-uris.sh \ - $(d)/add-lower.sh + $(d)/add-lower.sh \ + $(d)/verify.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh new file mode 100755 index 000000000..b68800e5c --- /dev/null +++ b/tests/overlay-local-store/verify-inner.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +#path=$(nix-store --store "$storeB" --add ../dummy) + +path=$(nix-build --store $storeB ../hermetic.nix --arg busybox "$busybox" --arg seed 1) + +inputDrvPath=$(find "$storeA" -name "*-hermetic-input-1.drv") +rm -v "$inputDrvPath" + +#tree "$TEST_ROOT" + +#rm -v "$TEST_ROOT/store-a/$path" + +nix-store --store "$storeB" --verify + +echo "SUCCESS" diff --git a/tests/overlay-local-store/verify.sh b/tests/overlay-local-store/verify.sh new file mode 100755 index 000000000..8b44603ff --- /dev/null +++ b/tests/overlay-local-store/verify.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./verify-inner.sh From 58085e4eff1e06524f9ad8f6e9f271c19dcbba9e Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 13:10:34 +0100 Subject: [PATCH 071/211] Have verify test exercise check-contents too. --- tests/overlay-local-store/verify-inner.sh | 25 +++++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh index b68800e5c..5f8cbcf0e 100755 --- a/tests/overlay-local-store/verify-inner.sh +++ b/tests/overlay-local-store/verify-inner.sh @@ -16,17 +16,24 @@ initLowerStore mountOverlayfs -#path=$(nix-store --store "$storeB" --add ../dummy) +# Realise a derivation from the lower store to propagate paths to overlay DB +nix-store --store "$storeB" --realise $drvPath -path=$(nix-build --store $storeB ../hermetic.nix --arg busybox "$busybox" --arg seed 1) +# Also ensure dummy file exists in overlay DB +dummyPath=$(nix-store --store "$storeB" --add ../dummy) -inputDrvPath=$(find "$storeA" -name "*-hermetic-input-1.drv") -rm -v "$inputDrvPath" +# Verify should be successful at this point +nix-store --store "$storeB" --verify --check-contents -#tree "$TEST_ROOT" +# Now delete one of the derivation inputs in the lower store +inputDrvFullPath=$(find "$storeA" -name "*-hermetic-input-1.drv") +inputDrvPath=${inputDrvFullPath/*\/nix\/store\///nix/store/} +rm -v "$inputDrvFullPath" -#rm -v "$TEST_ROOT/store-a/$path" +# And truncate the contents of dummy file in lower store +find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; -nix-store --store "$storeB" --verify - -echo "SUCCESS" +# Verify should fail with the messages about missing input and modified dummy file +verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents) +<<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!" +<<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash" From d5cd74a4012d583d11d491bec9f1fa9a07b11973 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 13:49:13 +0100 Subject: [PATCH 072/211] Override verifyStore to always pass NoRepair for LocalOverlayStore. --- src/libstore/local-overlay-store.cc | 7 +++++++ src/libstore/local-overlay-store.hh | 2 ++ tests/overlay-local-store/verify-inner.sh | 3 ++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ddccb8c5a..182955fbe 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -188,6 +188,13 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) } } +bool LocalOverlayStore::verifyStore(bool checkContents, RepairFlag repair) +{ + if (repair) + warn("local-overlay: store does not support --verify --repair"); + return LocalStore::verifyStore(checkContents, NoRepair); +} + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 6f995ba39..8fa99ee14 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -112,6 +112,8 @@ private: Callback> callback) noexcept override; void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; + + bool verifyStore(bool checkContents, RepairFlag repair) override; }; } diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh index 5f8cbcf0e..8f8839302 100755 --- a/tests/overlay-local-store/verify-inner.sh +++ b/tests/overlay-local-store/verify-inner.sh @@ -34,6 +34,7 @@ rm -v "$inputDrvFullPath" find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; # Verify should fail with the messages about missing input and modified dummy file -verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents) +verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents --repair) <<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!" <<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash" +<<<"$verifyOutput" grepQuiet "store does not support --verify --repair" From 614efc1240dbfefbdabced808decd8defd33df8e Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 18 Jul 2023 13:59:22 +0100 Subject: [PATCH 073/211] Add test for store optimise path deduplication. --- src/libstore/local-overlay-store.cc | 5 +++++ src/libstore/local-overlay-store.hh | 2 ++ tests/overlay-local-store/local.mk | 3 ++- tests/overlay-local-store/optimise-inner.sh | 19 +++++++++++++++++++ tests/overlay-local-store/optimise.sh | 5 +++++ 5 files changed, 33 insertions(+), 1 deletion(-) create mode 100755 tests/overlay-local-store/optimise-inner.sh create mode 100755 tests/overlay-local-store/optimise.sh diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 182955fbe..da0c8e3d5 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -188,6 +188,11 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) } } +void LocalOverlayStore::optimiseStore() +{ + warn("not implemented"); +} + bool LocalOverlayStore::verifyStore(bool checkContents, RepairFlag repair) { if (repair) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 8fa99ee14..ef377b7a6 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -113,6 +113,8 @@ private: void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; + void optimiseStore() override; + bool verifyStore(bool checkContents, RepairFlag repair) override; }; diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 5e8c76b4e..3a6d00bc1 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,6 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ - $(d)/verify.sh + $(d)/verify.sh \ + $(d)/optimise.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh new file mode 100755 index 000000000..76c5a0cb6 --- /dev/null +++ b/tests/overlay-local-store/optimise-inner.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +nix-store --store "$storeB" --optimise diff --git a/tests/overlay-local-store/optimise.sh b/tests/overlay-local-store/optimise.sh new file mode 100755 index 000000000..569afa248 --- /dev/null +++ b/tests/overlay-local-store/optimise.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./optimise-inner.sh From a9510f950228957cde98001b67e123aa2d4d62c9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 11:23:54 +0100 Subject: [PATCH 074/211] Implement test for store path deduplication. --- tests/overlay-local-store/common.sh | 16 ++++------ tests/overlay-local-store/optimise-inner.sh | 33 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 7850c068b..5a678c947 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -22,11 +22,12 @@ storeDirs () { # Mounting Overlay Store mountOverlayfs () { + mergedStorePath="$TEST_ROOT/merged-store/nix/store" mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ -o upperdir="$storeBTop" \ -o workdir="$TEST_ROOT/workdir" \ - "$TEST_ROOT/merged-store/nix/store" \ + "$mergedStorePath" \ || skipTest "overlayfs is not supported" cleanupOverlay () { @@ -36,6 +37,10 @@ mountOverlayfs () { trap cleanupOverlay EXIT } +remountOverlayfs () { + mount -o remount "$mergedStorePath" +} + toRealPath () { storeDir=$1; shift storePath=$1; shift @@ -54,12 +59,3 @@ initLowerStore () { execUnshare () { exec unshare --mount --map-root-user "$@" } - -addTextToStore() { - storeDir=$1; shift - filename=$1; shift - content=$1; shift - filePath="$TEST_HOME/$filename" - echo "$content" > "$filePath" - nix-store --store "$storeDir" --add "$filePath" -} diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index 76c5a0cb6..c60c742a2 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -16,4 +16,37 @@ initLowerStore mountOverlayfs +# Create a file to add to store +dupFilePath="$TEST_ROOT/dup-file" +echo Duplicate > "$dupFilePath" + +# Add it to the overlay store (it will be written to the upper layer) +dupFileStorePath=$(nix-store --store "$storeB" --add "$dupFilePath") + +# Now add it to the lower store so the store path is duplicated +nix-store --store "$storeA" --add "$dupFilePath" + +# Ensure overlayfs and layers and synchronised +remountOverlayfs + +dupFilename="${dupFileStorePath#/nix/store}" +lowerPath="$storeA/$dupFileStorePath" +upperPath="$storeBTop/$dupFilename" +overlayPath="$mergedStorePath/$dupFilename" + +# Check store path exists in both layers and overlay +lowerInode=$(stat -c %i "$lowerPath") +upperInode=$(stat -c %i "$upperPath") +overlayInode=$(stat -c %i "$overlayPath") +[[ $upperInode == $overlayInode ]] +[[ $upperInode != $lowerInode ]] + +# Run optimise to deduplicate store paths nix-store --store "$storeB" --optimise +remountOverlayfs + +stat "$lowerPath" +stat "$overlayPath" +expect 1 stat "$upperPath" + +#expect 1 stat $(toRealPath "$storeA/nix/store" "$path") From 8ddbcb736a17d08ce899de345548064eebf673c9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 12:32:32 +0100 Subject: [PATCH 075/211] Implement overlay store deduplication. --- src/libstore/local-overlay-store.cc | 18 +++++++++++++++++- tests/overlay-local-store/optimise-inner.sh | 3 +-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index da0c8e3d5..9e530ed9b 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -190,7 +190,23 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) void LocalOverlayStore::optimiseStore() { - warn("not implemented"); + Activity act(*logger, actOptimiseStore); + + // Note for LocalOverlayStore, queryAllValidPaths only returns paths in upper layer + auto paths = queryAllValidPaths(); + + act.progress(0, paths.size()); + + uint64_t done = 0; + + for (auto & path : paths) { + if (lowerStore->isValidPath(path)) { + // Deduplicate store path + deletePath(toUpperPath(path)); + } + done++; + act.progress(done, paths.size()); + } } bool LocalOverlayStore::verifyStore(bool checkContents, RepairFlag repair) diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index c60c742a2..2b778b311 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -45,8 +45,7 @@ overlayInode=$(stat -c %i "$overlayPath") nix-store --store "$storeB" --optimise remountOverlayfs +# Check path only exists in lower store stat "$lowerPath" stat "$overlayPath" expect 1 stat "$upperPath" - -#expect 1 stat $(toRealPath "$storeA/nix/store" "$path") From d1c77b201a1b0e6c14470b69c134652a8858fc18 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 13:25:37 +0100 Subject: [PATCH 076/211] Explicitly exec shell to fix ENOENT errors. --- tests/overlay-local-store/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 5a678c947..0e98e931f 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -57,5 +57,5 @@ initLowerStore () { } execUnshare () { - exec unshare --mount --map-root-user "$@" + exec unshare --mount --map-root-user "$SHELL" "$@" } From 44f855d14ee40dfeaf52c4f01e2de124fe0f18fc Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 15:09:58 +0100 Subject: [PATCH 077/211] Missing addTextToStore function. --- tests/overlay-local-store/common.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 0e98e931f..2fba2e7f6 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -59,3 +59,12 @@ initLowerStore () { execUnshare () { exec unshare --mount --map-root-user "$SHELL" "$@" } + +addTextToStore() { + storeDir=$1; shift + filename=$1; shift + content=$1; shift + filePath="$TEST_HOME/$filename" + echo "$content" > "$filePath" + nix-store --store "$storeDir" --add "$filePath" +} From 7fda19e2f139a7e41cb53cdbdd692dbc98755d91 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 19 Jul 2023 16:46:51 +0100 Subject: [PATCH 078/211] Mount tmpfs first to ensure overlayfs works consistently. --- tests/overlay-local-store/common.sh | 19 +++++++++++-------- tests/overlay-local-store/optimise-inner.sh | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 2fba2e7f6..6aec96ba1 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -13,26 +13,29 @@ setupConfig () { } storeDirs () { - storeA="$TEST_ROOT/store-a" - storeBTop="$TEST_ROOT/store-b" - storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + storesRoot="$TEST_ROOT/stores" + mkdir -p "$storesRoot" + mount -t tmpfs tmpfs "$storesRoot" + storeA="$storesRoot/store-a" + storeBTop="$storesRoot/store-b" + storeB="local-overlay?root=$storesRoot/merged-store&lower-store=$storeA&upper-layer=$storeBTop" # Creating testing directories - mkdir -p "$TEST_ROOT"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} + mkdir -p "$storesRoot"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } # Mounting Overlay Store mountOverlayfs () { - mergedStorePath="$TEST_ROOT/merged-store/nix/store" + mergedStorePath="$storesRoot/merged-store/nix/store" mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ -o upperdir="$storeBTop" \ - -o workdir="$TEST_ROOT/workdir" \ + -o workdir="$storesRoot/workdir" \ "$mergedStorePath" \ || skipTest "overlayfs is not supported" cleanupOverlay () { - umount "$TEST_ROOT/merged-store/nix/store" - rm -r $TEST_ROOT/workdir + umount "$storesRoot/merged-store/nix/store" + rm -r $storesRoot/workdir } trap cleanupOverlay EXIT } diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index 2b778b311..079e22326 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -17,7 +17,7 @@ initLowerStore mountOverlayfs # Create a file to add to store -dupFilePath="$TEST_ROOT/dup-file" +dupFilePath="$storesRoot/dup-file" echo Duplicate > "$dupFilePath" # Add it to the overlay store (it will be written to the upper layer) From 9769a0ae7d840f1df7db3de4524180016830e57e Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 20 Jul 2023 10:27:11 +0100 Subject: [PATCH 079/211] Ensure all overlay tests use new tmpfs store paths. --- tests/overlay-local-store/add-lower-inner.sh | 8 +++--- tests/overlay-local-store/bad-uris.sh | 4 +-- .../check-post-init-inner.sh | 2 +- tests/overlay-local-store/common.sh | 28 +++++++++++-------- tests/overlay-local-store/optimise-inner.sh | 2 +- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/tests/overlay-local-store/add-lower-inner.sh b/tests/overlay-local-store/add-lower-inner.sh index 40a1f397c..8c71f0780 100755 --- a/tests/overlay-local-store/add-lower-inner.sh +++ b/tests/overlay-local-store/add-lower-inner.sh @@ -18,14 +18,14 @@ mountOverlayfs # Add something to the overlay store overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store") -stat "$TEST_ROOT/merged-store/$overlayPath" +stat "$storeVolume/merged-store/$overlayPath" # Now add something to the lower store lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store") -stat "$TEST_ROOT/store-a/$lowerPath" +stat "$storeVolume/store-a/$lowerPath" # Remount overlayfs to ensure synchronization -mount -o remount "$TEST_ROOT/merged-store/nix/store" +mount -o remount "$storeVolume/merged-store/nix/store" # Path should be accessible via overlay store -stat "$TEST_ROOT/merged-store/$lowerPath" +stat "$storeVolume/merged-store/$lowerPath" diff --git a/tests/overlay-local-store/bad-uris.sh b/tests/overlay-local-store/bad-uris.sh index d4261bd97..c59264735 100644 --- a/tests/overlay-local-store/bad-uris.sh +++ b/tests/overlay-local-store/bad-uris.sh @@ -7,8 +7,8 @@ storeDirs mkdir -p $TEST_ROOT/bad_test badTestRoot=$TEST_ROOT/bad_test storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" -storeBadLower="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" -storeBadUpper="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" +storeBadLower="local-overlay?root=$storeVolume/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay?root=$storeVolume/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" declare -a storesBad=( "$storeBadRoot" "$storeBadLower" "$storeBadUpper" diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/overlay-local-store/check-post-init-inner.sh index 37ed8c113..c7d1c70d4 100755 --- a/tests/overlay-local-store/check-post-init-inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -25,7 +25,7 @@ stat $(toRealPath "$storeA/nix/store" "$path") expect 1 stat $(toRealPath "$storeBTop" "$path") # Checking for path in overlay store matching lower layer -diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-store/nix/store" "$path") +diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeVolume/merged-store/nix/store" "$path") # Checking requisites query agreement [[ \ diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 6aec96ba1..686523e9c 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -12,30 +12,36 @@ setupConfig () { echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf } + + storeDirs () { - storesRoot="$TEST_ROOT/stores" - mkdir -p "$storesRoot" - mount -t tmpfs tmpfs "$storesRoot" - storeA="$storesRoot/store-a" - storeBTop="$storesRoot/store-b" - storeB="local-overlay?root=$storesRoot/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + # Attempt to create store dirs on tmpfs volume. + # This ensures lowerdir, upperdir and workdir will be on + # a consistent filesystem that fully supports OverlayFS. + storeVolume="$TEST_ROOT/stores" + mkdir -p "$storeVolume" + mount -t tmpfs tmpfs "$storeVolume" || true # But continue anyway if that fails. + + storeA="$storeVolume/store-a" + storeBTop="$storeVolume/store-b" + storeB="local-overlay?root=$storeVolume/merged-store&lower-store=$storeA&upper-layer=$storeBTop" # Creating testing directories - mkdir -p "$storesRoot"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} + mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } # Mounting Overlay Store mountOverlayfs () { - mergedStorePath="$storesRoot/merged-store/nix/store" + mergedStorePath="$storeVolume/merged-store/nix/store" mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ -o upperdir="$storeBTop" \ - -o workdir="$storesRoot/workdir" \ + -o workdir="$storeVolume/workdir" \ "$mergedStorePath" \ || skipTest "overlayfs is not supported" cleanupOverlay () { - umount "$storesRoot/merged-store/nix/store" - rm -r $storesRoot/workdir + umount "$storeVolume/merged-store/nix/store" + rm -r $storeVolume/workdir } trap cleanupOverlay EXIT } diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index 079e22326..2b778b311 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -17,7 +17,7 @@ initLowerStore mountOverlayfs # Create a file to add to store -dupFilePath="$storesRoot/dup-file" +dupFilePath="$TEST_ROOT/dup-file" echo Duplicate > "$dupFilePath" # Add it to the overlay store (it will be written to the upper layer) From 878c84d5ee88f2e6c56abb42211a729c4d342e14 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 20 Jul 2023 10:27:35 +0100 Subject: [PATCH 080/211] Fix errors about NIX_STORE_DIR being unset. --- tests/overlay-local-store/check-post-init-inner.sh | 2 +- tests/overlay-local-store/common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/overlay-local-store/check-post-init-inner.sh index c7d1c70d4..2e7db2adc 100755 --- a/tests/overlay-local-store/check-post-init-inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -62,7 +62,7 @@ nix-store --verify-path --store "$storeA" "$path" # Verifying path in merged-store nix-store --verify-path --store "$storeB" "$path" -hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//') +hashPart=$(echo $path | sed "s^${NIX_STORE_DIR:-/nix/store}/^^" | sed 's/-.*//') # Lower store can find from hash part [[ $(nix store --store $storeA path-from-hash-part $hashPart) == $path ]] diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 686523e9c..ac4a59cd8 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -53,7 +53,7 @@ remountOverlayfs () { toRealPath () { storeDir=$1; shift storePath=$1; shift - echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^") + echo $storeDir$(echo $storePath | sed "s^${NIX_STORE_DIR:-/nix/store}^^") } initLowerStore () { From 2c66a093e05bc466d00c9365bc43567fd9d5c334 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 20 Jul 2023 11:03:14 +0100 Subject: [PATCH 081/211] Define storeBRoot variable distinct from storeB URI. --- tests/overlay-local-store/add-lower-inner.sh | 6 +++--- tests/overlay-local-store/bad-uris.sh | 4 ++-- tests/overlay-local-store/check-post-init-inner.sh | 2 +- tests/overlay-local-store/common.sh | 10 +++++----- tests/overlay-local-store/redundant-add-inner.sh | 5 ++--- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/overlay-local-store/add-lower-inner.sh b/tests/overlay-local-store/add-lower-inner.sh index 8c71f0780..ca7db7ab6 100755 --- a/tests/overlay-local-store/add-lower-inner.sh +++ b/tests/overlay-local-store/add-lower-inner.sh @@ -18,14 +18,14 @@ mountOverlayfs # Add something to the overlay store overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store") -stat "$storeVolume/merged-store/$overlayPath" +stat "$storeBRoot/$overlayPath" # Now add something to the lower store lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store") stat "$storeVolume/store-a/$lowerPath" # Remount overlayfs to ensure synchronization -mount -o remount "$storeVolume/merged-store/nix/store" +remountOverlayfs # Path should be accessible via overlay store -stat "$storeVolume/merged-store/$lowerPath" +stat "$storeBRoot/$lowerPath" diff --git a/tests/overlay-local-store/bad-uris.sh b/tests/overlay-local-store/bad-uris.sh index c59264735..462bf27eb 100644 --- a/tests/overlay-local-store/bad-uris.sh +++ b/tests/overlay-local-store/bad-uris.sh @@ -7,8 +7,8 @@ storeDirs mkdir -p $TEST_ROOT/bad_test badTestRoot=$TEST_ROOT/bad_test storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" -storeBadLower="local-overlay?root=$storeVolume/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop" -storeBadUpper="local-overlay?root=$storeVolume/merged-store&lower-store=$storeA&upper-layer=$badTestRoot" +storeBadLower="local-overlay?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot" declare -a storesBad=( "$storeBadRoot" "$storeBadLower" "$storeBadUpper" diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/overlay-local-store/check-post-init-inner.sh index 2e7db2adc..0f4654cc2 100755 --- a/tests/overlay-local-store/check-post-init-inner.sh +++ b/tests/overlay-local-store/check-post-init-inner.sh @@ -25,7 +25,7 @@ stat $(toRealPath "$storeA/nix/store" "$path") expect 1 stat $(toRealPath "$storeBTop" "$path") # Checking for path in overlay store matching lower layer -diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeVolume/merged-store/nix/store" "$path") +diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeBRoot/nix/store" "$path") # Checking requisites query agreement [[ \ diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index ac4a59cd8..2b23352ab 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -24,30 +24,30 @@ storeDirs () { storeA="$storeVolume/store-a" storeBTop="$storeVolume/store-b" - storeB="local-overlay?root=$storeVolume/merged-store&lower-store=$storeA&upper-layer=$storeBTop" + storeBRoot="$storeVolume/merged-store" + storeB="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop" # Creating testing directories mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } # Mounting Overlay Store mountOverlayfs () { - mergedStorePath="$storeVolume/merged-store/nix/store" mount -t overlay overlay \ -o lowerdir="$storeA/nix/store" \ -o upperdir="$storeBTop" \ -o workdir="$storeVolume/workdir" \ - "$mergedStorePath" \ + "$storeBRoot/nix/store" \ || skipTest "overlayfs is not supported" cleanupOverlay () { - umount "$storeVolume/merged-store/nix/store" + umount "$storeBRoot/nix/store" rm -r $storeVolume/workdir } trap cleanupOverlay EXIT } remountOverlayfs () { - mount -o remount "$mergedStorePath" + mount -o remount "$storeBRoot/nix/store" } toRealPath () { diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/overlay-local-store/redundant-add-inner.sh index cfdae68b4..34b841e38 100755 --- a/tests/overlay-local-store/redundant-add-inner.sh +++ b/tests/overlay-local-store/redundant-add-inner.sh @@ -7,7 +7,7 @@ set -x source common.sh # Avoid store dir being inside sandbox build-dir -unset NIX_STORE_DIR # TODO: This causes toRealPath to fail (it expects this var to be set) +unset NIX_STORE_DIR unset NIX_STATE_DIR storeDirs @@ -27,5 +27,4 @@ path=$(nix-store --store "$storeB" --add ../dummy) stat $(toRealPath "$storeA/nix/store" "$path") # upper layer should still not have it (no redundant copy) -expect 1 stat $(toRealPath "$storeB/nix/store" "$path") # TODO: Check this is failing for the right reason. - # $storeB is a store URI not a directory path +expect 1 stat $(toRealPath "$storeBTop" "$path") From 2fc00ec19f8eb7ec2285135a180763d632102762 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 20 Jul 2023 11:27:28 +0100 Subject: [PATCH 082/211] Fix unbound variable error in optimise test. --- tests/overlay-local-store/optimise-inner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/overlay-local-store/optimise-inner.sh index 2b778b311..b7994054c 100755 --- a/tests/overlay-local-store/optimise-inner.sh +++ b/tests/overlay-local-store/optimise-inner.sh @@ -32,7 +32,7 @@ remountOverlayfs dupFilename="${dupFileStorePath#/nix/store}" lowerPath="$storeA/$dupFileStorePath" upperPath="$storeBTop/$dupFilename" -overlayPath="$mergedStorePath/$dupFilename" +overlayPath="$storeBRoot/nix/store/$dupFilename" # Check store path exists in both layers and overlay lowerInode=$(stat -c %i "$lowerPath") From 0e595a52a32003ed448b1da9319a639059f993a6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jul 2023 15:39:59 -0400 Subject: [PATCH 083/211] Remove trailing whitespace --- src/libstore/local-overlay-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 9e530ed9b..aced582ca 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -87,7 +87,7 @@ void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput, - Callback> callback) noexcept + Callback> callback) noexcept { auto callbackPtr = std::make_shared(std::move(callback)); From 3731208dc120b5d12f1c5c63ec58bcdb34c42195 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jul 2023 16:54:14 -0400 Subject: [PATCH 084/211] Adopt GC test for local-overlay store Doesn't yet pass. Fixes are needed. --- tests/overlay-local-store/gc-inner.sh | 57 +++++++++++++++++++++++++++ tests/overlay-local-store/gc.sh | 5 +++ tests/overlay-local-store/local.mk | 1 + 3 files changed, 63 insertions(+) create mode 100644 tests/overlay-local-store/gc-inner.sh create mode 100755 tests/overlay-local-store/gc.sh diff --git a/tests/overlay-local-store/gc-inner.sh b/tests/overlay-local-store/gc-inner.sh new file mode 100644 index 000000000..01ec48b84 --- /dev/null +++ b/tests/overlay-local-store/gc-inner.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +export NIX_REMOTE="$storeB" +stateB="$storeBRoot/nix/var/nix" +outPath=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2) + +# Set a GC root. +mkdir -p "$stateB" +rm -f "$stateB"/gcroots/foo +ln -sf $outPath "$stateB"/gcroots/foo + +[ "$(nix-store -q --roots $outPath)" = "$stateB/gcroots/foo -> $outPath" ] + +nix-store --gc --print-roots | grep $outPath +nix-store --gc --print-live | grep $outPath +if nix-store --gc --print-dead | grep -E $outPath$; then false; fi + +nix-store --gc --print-dead + +expect 1 nix-store --delete $outPath +test -e "$storeBRoot/$outPath" + +shopt -s nullglob +for i in $storeBRoot/*; do + if [[ $i =~ /trash ]]; then continue; fi # compat with old daemon + touch $i.lock + touch $i.chroot +done + +nix-collect-garbage + +# Check that the root and its dependencies haven't been deleted. +cat "$storeBRoot/$outPath" + +rm "$stateB"/gcroots/foo + +nix-collect-garbage + +# Check that the output has been GC'd. +test ! -e $outPath + +# Check that the store is empty. +[ "$(ls -1 "$storeBTop" | wc -l)" = "0" ] diff --git a/tests/overlay-local-store/gc.sh b/tests/overlay-local-store/gc.sh new file mode 100755 index 000000000..1e1fb203e --- /dev/null +++ b/tests/overlay-local-store/gc.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./gc-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 3a6d00bc1..1f8de8f27 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,6 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ + $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh From 497464f4945200522f05a318720ad0a158095e8e Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 25 Jul 2023 13:30:21 +0100 Subject: [PATCH 085/211] Extend verify test to check that repair is supported. --- src/libstore/local-overlay-store.cc | 7 ----- src/libstore/local-overlay-store.hh | 2 -- tests/overlay-local-store/common.sh | 8 +++-- tests/overlay-local-store/verify-inner.sh | 36 +++++++++++++++++++---- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 9e530ed9b..c8464f64d 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -209,13 +209,6 @@ void LocalOverlayStore::optimiseStore() } } -bool LocalOverlayStore::verifyStore(bool checkContents, RepairFlag repair) -{ - if (repair) - warn("local-overlay: store does not support --verify --repair"); - return LocalStore::verifyStore(checkContents, NoRepair); -} - static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index ef377b7a6..349d9e6ed 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -114,8 +114,6 @@ private: void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; void optimiseStore() override; - - bool verifyStore(bool checkContents, RepairFlag repair) override; }; } diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 2b23352ab..6bb5bc391 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -7,9 +7,13 @@ requireEnvironment () { needLocalStore "The test uses --store always so we would just be bypassing the daemon" } +addConfig () { + echo "$1" >> "$NIX_CONF_DIR/nix.conf" +} + setupConfig () { - echo "require-drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf - echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf + addConfig "require-drop-supplementary-groups = false" + addConfig "build-users-group = " } diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh index 8f8839302..de40c3d05 100755 --- a/tests/overlay-local-store/verify-inner.sh +++ b/tests/overlay-local-store/verify-inner.sh @@ -16,25 +16,51 @@ initLowerStore mountOverlayfs + +## Initialise stores for test + # Realise a derivation from the lower store to propagate paths to overlay DB nix-store --store "$storeB" --realise $drvPath # Also ensure dummy file exists in overlay DB dummyPath=$(nix-store --store "$storeB" --add ../dummy) +# Add something to the lower store that will not be propagated to overlay DB +lowerOnlyPath=$(addTextToStore "$storeA" lower-only "Only in lower store") + # Verify should be successful at this point nix-store --store "$storeB" --verify --check-contents -# Now delete one of the derivation inputs in the lower store +# Make a backup so we can repair later +backupStore="$storeVolume/backup" +mkdir "$backupStore" +tar -cC "$storeBRoot" nix | tar -xC "$backupStore" + + +## Deliberately corrupt store paths + +# Delete one of the derivation inputs in the lower store inputDrvFullPath=$(find "$storeA" -name "*-hermetic-input-1.drv") inputDrvPath=${inputDrvFullPath/*\/nix\/store\///nix/store/} rm -v "$inputDrvFullPath" -# And truncate the contents of dummy file in lower store +# Truncate the contents of dummy file in lower store find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; -# Verify should fail with the messages about missing input and modified dummy file -verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents --repair) +# Also truncate the file that only exists in lower store +truncate -s 0 "$storeA/$lowerOnlyPath" + + +## Now test that verify and repair work as expected + +# Verify overlay store without attempting to repair it +verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents) <<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!" <<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash" -<<<"$verifyOutput" grepQuiet "store does not support --verify --repair" +<<<"$verifyOutput" expectStderr 1 grepQuiet "$lowerOnlyPath" # Expect no error for corrupted lower-only path + +# Attempt to repair using backup +addConfig "substituters = $backupStore" +repairOutput=$(nix-store --store "$storeB" --verify --check-contents --repair 2>&1) +<<<"$repairOutput" grepQuiet "copying path '$inputDrvPath'" +<<<"$repairOutput" grepQuiet "copying path '$dummyPath'" From 9ef0a9e8aa2e27991434c104ad73b1b95b241f08 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 25 Jul 2023 10:28:11 -0400 Subject: [PATCH 086/211] Fix hard linking issue causing overlay fs copy-ups --- src/libstore/build/local-derivation-goal.cc | 7 ++++--- src/libstore/local-fs-store.hh | 10 ++++++++++ src/libstore/local-overlay-store.cc | 7 +++++++ src/libstore/local-overlay-store.hh | 2 ++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b7a27490c..3bc88ee86 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -386,8 +386,9 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() #if __linux__ -static void linkOrCopy(const Path & from, const Path & to) +static void linkOrCopy(LocalFSStore & store, const StorePath & from_, const Path & to) { + auto from = store.toRealPathForHardLink(from_); if (link(from.c_str(), to.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), which is quite possible after a @@ -712,7 +713,7 @@ void LocalDerivationGoal::startBuilder() if (S_ISDIR(lstat(r).st_mode)) dirsInChroot.insert_or_assign(p, r); else - linkOrCopy(r, chrootRootDir + p); + linkOrCopy(getLocalStore(), i, chrootRootDir + p); } /* If we're repairing, checking or rebuilding part of a @@ -1574,7 +1575,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); } else - linkOrCopy(source, target); + linkOrCopy(getLocalStore(), path, target); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 488109501..19858f5c8 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -73,6 +73,16 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } + /** + * If the real path is hardlinked with something else, we might + * prefer to refer to the other path instead. This is the case with + * overlayfs, for example. + */ + virtual Path toRealPathForHardLink(const StorePath & storePath) + { + return Store::toRealPath(storePath); + } + std::optional getBuildLogExact(const StorePath & path) override; }; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 700a5227b..47d09dc75 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -209,6 +209,13 @@ void LocalOverlayStore::optimiseStore() } } +Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) +{ + return lowerStore->isValidPath(path) + ? lowerStore->Store::toRealPath(path) + : Store::toRealPath(path); +} + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 349d9e6ed..64e2ef488 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -114,6 +114,8 @@ private: void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; void optimiseStore() override; + + Path toRealPathForHardLink(const StorePath & storePath) override; }; } From 19c43c5d787c113053ef651c3d22fdfde61075e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 25 Jul 2023 11:44:39 -0400 Subject: [PATCH 087/211] Write test for deleting objects referenced from below Currently fails, as expected. --- tests/hermetic.nix | 4 +-- tests/overlay-local-store/delete-inner.sh | 39 +++++++++++++++++++++++ tests/overlay-local-store/delete.sh | 5 +++ tests/overlay-local-store/local.mk | 1 + 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tests/overlay-local-store/delete-inner.sh create mode 100755 tests/overlay-local-store/delete.sh diff --git a/tests/hermetic.nix b/tests/hermetic.nix index 4c9d7a51f..c4fbbfa14 100644 --- a/tests/hermetic.nix +++ b/tests/hermetic.nix @@ -37,7 +37,7 @@ let buildCommand = '' echo hi-input3 read x < ${input2} - echo $x BAZ > $out + echo ${input2} $x BAZ > $out ''; }; @@ -51,6 +51,6 @@ in '' read x < ${input1} read y < ${input3} - echo "$x $y" > $out + echo ${input1} ${input3} "$x $y" > $out ''; } diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-inner.sh new file mode 100644 index 000000000..0702d1693 --- /dev/null +++ b/tests/overlay-local-store/delete-inner.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +export NIX_REMOTE="$storeB" +stateB="$storeBRoot/nix/var/nix" +hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2) +input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input1 -j0) +input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input2 -j0) +input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input3 -j0) + +# Can't delete because referenced +expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input3 | grepQuiet "Cannot delete path" + +# These same paths are referenced in the lower layer (by the seed 1 +# build done in `initLowerStore`). +expectStderr 1 nix-store --store "$storeA" --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --store "$storeA" --delete $input3 | grepQuiet "Cannot delete path" + +# Can delete +nix-store --delete $hermetic + +# Now unreferenced in upper layer, can delete +nix-store --delete $input3 +nix-store --delete $input2 diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete.sh new file mode 100755 index 000000000..79d67da71 --- /dev/null +++ b/tests/overlay-local-store/delete.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./delete-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 1f8de8f27..c9cb0b7f2 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,6 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ + $(d)/delete.sh \ $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh From 07b34edc449f25e9b4062d7f6d90dad1f6c71760 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 25 Jul 2023 17:09:23 -0400 Subject: [PATCH 088/211] Fix deletion test Lower layer references are ignored for deleting just in the upper layer. --- src/libstore/gc.cc | 2 +- src/libstore/local-overlay-store.cc | 8 ++++++++ src/libstore/local-overlay-store.hh | 9 +++++++++ src/libstore/local-store.hh | 12 ++++++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ea2868c58..8f5ce0994 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -741,7 +741,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto i = referrersCache.find(*path); if (i == referrersCache.end()) { StorePathSet referrers; - queryReferrers(*path, referrers); + queryGCReferrers(*path, referrers); referrersCache.emplace(*path, std::move(referrers)); i = referrersCache.find(*path); } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 47d09dc75..e8ca8074e 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -138,6 +138,12 @@ void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & re } +void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & referrers) +{ + LocalStore::queryReferrers(path, referrers); +} + + StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) { auto res = LocalStore::queryValidDerivers(path); @@ -188,6 +194,7 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) } } + void LocalOverlayStore::optimiseStore() { Activity act(*logger, actOptimiseStore); @@ -216,6 +223,7 @@ Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) : Store::toRealPath(path); } + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 64e2ef488..21658c6ab 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -115,7 +115,16 @@ private: void optimiseStore() override; + /** + * For lower-store paths, we used the lower store location. This avoids the + * wasteful "copying up" that would otherwise happen. + */ Path toRealPathForHardLink(const StorePath & storePath) override; + + /** + * Deletion only effects the upper layer, so we ignore lower-layer referrers. + */ + void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override; }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9a44722d4..c049de02e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -230,6 +230,18 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; + /** + * Called by `collectGarbage` to trace in reverse. + * + * Using this rather than `queryReferrers` directly allows us to + * fine-tune which referrers we consider for garbage collection; + * some store implementations take advantage of this. + */ + virtual void queryGCReferrers(const StorePath & path, StorePathSet & referrers) + { + return queryReferrers(path, referrers); + } + /** * Called by `collectGarbage` to recursively delete a path. * The default implementation simply calls `deletePath`, but it can be From b0877ad3c90a806e70cac3e7f6e5c26151239bb8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 26 Jul 2023 09:29:04 -0400 Subject: [PATCH 089/211] Give test a more specific name --- .../{delete-inner.sh => delete-refs-inner.sh} | 0 tests/overlay-local-store/{delete.sh => delete-refs.sh} | 2 +- tests/overlay-local-store/local.mk | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/overlay-local-store/{delete-inner.sh => delete-refs-inner.sh} (100%) rename tests/overlay-local-store/{delete.sh => delete-refs.sh} (58%) diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-refs-inner.sh similarity index 100% rename from tests/overlay-local-store/delete-inner.sh rename to tests/overlay-local-store/delete-refs-inner.sh diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete-refs.sh similarity index 58% rename from tests/overlay-local-store/delete.sh rename to tests/overlay-local-store/delete-refs.sh index 79d67da71..942d7fbdc 100755 --- a/tests/overlay-local-store/delete.sh +++ b/tests/overlay-local-store/delete-refs.sh @@ -2,4 +2,4 @@ source common.sh requireEnvironment setupConfig -execUnshare ./delete-inner.sh +execUnshare ./delete-refs-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index c9cb0b7f2..8bfb22ae7 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,7 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ - $(d)/delete.sh \ + $(d)/delete-refs.sh \ $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh From d9688ba70881f1735cf36161244a9a6b2acf4d48 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 11:31:26 +0100 Subject: [PATCH 090/211] Add new remount-hook store parameter. --- src/libstore/local-overlay-store.hh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 64e2ef488..f489c6ed6 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -41,6 +41,13 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig default, but can be disabled if needed. )"}; + const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", + R"( + Script or program to run when overlay filesystem needs remounting. + + TODO: Document this in more detail. + )"}; + const std::string name() override { return "Experimental Local Overlay Store"; } std::string doc() override From cc6f8aa91a1716ed1801f51fb3a6a1fa468f338f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 12:33:26 +0100 Subject: [PATCH 091/211] Test that delete works for duplicate file edge case. --- tests/overlay-local-store/delete-inner.sh | 38 +++++++++++++++++++++++ tests/overlay-local-store/delete.sh | 5 +++ tests/overlay-local-store/local.mk | 3 +- 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/overlay-local-store/delete-inner.sh create mode 100644 tests/overlay-local-store/delete.sh diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-inner.sh new file mode 100644 index 000000000..20b39c7fa --- /dev/null +++ b/tests/overlay-local-store/delete-inner.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +# Add to overlay before lower to ensure file is duplicated +upperPath=$(nix-store --store "$storeB" --add delete.sh) +lowerPath=$(nix-store --store "$storeA" --add delete.sh) +[[ "$upperPath" = "$lowerPath" ]] + +# Check there really are two files with different inodes +upperInode=$(stat -c %i "$storeBRoot/$upperPath") +lowerInode=$(stat -c %i "$storeA/$lowerPath") +[[ "$upperInode" != "$lowerInode" ]] + +# Now delete file via the overlay store +nix-store --store "$storeB" --delete "$upperPath" + +# Check there is no longer a file in upper layer +expect 1 stat "$storeBTop/${upperPath##/nix/store/}" + +# Check that overlay file is now the one in lower layer +upperInode=$(stat -c %i "$storeBRoot/$upperPath") +lowerInode=$(stat -c %i "$storeA/$lowerPath") +[[ "$upperInode" = "$lowerInode" ]] diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete.sh new file mode 100644 index 000000000..79d67da71 --- /dev/null +++ b/tests/overlay-local-store/delete.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./delete-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 1f8de8f27..9233462a3 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -6,6 +6,7 @@ overlay-local-store-tests := \ $(d)/add-lower.sh \ $(d)/gc.sh \ $(d)/verify.sh \ - $(d)/optimise.sh + $(d)/optimise.sh \ + $(d)/delete.sh install-tests-groups += overlay-local-store From 11c493f8fa88a81645318844077392020618887f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 13:21:38 +0100 Subject: [PATCH 092/211] Avoid creating whiteout for duplicate store paths. --- src/libstore/local-overlay-store.cc | 22 ++++++++++++++++++++-- src/libstore/local-overlay-store.hh | 2 ++ tests/overlay-local-store/delete-inner.sh | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 47d09dc75..38c40fbad 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -183,11 +183,27 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) warn("local-overlay: unexpected gc path '%s' ", path); return; } - if (pathExists(toUpperPath({path.substr(mergedDir.length())}))) { - LocalStore::deleteGCPath(path, bytesFreed); + + StorePath storePath = {path.substr(mergedDir.length())}; + auto upperPath = toUpperPath(storePath); + + if (pathExists(upperPath)) { + std::cerr << " upper exists" << std::endl; + if (lowerStore->isValidPath(storePath)) { + std::cerr << " lower exists" << std::endl; + // Path also exists in lower store. + // We must delete via upper layer to avoid creating a whiteout. + deletePath(upperPath, bytesFreed); + _remountRequired = true; + } else { + // Path does not exist in lower store. + // So we can delete via overlayfs and not need to remount. + LocalStore::deleteGCPath(path, bytesFreed); + } } } + void LocalOverlayStore::optimiseStore() { Activity act(*logger, actOptimiseStore); @@ -209,6 +225,7 @@ void LocalOverlayStore::optimiseStore() } } + Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) { return lowerStore->isValidPath(path) @@ -216,6 +233,7 @@ Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) : Store::toRealPath(path); } + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index f489c6ed6..ffb310b7b 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -123,6 +123,8 @@ private: void optimiseStore() override; Path toRealPathForHardLink(const StorePath & storePath) override; + + bool _remountRequired = false; }; } diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-inner.sh index 20b39c7fa..051b59d5f 100644 --- a/tests/overlay-local-store/delete-inner.sh +++ b/tests/overlay-local-store/delete-inner.sh @@ -28,6 +28,7 @@ lowerInode=$(stat -c %i "$storeA/$lowerPath") # Now delete file via the overlay store nix-store --store "$storeB" --delete "$upperPath" +remountOverlayfs # Check there is no longer a file in upper layer expect 1 stat "$storeBTop/${upperPath##/nix/store/}" From 33ebae75ca4a7fd81585928b36a1cc18995f8212 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 13:29:31 +0100 Subject: [PATCH 093/211] Reuse deletion logic for optimiseStore and rename method. --- src/libstore/gc.cc | 2 +- src/libstore/local-overlay-store.cc | 7 ++++--- src/libstore/local-overlay-store.hh | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ea2868c58..a909d063b 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -653,7 +653,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) results.paths.insert(path); uint64_t bytesFreed; - deleteGCPath(realPath, bytesFreed); + deleteStorePath(realPath, bytesFreed); results.bytesFreed += bytesFreed; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 38c40fbad..8b89f68bc 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -176,7 +176,7 @@ void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) } -void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { auto mergedDir = realStoreDir.get() + "/"; if (path.substr(0, mergedDir.length()) != mergedDir) { @@ -198,7 +198,7 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) } else { // Path does not exist in lower store. // So we can delete via overlayfs and not need to remount. - LocalStore::deleteGCPath(path, bytesFreed); + LocalStore::deleteStorePath(path, bytesFreed); } } } @@ -217,8 +217,9 @@ void LocalOverlayStore::optimiseStore() for (auto & path : paths) { if (lowerStore->isValidPath(path)) { + uint64_t bytesFreed = 0; // Deduplicate store path - deletePath(toUpperPath(path)); + deleteStorePath(Store::toRealPath(path), bytesFreed); } done++; act.progress(done, paths.size()); diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index ffb310b7b..d45f6bfc5 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -118,7 +118,7 @@ private: void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; - void deleteGCPath(const Path & path, uint64_t & bytesFreed) override; + void deleteStorePath(const Path & path, uint64_t & bytesFreed) override; void optimiseStore() override; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 21acb3c38..2c18867f5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -457,7 +457,7 @@ AutoCloseFD LocalStore::openGCLock() } -void LocalStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) +void LocalStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { deletePath(path, bytesFreed); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9a44722d4..9146f27a5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -235,7 +235,7 @@ public: * The default implementation simply calls `deletePath`, but it can be * overridden by stores that wish to provide their own deletion behaviour. */ - virtual void deleteGCPath(const Path & path, uint64_t & bytesFreed); + virtual void deleteStorePath(const Path & path, uint64_t & bytesFreed); /** * Optimise the disk space usage of the Nix store by hard-linking From ed1428692409f94cd5553b66c9f5e6aa3c048e86 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:05:54 +0100 Subject: [PATCH 094/211] Invoke remount-hook program when necessary. --- src/libstore/local-overlay-store.cc | 22 ++++++++++++++++++++++ src/libstore/local-overlay-store.hh | 4 ++++ tests/overlay-local-store/delete-inner.sh | 3 +-- tests/overlay-local-store/remount.sh | 2 ++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 tests/overlay-local-store/remount.sh diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 8b89f68bc..76c3adf5f 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -176,6 +176,14 @@ void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) } +void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + LocalStore::collectGarbage(options, results); + + remountIfNecessary(); +} + + void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { auto mergedDir = realStoreDir.get() + "/"; @@ -224,6 +232,8 @@ void LocalOverlayStore::optimiseStore() done++; act.progress(done, paths.size()); } + + remountIfNecessary(); } @@ -235,6 +245,18 @@ Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) } +void LocalOverlayStore::remountIfNecessary() +{ + if (remountHook.get().empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); + } else { + runProgram(remountHook, false, {realStoreDir}); + } + + _remountRequired = false; +} + + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index d45f6bfc5..a26392523 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -118,12 +118,16 @@ private: void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; + void collectGarbage(const GCOptions & options, GCResults & results) override; + void deleteStorePath(const Path & path, uint64_t & bytesFreed) override; void optimiseStore() override; Path toRealPathForHardLink(const StorePath & storePath) override; + void remountIfNecessary(); + bool _remountRequired = false; }; diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-inner.sh index 051b59d5f..f3878f657 100644 --- a/tests/overlay-local-store/delete-inner.sh +++ b/tests/overlay-local-store/delete-inner.sh @@ -27,8 +27,7 @@ lowerInode=$(stat -c %i "$storeA/$lowerPath") [[ "$upperInode" != "$lowerInode" ]] # Now delete file via the overlay store -nix-store --store "$storeB" --delete "$upperPath" -remountOverlayfs +nix-store --store "$storeB&remount-hook=$PWD/remount.sh" --delete "$upperPath" # Check there is no longer a file in upper layer expect 1 stat "$storeBTop/${upperPath##/nix/store/}" diff --git a/tests/overlay-local-store/remount.sh b/tests/overlay-local-store/remount.sh new file mode 100755 index 000000000..ee91c6b43 --- /dev/null +++ b/tests/overlay-local-store/remount.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +mount -o remount "$1" From 6da05c0a11365c8cd138c15600ec966e9c2b5940 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:29:36 +0100 Subject: [PATCH 095/211] Rename test to delete-duplicate. --- .../{delete-inner.sh => delete-duplicate-inner.sh} | 4 ++-- tests/overlay-local-store/{delete.sh => delete-duplicate.sh} | 2 +- tests/overlay-local-store/local.mk | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename tests/overlay-local-store/{delete-inner.sh => delete-duplicate-inner.sh} (87%) rename tests/overlay-local-store/{delete.sh => delete-duplicate.sh} (55%) diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-duplicate-inner.sh similarity index 87% rename from tests/overlay-local-store/delete-inner.sh rename to tests/overlay-local-store/delete-duplicate-inner.sh index f3878f657..6053451d8 100644 --- a/tests/overlay-local-store/delete-inner.sh +++ b/tests/overlay-local-store/delete-duplicate-inner.sh @@ -17,8 +17,8 @@ initLowerStore mountOverlayfs # Add to overlay before lower to ensure file is duplicated -upperPath=$(nix-store --store "$storeB" --add delete.sh) -lowerPath=$(nix-store --store "$storeA" --add delete.sh) +upperPath=$(nix-store --store "$storeB" --add delete-duplicate.sh) +lowerPath=$(nix-store --store "$storeA" --add delete-duplicate.sh) [[ "$upperPath" = "$lowerPath" ]] # Check there really are two files with different inodes diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete-duplicate.sh similarity index 55% rename from tests/overlay-local-store/delete.sh rename to tests/overlay-local-store/delete-duplicate.sh index 79d67da71..0c0b1a3b2 100644 --- a/tests/overlay-local-store/delete.sh +++ b/tests/overlay-local-store/delete-duplicate.sh @@ -2,4 +2,4 @@ source common.sh requireEnvironment setupConfig -execUnshare ./delete-inner.sh +execUnshare ./delete-duplicate-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 9233462a3..7de46ab02 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -7,6 +7,6 @@ overlay-local-store-tests := \ $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh \ - $(d)/delete.sh + $(d)/delete-duplicate.sh install-tests-groups += overlay-local-store From 5744a500d66214f6d833100924a2f7362b7b82a7 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:33:47 +0100 Subject: [PATCH 096/211] Use debug instead of writing directly to stderr. --- src/libstore/local-overlay-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 76c3adf5f..dfd2f0d34 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -196,9 +196,9 @@ void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed auto upperPath = toUpperPath(storePath); if (pathExists(upperPath)) { - std::cerr << " upper exists" << std::endl; + debug("upper exists: %s", path); if (lowerStore->isValidPath(storePath)) { - std::cerr << " lower exists" << std::endl; + debug("lower exists: %s", storePath.to_string()); // Path also exists in lower store. // We must delete via upper layer to avoid creating a whiteout. deletePath(upperPath, bytesFreed); From ca1a108dad93f00f18929c02bf29bbe5347035ac Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:37:35 +0100 Subject: [PATCH 097/211] Update tests/overlay-local-store/remount.sh Co-authored-by: John Ericson --- tests/overlay-local-store/remount.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/overlay-local-store/remount.sh b/tests/overlay-local-store/remount.sh index ee91c6b43..0b06debb5 100755 --- a/tests/overlay-local-store/remount.sh +++ b/tests/overlay-local-store/remount.sh @@ -1,2 +1,2 @@ -#!/usr/bin/env bash +#!/bin/sh mount -o remount "$1" From 3a9fe1a085edc8db3e3cd35531829ddab51bcd81 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:44:14 +0100 Subject: [PATCH 098/211] Made remountRequired atomic to avoid concurrency issues. --- src/libstore/local-overlay-store.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index a26392523..66a43d84a 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -128,7 +128,7 @@ private: void remountIfNecessary(); - bool _remountRequired = false; + std::atomic_bool _remountRequired = false; }; } From c2d54496a0c8611a7008181e2dc4fc57d520752f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Wed, 26 Jul 2023 14:47:44 +0100 Subject: [PATCH 099/211] Forgot to check flag and early out. --- src/libstore/local-overlay-store.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index dfd2f0d34..04a1717cd 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -247,6 +247,8 @@ Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) void LocalOverlayStore::remountIfNecessary() { + if (!_remountRequired) return; + if (remountHook.get().empty()) { warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); } else { From 50ce8d15eb5fed11080a3b94991c55e880c22252 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Fri, 28 Jul 2023 10:11:45 +0100 Subject: [PATCH 100/211] Preparatory refactor of LocalStore::verifyStore. --- src/libstore/local-store.cc | 51 +++++++++++++++++++++---------------- src/libstore/local-store.hh | 6 +++-- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2c18867f5..6c3584b49 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1498,24 +1498,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { printInfo("reading the Nix store..."); - bool errors = false; - /* Acquire the global GC lock to get a consistent snapshot of existing and valid paths. */ auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StringSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); - - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); - StorePathSet validPaths; - PathSet done; - for (auto & i : queryAllValidPaths()) - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + bool errors = verifyAllValidPaths(repair, validPaths); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1601,28 +1591,45 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & pathS, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) +bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) +{ + StringSet store; + for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); + + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); + + StorePathSet done; + bool errors = false; + + auto existsInStoreDir = [&](const StorePath & storePath) { + return store.count(std::string(storePath.to_string())); + }; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); + + return errors; +} + + +void LocalStore::verifyPath(const StorePath & path, std::function existsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(pathS).second) return; + if (!done.insert(path).second) return; - if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); - return; - } + auto pathS = printStorePath(path); - auto path = parseStorePath(pathS); - - if (!store.count(std::string(path.to_string()))) { + if (!existsInStoreDir(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 514ac256b..f9db43517 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -265,6 +265,8 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; + virtual bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths); + /** * Register the validity of a path, i.e., that `path` exists, that * the paths referenced by it exists, and in the case of an output @@ -333,8 +335,8 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + void verifyPath(const StorePath & path, std::function existsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); From 6a8de4c9dcd5c329f6488818517995647c577c87 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Fri, 28 Jul 2023 10:59:16 +0100 Subject: [PATCH 101/211] Avoid enumerating entire overlay store dir upfront. As an optimisation for LocalStore, we read all the store directory entries into a set. Checking for membership of this set is much faster than a stat syscall. However for LocalOverlayStore, the lower store directory is expected to contain a vast number of entries and reading them all can take a very long time. So instead of enumerating them all upfront, we call pathExists as needed. This means making stat syscalls for each store path, but the upper layer is expected to be relatively small compared to the lower store so that should be okay. --- src/libstore/local-overlay-store.cc | 16 ++++++++++++++++ src/libstore/local-overlay-store.hh | 2 ++ src/libstore/local-store.hh | 8 +++++--- tests/overlay-local-store/verify-inner.sh | 3 +++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 30008de67..4bfad6d32 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -243,6 +243,22 @@ void LocalOverlayStore::optimiseStore() } +bool LocalOverlayStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) +{ + StorePathSet done; + bool errors = false; + + auto existsInStoreDir = [&](const StorePath & storePath) { + return pathExists(realStoreDir.get() + "/" + storePath.to_string()); + }; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); + + return errors; +} + + Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) { return lowerStore->isValidPath(path) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 1c1d54a8f..21af396a6 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -124,6 +124,8 @@ private: void optimiseStore() override; + bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) override; + /** * For lower-store paths, we used the lower store location. This avoids the * wasteful "copying up" that would otherwise happen. diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f9db43517..322d27932 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -312,6 +312,11 @@ public: std::optional getVersion() override; +protected: + + void verifyPath(const StorePath & path, std::function existsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + private: /** @@ -335,9 +340,6 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const StorePath & path, std::function existsInStoreDir, - StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); - std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); void updatePathInfo(State & state, const ValidPathInfo & info); diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/overlay-local-store/verify-inner.sh index de40c3d05..64d3b63c1 100755 --- a/tests/overlay-local-store/verify-inner.sh +++ b/tests/overlay-local-store/verify-inner.sh @@ -50,6 +50,9 @@ find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; # Also truncate the file that only exists in lower store truncate -s 0 "$storeA/$lowerOnlyPath" +# Ensure overlayfs is synchronised +remountOverlayfs + ## Now test that verify and repair work as expected From 1255866e16bbbf2ac93ade99ae5ef2f68f64d8c6 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Fri, 28 Jul 2023 12:23:42 +0100 Subject: [PATCH 102/211] Update src/libstore/local-overlay-store.hh Co-authored-by: John Ericson --- src/libstore/local-overlay-store.hh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 21af396a6..b91f68eea 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -124,6 +124,13 @@ private: void optimiseStore() override; + /** + * Check all paths registered in the upper DB. + * + * Note that this includes store objects that reside in either overlayfs layer; just enumerating the contents of the upper layer would skip them. + * + * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, and also the observation that anything not in the upper db the overlayfs doesn't yet care about. + */ bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) override; /** From c409a753dbc6318f5454f058dbf55786327035b7 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 1 Aug 2023 11:11:16 +0100 Subject: [PATCH 103/211] Fix new lines in comment. --- src/libstore/local-overlay-store.hh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index b91f68eea..874f3b779 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -127,9 +127,11 @@ private: /** * Check all paths registered in the upper DB. * - * Note that this includes store objects that reside in either overlayfs layer; just enumerating the contents of the upper layer would skip them. + * Note that this includes store objects that reside in either overlayfs layer; + * just enumerating the contents of the upper layer would skip them. * - * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, and also the observation that anything not in the upper db the overlayfs doesn't yet care about. + * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, + * and also the observation that anything not in the upper db the overlayfs doesn't yet care about. */ bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) override; From c712369ec597c706f985f2f998215f5ab913d61a Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 1 Aug 2023 11:55:55 +0100 Subject: [PATCH 104/211] Document remount-hook store parameter. --- src/libstore/local-overlay-store.hh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 874f3b779..c0fa0ffa7 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -43,9 +43,15 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", R"( - Script or program to run when overlay filesystem needs remounting. + Script or other executable to run when overlay filesystem needs remounting. - TODO: Document this in more detail. + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. )"}; const std::string name() override { return "Experimental Local Overlay Store"; } From 19164cf727030dc75c11533f11dc84645658b7c3 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 1 Aug 2023 12:49:46 +0100 Subject: [PATCH 105/211] Test that remounting fixes 'stale file handle' errors. --- tests/overlay-local-store/local.mk | 3 +- .../stale-file-handle-inner.sh | 47 +++++++++++++++++++ .../overlay-local-store/stale-file-handle.sh | 5 ++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100755 tests/overlay-local-store/stale-file-handle-inner.sh create mode 100755 tests/overlay-local-store/stale-file-handle.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 19ae6a3d0..34056683d 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -8,6 +8,7 @@ overlay-local-store-tests := \ $(d)/delete-duplicate.sh \ $(d)/gc.sh \ $(d)/verify.sh \ - $(d)/optimise.sh + $(d)/optimise.sh \ + $(d)/stale-file-handle.sh install-tests-groups += overlay-local-store diff --git a/tests/overlay-local-store/stale-file-handle-inner.sh b/tests/overlay-local-store/stale-file-handle-inner.sh new file mode 100755 index 000000000..86e39fa40 --- /dev/null +++ b/tests/overlay-local-store/stale-file-handle-inner.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +buildInStore () { + nix-build --store "$1" ../hermetic.nix --arg busybox "$busybox" --arg seed 1 --no-out-link +} + +triggerStaleFileHandle () { + # Arrange it so there are duplicate paths + nix-store --store "$storeA" --gc # Clear lower store + buildInStore "$storeB" # Build into upper layer first + buildInStore "$storeA" # Then build in lower store + + # Duplicate paths mean GC will have to delete via upper layer + nix-store --store "$storeB" --gc + + # Clear lower store again to force building in upper layer + nix-store --store "$storeA" --gc + + # Now attempting to build in upper layer will fail + buildInStore "$storeB" +} + +# Without remounting, we should encounter errors +expectStderr 1 triggerStaleFileHandle | grepQuiet 'Stale file handle' + +# Configure remount-hook and reset OverlayFS +storeB="$storeB&remount-hook=$PWD/remount.sh" +remountOverlayfs + +# Now it should succeed +triggerStaleFileHandle diff --git a/tests/overlay-local-store/stale-file-handle.sh b/tests/overlay-local-store/stale-file-handle.sh new file mode 100755 index 000000000..5e75628ca --- /dev/null +++ b/tests/overlay-local-store/stale-file-handle.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./stale-file-handle-inner.sh From 6b297e5895e62d91963be49e5d301604d0b8fd09 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 2 Aug 2023 14:38:22 -0400 Subject: [PATCH 106/211] Make `verifyAllValidPaths` more functional return map rather than mutate one passed in by reference --- src/libstore/local-overlay-store.cc | 8 +++++--- src/libstore/local-overlay-store.hh | 4 ++-- src/libstore/local-store.cc | 12 ++++++------ src/libstore/local-store.hh | 7 ++++++- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 4bfad6d32..4ba98a02d 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -243,19 +243,21 @@ void LocalOverlayStore::optimiseStore() } -bool LocalOverlayStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) +std::pair LocalOverlayStore::verifyAllValidPaths(RepairFlag repair) { StorePathSet done; - bool errors = false; auto existsInStoreDir = [&](const StorePath & storePath) { return pathExists(realStoreDir.get() + "/" + storePath.to_string()); }; + bool errors = false; + StorePathSet validPaths; + for (auto & i : queryAllValidPaths()) verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); - return errors; + return { errors, validPaths }; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index c0fa0ffa7..40ee75bca 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -135,11 +135,11 @@ private: * * Note that this includes store objects that reside in either overlayfs layer; * just enumerating the contents of the upper layer would skip them. - * + * * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, * and also the observation that anything not in the upper db the overlayfs doesn't yet care about. */ - bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) override; + std::pair verifyAllValidPaths(RepairFlag repair) override; /** * For lower-store paths, we used the lower store location. This avoids the diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2e662ad66..42347d80a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1503,9 +1503,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StorePathSet validPaths; - - bool errors = verifyAllValidPaths(repair, validPaths); + auto [errors, validPaths] = verifyAllValidPaths(repair); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1591,7 +1589,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) +std::pair LocalStore::verifyAllValidPaths(RepairFlag repair) { StorePathSet storePathsInStoreDir; /* Why aren't we using `queryAllValidPaths`? Because that would @@ -1613,16 +1611,18 @@ bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPath printInfo("checking path existence..."); StorePathSet done; - bool errors = false; auto existsInStoreDir = [&](const StorePath & storePath) { return storePathsInStoreDir.count(storePath); }; + bool errors = false; + StorePathSet validPaths; + for (auto & i : queryAllValidPaths()) verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); - return errors; + return { errors, validPaths }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 322d27932..88549eea5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -265,7 +265,12 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; - virtual bool verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths); + /** + * @return A pair of whether any errors were encountered, and a set of + * (so-far) valid paths. The store objects pointed to by those paths are + * suitable for further validation checking. + */ + virtual std::pair verifyAllValidPaths(RepairFlag repair); /** * Register the validity of a path, i.e., that `path` exists, that From 4b9a6218125c23e95b7397069f6fc2796fb70fd2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 2 Aug 2023 20:30:42 -0400 Subject: [PATCH 107/211] Guard the local overlay store behind an experimental feature --- src/libstore/local-overlay-store.hh | 5 +++++ src/libstore/store-api.cc | 5 ++++- src/libutil/experimental-features.cc | 9 ++++++++- src/libutil/experimental-features.hh | 1 + tests/overlay-local-store/common.sh | 2 +- 5 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 40ee75bca..44f0c77c8 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -56,6 +56,11 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const std::string name() override { return "Experimental Local Overlay Store"; } + std::optional experimentalFeature() const override + { + return ExperimentalFeature::LocalOverlayStore; + } + std::string doc() override { return diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1fbf8995b..1863a0876 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1444,7 +1444,9 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para } else if (uri == "local") { return std::make_shared(params); } else if (uri == "local-overlay") { - return std::make_shared(params); + auto store = std::make_shared(params); + experimentalFeatureSettings.require(store->experimentalFeature()); + return store; } else if (isNonUriPath(uri)) { Store::Params params2 = params; params2["root"] = absPath(uri); @@ -1512,6 +1514,7 @@ ref openStore(const std::string & uri_, params.insert(uriParams.begin(), uriParams.end()); if (auto store = openFromNonUri(uri, params)) { + experimentalFeatureSettings.require(store->experimentalFeature()); store->warnUnknownSettings(); return ref(store); } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 7c4112d32..422d18522 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -228,6 +228,13 @@ constexpr std::array xpFeatureDetails = {{ Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. )", }, + { + .tag = Xp::LocalOverlayStore, + .name = "local-overlay-store", + .description = R"( + Allow the use of [local overlay store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-overlay-store). + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index faf2e9398..c044a858e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -32,6 +32,7 @@ enum struct ExperimentalFeature DynamicDerivations, ParseTomlTimestamps, ReadOnlyLocalStore, + LocalOverlayStore, }; /** diff --git a/tests/overlay-local-store/common.sh b/tests/overlay-local-store/common.sh index 6bb5bc391..2d614b140 100644 --- a/tests/overlay-local-store/common.sh +++ b/tests/overlay-local-store/common.sh @@ -16,7 +16,7 @@ setupConfig () { addConfig "build-users-group = " } - +enableFeatures "local-overlay-store" storeDirs () { # Attempt to create store dirs on tmpfs volume. From 2556c4d7531c1303ec3e5db5d5dd1b3a9b91a3b4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 2 Aug 2023 20:32:45 -0400 Subject: [PATCH 108/211] Rename test group `overlay-local-store` -> `local-overlay-store` Makes it match the store name (`local-overlay`) and experimental feature name (`local-overlay-store`)._ --- Makefile | 2 +- .../add-lower-inner.sh | 0 .../{overlay-local-store => local-overlay-store}/add-lower.sh | 0 .../{overlay-local-store => local-overlay-store}/bad-uris.sh | 0 .../build-inner.sh | 0 tests/{overlay-local-store => local-overlay-store}/build.sh | 0 .../check-post-init-inner.sh | 0 .../check-post-init.sh | 0 tests/{overlay-local-store => local-overlay-store}/common.sh | 0 .../delete-duplicate-inner.sh | 0 .../delete-duplicate.sh | 0 .../delete-refs-inner.sh | 0 .../delete-refs.sh | 0 .../{overlay-local-store => local-overlay-store}/gc-inner.sh | 0 tests/{overlay-local-store => local-overlay-store}/gc.sh | 0 tests/{overlay-local-store => local-overlay-store}/local.mk | 4 ++-- .../optimise-inner.sh | 0 .../{overlay-local-store => local-overlay-store}/optimise.sh | 0 .../redundant-add-inner.sh | 0 .../redundant-add.sh | 0 tests/{overlay-local-store => local-overlay-store}/remount.sh | 0 .../stale-file-handle-inner.sh | 0 .../stale-file-handle.sh | 0 .../verify-inner.sh | 0 tests/{overlay-local-store => local-overlay-store}/verify.sh | 0 25 files changed, 3 insertions(+), 3 deletions(-) rename tests/{overlay-local-store => local-overlay-store}/add-lower-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/add-lower.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/bad-uris.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/build-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/build.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/check-post-init-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/check-post-init.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/common.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/delete-duplicate-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/delete-duplicate.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/delete-refs-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/delete-refs.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/gc-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/gc.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/local.mk (77%) rename tests/{overlay-local-store => local-overlay-store}/optimise-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/optimise.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/redundant-add-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/redundant-add.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/remount.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/stale-file-handle-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/stale-file-handle.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/verify-inner.sh (100%) rename tests/{overlay-local-store => local-overlay-store}/verify.sh (100%) diff --git a/Makefile b/Makefile index 715830811..ea15b03db 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ makefiles += \ tests/local.mk \ tests/ca/local.mk \ tests/dyn-drv/local.mk \ - tests/overlay-local-store/local.mk \ + tests/local-overlay-store/local.mk \ tests/test-libstoreconsumer/local.mk \ tests/plugins/local.mk else diff --git a/tests/overlay-local-store/add-lower-inner.sh b/tests/local-overlay-store/add-lower-inner.sh similarity index 100% rename from tests/overlay-local-store/add-lower-inner.sh rename to tests/local-overlay-store/add-lower-inner.sh diff --git a/tests/overlay-local-store/add-lower.sh b/tests/local-overlay-store/add-lower.sh similarity index 100% rename from tests/overlay-local-store/add-lower.sh rename to tests/local-overlay-store/add-lower.sh diff --git a/tests/overlay-local-store/bad-uris.sh b/tests/local-overlay-store/bad-uris.sh similarity index 100% rename from tests/overlay-local-store/bad-uris.sh rename to tests/local-overlay-store/bad-uris.sh diff --git a/tests/overlay-local-store/build-inner.sh b/tests/local-overlay-store/build-inner.sh similarity index 100% rename from tests/overlay-local-store/build-inner.sh rename to tests/local-overlay-store/build-inner.sh diff --git a/tests/overlay-local-store/build.sh b/tests/local-overlay-store/build.sh similarity index 100% rename from tests/overlay-local-store/build.sh rename to tests/local-overlay-store/build.sh diff --git a/tests/overlay-local-store/check-post-init-inner.sh b/tests/local-overlay-store/check-post-init-inner.sh similarity index 100% rename from tests/overlay-local-store/check-post-init-inner.sh rename to tests/local-overlay-store/check-post-init-inner.sh diff --git a/tests/overlay-local-store/check-post-init.sh b/tests/local-overlay-store/check-post-init.sh similarity index 100% rename from tests/overlay-local-store/check-post-init.sh rename to tests/local-overlay-store/check-post-init.sh diff --git a/tests/overlay-local-store/common.sh b/tests/local-overlay-store/common.sh similarity index 100% rename from tests/overlay-local-store/common.sh rename to tests/local-overlay-store/common.sh diff --git a/tests/overlay-local-store/delete-duplicate-inner.sh b/tests/local-overlay-store/delete-duplicate-inner.sh similarity index 100% rename from tests/overlay-local-store/delete-duplicate-inner.sh rename to tests/local-overlay-store/delete-duplicate-inner.sh diff --git a/tests/overlay-local-store/delete-duplicate.sh b/tests/local-overlay-store/delete-duplicate.sh similarity index 100% rename from tests/overlay-local-store/delete-duplicate.sh rename to tests/local-overlay-store/delete-duplicate.sh diff --git a/tests/overlay-local-store/delete-refs-inner.sh b/tests/local-overlay-store/delete-refs-inner.sh similarity index 100% rename from tests/overlay-local-store/delete-refs-inner.sh rename to tests/local-overlay-store/delete-refs-inner.sh diff --git a/tests/overlay-local-store/delete-refs.sh b/tests/local-overlay-store/delete-refs.sh similarity index 100% rename from tests/overlay-local-store/delete-refs.sh rename to tests/local-overlay-store/delete-refs.sh diff --git a/tests/overlay-local-store/gc-inner.sh b/tests/local-overlay-store/gc-inner.sh similarity index 100% rename from tests/overlay-local-store/gc-inner.sh rename to tests/local-overlay-store/gc-inner.sh diff --git a/tests/overlay-local-store/gc.sh b/tests/local-overlay-store/gc.sh similarity index 100% rename from tests/overlay-local-store/gc.sh rename to tests/local-overlay-store/gc.sh diff --git a/tests/overlay-local-store/local.mk b/tests/local-overlay-store/local.mk similarity index 77% rename from tests/overlay-local-store/local.mk rename to tests/local-overlay-store/local.mk index 34056683d..6348a4423 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/local-overlay-store/local.mk @@ -1,4 +1,4 @@ -overlay-local-store-tests := \ +local-overlay-store-tests := \ $(d)/check-post-init.sh \ $(d)/redundant-add.sh \ $(d)/build.sh \ @@ -11,4 +11,4 @@ overlay-local-store-tests := \ $(d)/optimise.sh \ $(d)/stale-file-handle.sh -install-tests-groups += overlay-local-store +install-tests-groups += local-overlay-store diff --git a/tests/overlay-local-store/optimise-inner.sh b/tests/local-overlay-store/optimise-inner.sh similarity index 100% rename from tests/overlay-local-store/optimise-inner.sh rename to tests/local-overlay-store/optimise-inner.sh diff --git a/tests/overlay-local-store/optimise.sh b/tests/local-overlay-store/optimise.sh similarity index 100% rename from tests/overlay-local-store/optimise.sh rename to tests/local-overlay-store/optimise.sh diff --git a/tests/overlay-local-store/redundant-add-inner.sh b/tests/local-overlay-store/redundant-add-inner.sh similarity index 100% rename from tests/overlay-local-store/redundant-add-inner.sh rename to tests/local-overlay-store/redundant-add-inner.sh diff --git a/tests/overlay-local-store/redundant-add.sh b/tests/local-overlay-store/redundant-add.sh similarity index 100% rename from tests/overlay-local-store/redundant-add.sh rename to tests/local-overlay-store/redundant-add.sh diff --git a/tests/overlay-local-store/remount.sh b/tests/local-overlay-store/remount.sh similarity index 100% rename from tests/overlay-local-store/remount.sh rename to tests/local-overlay-store/remount.sh diff --git a/tests/overlay-local-store/stale-file-handle-inner.sh b/tests/local-overlay-store/stale-file-handle-inner.sh similarity index 100% rename from tests/overlay-local-store/stale-file-handle-inner.sh rename to tests/local-overlay-store/stale-file-handle-inner.sh diff --git a/tests/overlay-local-store/stale-file-handle.sh b/tests/local-overlay-store/stale-file-handle.sh similarity index 100% rename from tests/overlay-local-store/stale-file-handle.sh rename to tests/local-overlay-store/stale-file-handle.sh diff --git a/tests/overlay-local-store/verify-inner.sh b/tests/local-overlay-store/verify-inner.sh similarity index 100% rename from tests/overlay-local-store/verify-inner.sh rename to tests/local-overlay-store/verify-inner.sh diff --git a/tests/overlay-local-store/verify.sh b/tests/local-overlay-store/verify.sh similarity index 100% rename from tests/overlay-local-store/verify.sh rename to tests/local-overlay-store/verify.sh From 4f5b01f5cd83f1ce0d36ea8f4be8b06e9526cbe3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Aug 2023 11:59:04 -0400 Subject: [PATCH 109/211] Start to document the local-overlay store --- src/libstore/local-overlay-store.cc | 7 +++++++ src/libstore/local-overlay-store.hh | 9 +-------- src/libstore/local-overlay-store.md | 7 +++++++ 3 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 src/libstore/local-overlay-store.md diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 4ba98a02d..73902e9ba 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -5,6 +5,13 @@ namespace nix { +std::string LocalOverlayStoreConfig::doc() +{ + return + #include "local-overlay-store.md" + ; +} + Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 44f0c77c8..e3fda2f93 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -61,14 +61,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig return ExperimentalFeature::LocalOverlayStore; } - std::string doc() override - { - return - "" - // FIXME write docs - //#include "local-overlay-store.md" - ; - } + std::string doc() override; /** * Given a store path, get its location (if it is exists) in the diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md new file mode 100644 index 000000000..0b8a1786b --- /dev/null +++ b/src/libstore/local-overlay-store.md @@ -0,0 +1,7 @@ +R"( + +**Store URL format**: `local-overlay` + +This store type is a variation of the [local store](#local-store) designed to leverage overlayfs. + +)" From 4d99e407fd8c2a257304a622b93abd69149413fc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Aug 2023 12:12:21 -0400 Subject: [PATCH 110/211] Remove FIXME on why something doesn't work I now know it is due to https://github.com/llvm/llvm-project/issues/64108. The workaround is just fine and already in use in this codebase. --- src/libstore/local-overlay-store.hh | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index e3fda2f93..12e49e6d2 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -7,9 +7,6 @@ namespace nix { */ struct LocalOverlayStoreConfig : virtual LocalStoreConfig { - // FIXME why doesn't this work? - // using LocalStoreConfig::LocalStoreConfig; - LocalOverlayStoreConfig(const StringMap & params) : StoreConfig(params) , LocalFSStoreConfig(params) From 7ad16c9d1283c14df410a933eff560da258c11bf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Aug 2023 16:10:32 -0400 Subject: [PATCH 111/211] Add some docs for the local overlay store --- src/libstore/local-overlay-store.md | 49 ++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index 0b8a1786b..a0b9c3adf 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -2,6 +2,53 @@ R"( **Store URL format**: `local-overlay` -This store type is a variation of the [local store](#local-store) designed to leverage overlayfs. +This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short). +Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost [local store]. +("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) + +### Parts of a local overlay store + +The parts of a local overlay store are as follows: + +- Lower store: + + This is any store implementation that includes a store directory as part of the native operating system filesystem. + For example, this could be a [local store], [local daemon store], or even another local overlay store. + + Specified with the `lower-store` setting. + + - Lower store directory. + This is the directory used/exposed by the lower store. + + Specified with `lower-store.real` setting. + + - Lower abstract read-only metadata source. + This is abstract, just some way to read the metadata of lower store [store objects](@docroot@/glossary.md#gloss-store-object). + For example it could be a SQLite database (for the [local store]), or a socket connection (for the [local daemon store]). + +- Upper almost-store: + + This is a [local store] that by itself would appear corrupted. + But combined with everything else as part of an overlay local store, it is valid. + + - Upper layer directory. + This contains additional [store objects] + (or, strictly speaking, their [file system objects](#gloss-file-system-object)) + that the local overlay store will extend the lower store with. + + Specified with `upper-layer` setting. + + - Upper store directory + The lower store directory and upper layer directory are combined via OverlayFS to create this directory. + This contains all the store objects from each of the two directories. + + Specified with the `real` setting. + + - Upper SQLite database + This contains the metadata of all of the upper layer [store objects]: everything beyond their file system objects, and also duplicate copies of some lower layer ones. + The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects] can entirely be found in the upper layer. + This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way. + + Specified with the `state` setting, is always `${state}/db`. )" From d137002e941ffdaa46efdffbca9fdc6fbb108522 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 3 Aug 2023 17:28:18 -0400 Subject: [PATCH 112/211] Add API docs for all overridden local overlay methods These docs explain the implementation relative to the local store originals. The original declaration of virtual methods can still be consulted for proper interface-level documentation. --- src/libstore/local-overlay-store.hh | 64 +++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 12e49e6d2..8ed7516db 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -68,7 +68,10 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig }; /** - * Variation of local store using overlayfs for the store dir. + * Variation of local store using OverlayFS for the store directory. + * + * Documentation on overridden methods states how they differ from their + * `LocalStore` counterparts. */ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore { @@ -99,30 +102,77 @@ public: } private: - // Overridden methods… - + /** + * First copy up any lower store realisation with the same key, so we + * merge rather than mask it. + */ void registerDrvOutput(const Realisation & info) override; + /** + * Check lower store if upper DB does not have. + */ void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept override; + /** + * Check lower store if upper DB does not have. + * + * In addition, copy up metadata for lower store objects (and their + * closure). (I.e. Optimistically cache in the upper DB.) + */ bool isValidPathUncached(const StorePath & path) override; + /** + * Check the lower store and upper DB. + */ void queryReferrers(const StorePath & path, StorePathSet & referrers) override; + /** + * Check the lower store and upper DB. + */ StorePathSet queryValidDerivers(const StorePath & path) override; + /** + * Check lower store if upper DB does not have. + */ std::optional queryPathFromHashPart(const std::string & hashPart) override; + /** + * First copy up any lower store realisation with the same key, so we + * merge rather than mask it. + */ void registerValidPaths(const ValidPathInfos & infos) override; + /** + * Check lower store if upper DB does not have. + */ void queryRealisationUncached(const DrvOutput&, Callback> callback) noexcept override; + /** + * Call `remountIfNecessary` after collecting garbage normally. + */ void collectGarbage(const GCOptions & options, GCResults & results) override; + /** + * Check which layers the store object exists in to try to avoid + * needing to remount. + */ void deleteStorePath(const Path & path, uint64_t & bytesFreed) override; + /** + * Deduplicate by removing store objects from the upper layer that + * are now in the lower layer. + * + * This implementation will not cause duplications, but addition of + * new store objects to the lower layer can instill induce them + * (there is no way to prevent that). This cleans up those + * duplications. + * + * @note We do not yet optomise the upper layer in the normal way + * (hardlink) yet. We would like to, but it requires more + * refactoring of existing code to support this sustainably. + */ void optimiseStore() override; /** @@ -147,8 +197,16 @@ private: */ void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override; + /** + * Call the `remountHook` if we have done something such that the + * OverlayFS needed to be remounted. See that hook's user-facing + * documentation for further details. + */ void remountIfNecessary(); + /** + * State for `remountIfNecessary` + */ std::atomic_bool _remountRequired = false; }; From 6f0a95897cacf6edb8d753e19aedf2f4f1f12108 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 25 Oct 2023 14:20:40 -0400 Subject: [PATCH 113/211] Revert "Fix hard linking issue causing overlay fs copy-ups" This reverts commit 9ef0a9e8aa2e27991434c104ad73b1b95b241f08. Master now has a better solution. --- src/libstore/build/local-derivation-goal.cc | 7 +++---- src/libstore/local-fs-store.hh | 10 ---------- src/libstore/local-overlay-store.cc | 8 -------- src/libstore/local-overlay-store.hh | 6 ------ 4 files changed, 3 insertions(+), 28 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9da3afffd..40a0385af 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -388,9 +388,8 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() #if __linux__ -static void linkOrCopy(LocalFSStore & store, const StorePath & from_, const Path & to) +static void linkOrCopy(const Path & from, const Path & to) { - auto from = store.toRealPathForHardLink(from_); if (link(from.c_str(), to.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), which is quite possible after a @@ -715,7 +714,7 @@ void LocalDerivationGoal::startBuilder() if (S_ISDIR(lstat(r).st_mode)) dirsInChroot.insert_or_assign(p, r); else - linkOrCopy(getLocalStore(), i, chrootRootDir + p); + linkOrCopy(r, chrootRootDir + p); } /* If we're repairing, checking or rebuilding part of a @@ -1600,7 +1599,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); } else - linkOrCopy(getLocalStore(), path, target); + linkOrCopy(source, target); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 19858f5c8..488109501 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -73,16 +73,6 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - /** - * If the real path is hardlinked with something else, we might - * prefer to refer to the other path instead. This is the case with - * overlayfs, for example. - */ - virtual Path toRealPathForHardLink(const StorePath & storePath) - { - return Store::toRealPath(storePath); - } - std::optional getBuildLogExact(const StorePath & path) override; }; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 73902e9ba..732b4d6ce 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -268,14 +268,6 @@ std::pair LocalOverlayStore::verifyAllValidPaths(RepairFlag } -Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) -{ - return lowerStore->isValidPath(path) - ? lowerStore->Store::toRealPath(path) - : Store::toRealPath(path); -} - - void LocalOverlayStore::remountIfNecessary() { if (!_remountRequired) return; diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 8ed7516db..ac730c77c 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -186,12 +186,6 @@ private: */ std::pair verifyAllValidPaths(RepairFlag repair) override; - /** - * For lower-store paths, we used the lower store location. This avoids the - * wasteful "copying up" that would otherwise happen. - */ - Path toRealPathForHardLink(const StorePath & storePath) override; - /** * Deletion only effects the upper layer, so we ignore lower-layer referrers. */ From 250c3541bb7ffffae81e873f09f781d3b57e5935 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 25 Oct 2023 15:29:11 -0400 Subject: [PATCH 114/211] Use `local-overlay://` not `local-overlay` for store URL This is a bit uglier, but allows us to avoid an ad-hoc special case in `store-api.cc`. --- src/libstore/local-overlay-store.hh | 9 ++++++--- src/libstore/store-api.cc | 5 ----- tests/functional/local-overlay-store/bad-uris.sh | 6 +++--- tests/functional/local-overlay-store/common.sh | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index ac730c77c..060ca37c3 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -90,15 +90,18 @@ public: LocalOverlayStore(std::string scheme, std::string path, const Params & params) : LocalOverlayStore(params) { - throw UnimplementedError("LocalOverlayStore"); + if (!path.empty()) + throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params"); } static std::set uriSchemes() - { return {}; } + { + return { "local-overlay" }; + } std::string getUri() override { - return "local-overlay"; + return "local-overlay://"; } private: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 55cdd71a9..069fad420 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,7 +11,6 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" -#include "local-overlay-store.hh" // FIXME this should not be here, see TODO below on // `addMultipleToStore`. #include "worker-protocol.hh" @@ -1456,10 +1455,6 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para return std::make_shared(params); } else if (uri == "local") { return std::make_shared(params); - } else if (uri == "local-overlay") { - auto store = std::make_shared(params); - experimentalFeatureSettings.require(store->experimentalFeature()); - return store; } else if (isNonUriPath(uri)) { Store::Params params2 = params; params2["root"] = absPath(uri); diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index 462bf27eb..07b2c6f28 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -6,9 +6,9 @@ storeDirs mkdir -p $TEST_ROOT/bad_test badTestRoot=$TEST_ROOT/bad_test -storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" -storeBadLower="local-overlay?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop" -storeBadUpper="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot" +storeBadRoot="local-overlay://?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" +storeBadLower="local-overlay://?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot" declare -a storesBad=( "$storeBadRoot" "$storeBadLower" "$storeBadUpper" diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 2d614b140..bd144c925 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -29,7 +29,7 @@ storeDirs () { storeA="$storeVolume/store-a" storeBTop="$storeVolume/store-b" storeBRoot="$storeVolume/merged-store" - storeB="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop" + storeB="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop" # Creating testing directories mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} } From c30b5d8a0bc7766e55b5254c75b264597bd2935e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:18:34 -0500 Subject: [PATCH 115/211] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libstore/local-overlay-store.hh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 060ca37c3..64ed2f20a 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -24,7 +24,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", R"( - Must be used as OverlayFS upper layer for this store's store dir. + Directory containing the OverlayFS upper layer for this store's store dir. )"}; Setting checkMount{(StoreConfig*) this, true, "check-mount", @@ -60,6 +60,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig std::string doc() override; +protected: /** * Given a store path, get its location (if it is exists) in the * upper layer of the overlayfs. @@ -167,7 +168,7 @@ private: * Deduplicate by removing store objects from the upper layer that * are now in the lower layer. * - * This implementation will not cause duplications, but addition of + * Operations on a layered store will not cause duplications, but addition of * new store objects to the lower layer can instill induce them * (there is no way to prevent that). This cleans up those * duplications. From b21ee60594c637b4ea5ab98d5cc60b34a66bdde4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:28:40 -0500 Subject: [PATCH 116/211] Get rid of `verifyAllValidPaths` boolean blindness --- src/libstore/local-overlay-store.cc | 7 +++++-- src/libstore/local-overlay-store.hh | 2 +- src/libstore/local-store.cc | 7 +++++-- src/libstore/local-store.hh | 26 ++++++++++++++++++++++---- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index f657d8341..598415db8 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -252,7 +252,7 @@ void LocalOverlayStore::optimiseStore() } -std::pair LocalOverlayStore::verifyAllValidPaths(RepairFlag repair) +LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag repair) { StorePathSet done; @@ -266,7 +266,10 @@ std::pair LocalOverlayStore::verifyAllValidPaths(RepairFlag for (auto & i : queryAllValidPaths()) verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); - return { errors, validPaths }; + return { + .errors = errors, + .validPaths = validPaths, + }; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 64ed2f20a..a93b069cc 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -188,7 +188,7 @@ private: * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, * and also the observation that anything not in the upper db the overlayfs doesn't yet care about. */ - std::pair verifyAllValidPaths(RepairFlag repair) override; + VerificationResult verifyAllValidPaths(RepairFlag repair) override; /** * Deletion only effects the upper layer, so we ignore lower-layer referrers. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8561d58be..b35d59bbb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1443,7 +1443,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -std::pair LocalStore::verifyAllValidPaths(RepairFlag repair) +LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair) { StorePathSet storePathsInStoreDir; /* Why aren't we using `queryAllValidPaths`? Because that would @@ -1476,7 +1476,10 @@ std::pair LocalStore::verifyAllValidPaths(RepairFlag repair) for (auto & i : queryAllValidPaths()) verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); - return { errors, validPaths }; + return { + .errors = errors, + .validPaths = validPaths, + }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9a9171e2f..dc2d5cb7c 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -264,12 +264,30 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; +protected: + /** - * @return A pair of whether any errors were encountered, and a set of - * (so-far) valid paths. The store objects pointed to by those paths are - * suitable for further validation checking. + * Result of `verifyAllValidPaths` */ - virtual std::pair verifyAllValidPaths(RepairFlag repair); + struct VerificationResult { + /** + * Whether any errors were encountered + */ + bool errors; + + /** + * A set of so-far valid paths. The store objects pointed to by + * those paths are suitable for further validation checking. + */ + StorePathSet validPaths; + }; + + /** + * First, unconditional step of `verifyStore` + */ + virtual VerificationResult verifyAllValidPaths(RepairFlag repair); + +public: /** * Register the validity of a path, i.e., that `path` exists, that From bf0bf3d1be47aa1bcf9cb3c8f0923e1304fc7065 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:30:40 -0500 Subject: [PATCH 117/211] local-overlay store tests: `storeDirs` -> `setupStoreDirs` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Théophane Hufschmitt --- tests/functional/local-overlay-store/add-lower-inner.sh | 2 +- tests/functional/local-overlay-store/bad-uris.sh | 4 ++-- tests/functional/local-overlay-store/build-inner.sh | 2 +- tests/functional/local-overlay-store/check-post-init-inner.sh | 2 +- tests/functional/local-overlay-store/common.sh | 2 +- .../functional/local-overlay-store/delete-duplicate-inner.sh | 2 +- tests/functional/local-overlay-store/delete-refs-inner.sh | 2 +- tests/functional/local-overlay-store/gc-inner.sh | 2 +- tests/functional/local-overlay-store/optimise-inner.sh | 2 +- tests/functional/local-overlay-store/redundant-add-inner.sh | 2 +- .../functional/local-overlay-store/stale-file-handle-inner.sh | 2 +- tests/functional/local-overlay-store/verify-inner.sh | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/functional/local-overlay-store/add-lower-inner.sh b/tests/functional/local-overlay-store/add-lower-inner.sh index ca7db7ab6..4efa7d088 100755 --- a/tests/functional/local-overlay-store/add-lower-inner.sh +++ b/tests/functional/local-overlay-store/add-lower-inner.sh @@ -10,7 +10,7 @@ source common.sh unset NIX_STORE_DIR unset NIX_STATE_DIR -storeDirs +setupStoreDirs initLowerStore diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index 07b2c6f28..2517681dd 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -2,7 +2,7 @@ source common.sh requireEnvironment setupConfig -storeDirs +setupStoreDirs mkdir -p $TEST_ROOT/bad_test badTestRoot=$TEST_ROOT/bad_test @@ -18,7 +18,7 @@ for i in "${storesBad[@]}"; do echo $i unshare --mount --map-root-user bash < Date: Mon, 11 Dec 2023 13:34:09 -0500 Subject: [PATCH 118/211] Update tests/functional/local-overlay-store/redundant-add-inner.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- tests/functional/local-overlay-store/redundant-add-inner.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/functional/local-overlay-store/redundant-add-inner.sh b/tests/functional/local-overlay-store/redundant-add-inner.sh index 71899a9a7..0fb3af2a6 100755 --- a/tests/functional/local-overlay-store/redundant-add-inner.sh +++ b/tests/functional/local-overlay-store/redundant-add-inner.sh @@ -19,9 +19,13 @@ mountOverlayfs ### Do a redundant add # upper layer should not have it +path=$(nix-store --store "$storeA" ../dummy) + expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") -path=$(nix-store --store "$storeB" --add ../dummy) +pathFromB=$(nix-store --store "$storeB" --add ../dummy) + +[[ $path == $pathFromB ]] # lower store should have it from before stat $(toRealPath "$storeA/nix/store" "$path") From c93f78f6fabdc4f01d3af98c68c4225be9d2c546 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:36:53 -0500 Subject: [PATCH 119/211] Fix test a bit from previous commit --- tests/functional/local-overlay-store/redundant-add-inner.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/functional/local-overlay-store/redundant-add-inner.sh b/tests/functional/local-overlay-store/redundant-add-inner.sh index 0fb3af2a6..d2c95b187 100755 --- a/tests/functional/local-overlay-store/redundant-add-inner.sh +++ b/tests/functional/local-overlay-store/redundant-add-inner.sh @@ -18,9 +18,10 @@ mountOverlayfs ### Do a redundant add -# upper layer should not have it -path=$(nix-store --store "$storeA" ../dummy) +# (Already done in `initLowerStore`, but repeated here for clarity.) +path=$(nix-store --store "$storeA" --add ../dummy) +# upper layer should not have it expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") pathFromB=$(nix-store --store "$storeB" --add ../dummy) From b3bdd70ea2a69ec2ddd547ee475219f5b38b8f59 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:43:17 -0500 Subject: [PATCH 120/211] Clarify `toUpperPath` docs We're just mapping store paths to host OS paths, there is no checking what is actually at this location. --- src/libstore/local-overlay-store.hh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index a93b069cc..2c24285dd 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -62,8 +62,12 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig protected: /** - * Given a store path, get its location (if it is exists) in the - * upper layer of the overlayfs. + * @return The host OS path corresponding to the store path for the + * upper layer. + * + * @note The there is no guarantee a store object is actually stored + * at that file path. It might be stored in the lower layer instead, + * or it might not be part of this store at all. */ Path toUpperPath(const StorePath & path); }; From c90e46d3f0c6614921b42ba5506a409ca11b5c30 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:45:46 -0500 Subject: [PATCH 121/211] Update tests/functional/local-overlay-store/common.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- tests/functional/local-overlay-store/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 0de688eb4..e1da00b42 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -66,7 +66,7 @@ initLowerStore () { # Build something in lower store drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg busybox "$busybox" --arg seed 1) - path=$(nix-store --store "$storeA" --realise $drvPath) + pathInLowerStore=$(nix-store --store "$storeA" --realise $drvPath) } execUnshare () { From 8d0a03b5a22df6f764686386edde6b90d290b8e4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:48:42 -0500 Subject: [PATCH 122/211] Fix tests after last rename (`path` -> `pathInLowerStore`) --- .../check-post-init-inner.sh | 20 +++++++++---------- .../redundant-add-inner.sh | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/functional/local-overlay-store/check-post-init-inner.sh b/tests/functional/local-overlay-store/check-post-init-inner.sh index ccc0461a3..ac2499002 100755 --- a/tests/functional/local-overlay-store/check-post-init-inner.sh +++ b/tests/functional/local-overlay-store/check-post-init-inner.sh @@ -19,13 +19,13 @@ mountOverlayfs ### Check status # Checking for path in lower layer -stat $(toRealPath "$storeA/nix/store" "$path") +stat $(toRealPath "$storeA/nix/store" "$pathInLowerStore") # Checking for path in upper layer (should fail) -expect 1 stat $(toRealPath "$storeBTop" "$path") +expect 1 stat $(toRealPath "$storeBTop" "$pathInLowerStore") # Checking for path in overlay store matching lower layer -diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeBRoot/nix/store" "$path") +diff $(toRealPath "$storeA/nix/store" "$pathInLowerStore") $(toRealPath "$storeBRoot/nix/store" "$pathInLowerStore") # Checking requisites query agreement [[ \ @@ -44,9 +44,9 @@ busyboxStore=$(nix store --store $storeA add-path $busybox) # Checking derivers query agreement [[ \ - $(nix-store --store $storeA --query --deriver $path) \ + $(nix-store --store $storeA --query --deriver $pathInLowerStore) \ == \ - $(nix-store --store $storeB --query --deriver $path) \ + $(nix-store --store $storeB --query --deriver $pathInLowerStore) \ ]] # Checking outputs query agreement @@ -57,15 +57,15 @@ busyboxStore=$(nix store --store $storeA add-path $busybox) ]] # Verifying path in lower layer -nix-store --verify-path --store "$storeA" "$path" +nix-store --verify-path --store "$storeA" "$pathInLowerStore" # Verifying path in merged-store -nix-store --verify-path --store "$storeB" "$path" +nix-store --verify-path --store "$storeB" "$pathInLowerStore" -hashPart=$(echo $path | sed "s^${NIX_STORE_DIR:-/nix/store}/^^" | sed 's/-.*//') +hashPart=$(echo $pathInLowerStore | sed "s^${NIX_STORE_DIR:-/nix/store}/^^" | sed 's/-.*//') # Lower store can find from hash part -[[ $(nix store --store $storeA path-from-hash-part $hashPart) == $path ]] +[[ $(nix store --store $storeA path-from-hash-part $hashPart) == $pathInLowerStore ]] # merged store can find from hash part -[[ $(nix store --store $storeB path-from-hash-part $hashPart) == $path ]] +[[ $(nix store --store $storeB path-from-hash-part $hashPart) == $pathInLowerStore ]] diff --git a/tests/functional/local-overlay-store/redundant-add-inner.sh b/tests/functional/local-overlay-store/redundant-add-inner.sh index d2c95b187..e37ef90e5 100755 --- a/tests/functional/local-overlay-store/redundant-add-inner.sh +++ b/tests/functional/local-overlay-store/redundant-add-inner.sh @@ -19,17 +19,17 @@ mountOverlayfs ### Do a redundant add # (Already done in `initLowerStore`, but repeated here for clarity.) -path=$(nix-store --store "$storeA" --add ../dummy) +pathInLowerStore=$(nix-store --store "$storeA" --add ../dummy) # upper layer should not have it -expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path") +expect 1 stat $(toRealPath "$storeBTop/nix/store" "$pathInLowerStore") pathFromB=$(nix-store --store "$storeB" --add ../dummy) -[[ $path == $pathFromB ]] +[[ $pathInLowerStore == $pathFromB ]] # lower store should have it from before -stat $(toRealPath "$storeA/nix/store" "$path") +stat $(toRealPath "$storeA/nix/store" "$pathInLowerStore") # upper layer should still not have it (no redundant copy) -expect 1 stat $(toRealPath "$storeBTop" "$path") +expect 1 stat $(toRealPath "$storeBTop" "$pathInLowerStore") From 4a2cee8e6cf2cfe0975b11fbccf6beb36b99cc60 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 11 Dec 2023 18:55:39 +0000 Subject: [PATCH 123/211] Document expected filesystem layout and OverlayFS mount command. --- src/libstore/local-overlay-store.md | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index a0b9c3adf..813efc3e9 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -5,6 +5,7 @@ R"( This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short). Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost [local store]. ("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) +To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly). ### Parts of a local overlay store @@ -15,6 +16,9 @@ The parts of a local overlay store are as follows: This is any store implementation that includes a store directory as part of the native operating system filesystem. For example, this could be a [local store], [local daemon store], or even another local overlay store. + The lower store must not change while it is mounted as part of an overlay store. + To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`). + Specified with the `lower-store` setting. - Lower store directory. @@ -51,4 +55,42 @@ The parts of a local overlay store are as follows: Specified with the `state` setting, is always `${state}/db`. + +### Example filesystem layout + +Say we have the following paths: + +- `/mnt/example/merged-store/nix/store` + +- `/mnt/example/store-a/nix/store` + +- `/mnt/example/store-b` + + +Then the following store URI can be used to access a local-overlay store at `/mnt/example/merged-store`: + +``` + local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b +``` + +The lower store is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`. + +Before accessing the overlay store you will need to ensure the OverlayFS mount is set up correctly: + +``` + mount -t overlay overlay \ + -o lowerdir="/mnt/example/store-a/nix/store" \ + -o upperdir="/mnt/example/store-b" \ + -o workdir="/mnt/example/workdir" \ + "/mnt/example/merged-store/nix/store" \ +``` + +Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`. + +By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not. +You can override this behaviour by passing [`check-mount=false`](???) if you need to. + + + + )" From eae2717e00e82a61884b3f53a50db94272a1fad1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 13:55:43 -0500 Subject: [PATCH 124/211] tests: Use `cp -ar` instead of tar-untar pipe --- tests/functional/local-overlay-store/verify-inner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/local-overlay-store/verify-inner.sh b/tests/functional/local-overlay-store/verify-inner.sh index c98ea1f96..659f2ae50 100755 --- a/tests/functional/local-overlay-store/verify-inner.sh +++ b/tests/functional/local-overlay-store/verify-inner.sh @@ -34,7 +34,7 @@ nix-store --store "$storeB" --verify --check-contents # Make a backup so we can repair later backupStore="$storeVolume/backup" mkdir "$backupStore" -tar -cC "$storeBRoot" nix | tar -xC "$backupStore" +cp -ar "$storeBRoot/nix" "$backupStore" ## Deliberately corrupt store paths From dc439eaf236ea1ef5c291b50de25817df8103041 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 1 Feb 2024 11:20:19 -0500 Subject: [PATCH 125/211] Fill in missing markdown link dest --- src/libstore/local-overlay-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index 813efc3e9..882d2c5ce 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -88,7 +88,7 @@ Before accessing the overlay store you will need to ensure the OverlayFS mount i Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`. By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not. -You can override this behaviour by passing [`check-mount=false`](???) if you need to. +You can override this behaviour by passing [`check-mount=false`](#store-experimental-local-overlay-store-check-mount) if you need to. From 9b506ff0c1e48fa805eab86eb43bd99ac1ae24a3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 29 Feb 2024 10:06:53 -0500 Subject: [PATCH 126/211] Activate `hermetic.nix` variation only for new layered store tests --- tests/functional/hermetic.nix | 9 +++++++-- tests/functional/local-overlay-store/common.sh | 2 +- .../functional/local-overlay-store/delete-refs-inner.sh | 8 ++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/functional/hermetic.nix b/tests/functional/hermetic.nix index 810180ac6..d1dccdff3 100644 --- a/tests/functional/hermetic.nix +++ b/tests/functional/hermetic.nix @@ -1,4 +1,9 @@ -{ busybox, seed }: +{ busybox +, seed +# If we want the final derivation output to have references to its +# dependencies. Some tests need/want this, other don't. +, withFinalRefs ? false +}: with import ./config.nix; @@ -54,6 +59,6 @@ in '' read x < ${input1} read y < ${input3} - echo ${input1} ${input3} "$x $y" > $out + echo ${if (builtins.trace withFinalRefs withFinalRefs) then "${input1} ${input3}" else ""} "$x $y" > $out ''; } diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index e1da00b42..13d921c5e 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -65,7 +65,7 @@ initLowerStore () { nix-store --store "$storeA" --add ../dummy # Build something in lower store - drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg busybox "$busybox" --arg seed 1) + drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg withFinalRefs true --arg busybox "$busybox" --arg seed 1) pathInLowerStore=$(nix-store --store "$storeA" --realise $drvPath) } diff --git a/tests/functional/local-overlay-store/delete-refs-inner.sh b/tests/functional/local-overlay-store/delete-refs-inner.sh index e24f34a85..385eeadc9 100644 --- a/tests/functional/local-overlay-store/delete-refs-inner.sh +++ b/tests/functional/local-overlay-store/delete-refs-inner.sh @@ -16,10 +16,10 @@ mountOverlayfs export NIX_REMOTE="$storeB" stateB="$storeBRoot/nix/var/nix" -hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2) -input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input1 -j0) -input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input2 -j0) -input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input3 -j0) +hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2) +input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input1 -j0) +input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input2 -j0) +input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input3 -j0) # Can't delete because referenced expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path" From bcd6b33dbcbd2e9c2ef3eaf50b9339fe9a33a49d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 29 Feb 2024 11:58:53 -0500 Subject: [PATCH 127/211] Polish local overlay store docs --- src/libstore/local-overlay-store.md | 95 ++++++++++++++++++----------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index 882d2c5ce..b85f69205 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -3,61 +3,92 @@ R"( **Store URL format**: `local-overlay` This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short). -Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost [local store]. +Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost-[local store]. ("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly). -### Parts of a local overlay store +### Conceptual parts of a local overlay store + +*This is a more abstract/conceptual description of the parts of a layered store, an authoritative reference. +For more "practical" instructions, see the worked-out example in the next subsection.* The parts of a local overlay store are as follows: -- Lower store: +- **Lower store**: This is any store implementation that includes a store directory as part of the native operating system filesystem. For example, this could be a [local store], [local daemon store], or even another local overlay store. + The local overlay store never tries to modify the lower store in any way. + Something else could modify the lower store, but there are restrictions on this + Nix itself requires that this store only grow, and not change in other ways. + For example, new store objects can be added, but deleting or modifying store objects is not allowed in general, because that will confuse and corrupt any local overlay store using those objects. + (In addition, the underlying filesystem overlay mechanism may imposed additional restrictions, see below.) + The lower store must not change while it is mounted as part of an overlay store. To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`). - Specified with the `lower-store` setting. + Specified with the [`lower-store`](#store-experimental-local-overlay-store-lower-store) setting. + + - **Lower store directory**: - - Lower store directory. This is the directory used/exposed by the lower store. Specified with `lower-store.real` setting. - - Lower abstract read-only metadata source. - This is abstract, just some way to read the metadata of lower store [store objects](@docroot@/glossary.md#gloss-store-object). + As specified above, Nix requires the local store can only grow not change in other ways. + Linux's OverlayFS in addition imposes the further requirement that this directory cannot change at all. + That means that, while any local overlay store exists that is using this store as a lower store, this directory must not change. + + - **Lower metadata source**: + + This is abstract, just some way to read the metadata of lower store [store objects][store object]. For example it could be a SQLite database (for the [local store]), or a socket connection (for the [local daemon store]). -- Upper almost-store: + This need not be writable. + As stated above a local overlay store never tries to modify its lower store. + The lower store's metadata is considered part of the lower store, just as the store's [file system objects][file system object] that appear in the store directory are. - This is a [local store] that by itself would appear corrupted. +- **Upper almost-store**: + + This is almost but not quite just a [local store]. + That is because taken in isolation, not as part of a local overlay store, by itself, it would appear corrupted. But combined with everything else as part of an overlay local store, it is valid. - - Upper layer directory. - This contains additional [store objects] - (or, strictly speaking, their [file system objects](#gloss-file-system-object)) - that the local overlay store will extend the lower store with. + - **Upper layer directory**: - Specified with `upper-layer` setting. + This contains additional [store objects][store object] + (or, strictly speaking, their [file system objects][file system object] that the local overlay store will extend the lower store with). + + Specified with [`upper-layer`](#store-experimental-local-overlay-store-upper-layer) setting. + + - **Upper store directory**: - - Upper store directory - The lower store directory and upper layer directory are combined via OverlayFS to create this directory. This contains all the store objects from each of the two directories. - Specified with the `real` setting. + The lower store directory and upper layer directory are combined via OverlayFS to create this directory. + Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first. + Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings. - - Upper SQLite database - This contains the metadata of all of the upper layer [store objects]: everything beyond their file system objects, and also duplicate copies of some lower layer ones. - The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects] can entirely be found in the upper layer. - This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way. + Specified with the [`real`](#store-experimental-local-overlay-store-real) setting. - Specified with the `state` setting, is always `${state}/db`. + - **Upper SQLite database**: + + This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta. + The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer. + (This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way.) + + The location of the database is directly specified, but depends on the [`state`](#store-experimental-local-overlay-store-state) setting. + It is is always `${state}/db`. + +[file system object]: @docroot@/store/file-system-object.md +[store object]: @docroot@/store/store-object.md ### Example filesystem layout +Here is a worked out example of usage, following the concepts in the previous section. + Say we have the following paths: - `/mnt/example/merged-store/nix/store` @@ -66,23 +97,22 @@ Say we have the following paths: - `/mnt/example/store-b` - Then the following store URI can be used to access a local-overlay store at `/mnt/example/merged-store`: ``` - local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b +local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b ``` -The lower store is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`. +The lower store directory is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`. Before accessing the overlay store you will need to ensure the OverlayFS mount is set up correctly: -``` - mount -t overlay overlay \ - -o lowerdir="/mnt/example/store-a/nix/store" \ - -o upperdir="/mnt/example/store-b" \ - -o workdir="/mnt/example/workdir" \ - "/mnt/example/merged-store/nix/store" \ +```shell +mount -t overlay overlay \ + -o lowerdir="/mnt/example/store-a/nix/store" \ + -o upperdir="/mnt/example/store-b" \ + -o workdir="/mnt/example/workdir" \ + "/mnt/example/merged-store/nix/store" ``` Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`. @@ -90,7 +120,4 @@ Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not. You can override this behaviour by passing [`check-mount=false`](#store-experimental-local-overlay-store-check-mount) if you need to. - - - )" From d6d7d2cb4617c571e1ee7c152818f3b7a07a5f81 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 29 Feb 2024 14:39:29 -0500 Subject: [PATCH 128/211] Revert "Merge pull request #9546 from NixOS/nixos-23.11" This reverts commit 587c7dcb2bc4e504159f527c7e776ede231bd0db, reversing changes made to 864fc85fc88ff092725ba99907611b2b8d2205fb. --- Makefile | 1 - flake.lock | 8 ++++---- flake.nix | 26 +++++++++++++++----------- package.nix | 2 +- src/libstore/local-store.cc | 2 +- tests/functional/git-hashing/common.sh | 2 +- tests/functional/impure-env.sh | 2 +- 7 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index af00e58b2..a33614d26 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,6 @@ ifeq ($(OPTIMIZE), 1) GLOBAL_LDFLAGS += $(CXXLTO) else GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE - unexport NIX_HARDENING_ENABLE endif include mk/platform.mk diff --git a/flake.lock b/flake.lock index bb2e400c0..f0efb4036 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1705033721, + "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 42aaace67..0bc70768e 100644 --- a/flake.nix +++ b/flake.nix @@ -1,9 +1,7 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; @@ -12,10 +10,20 @@ let inherit (nixpkgs) lib; - inherit (lib) fileset; + + # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 + # Not an "idiomatic" flake input because: + # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 + # - Subflake would download redundant and huge parent flake + # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 + inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) + fileset; officialRelease = false; + # Set to true to build the release notes for the next release. + buildUnreleasedNotes = false; + version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease @@ -396,11 +404,8 @@ # Make bash completion work. XDG_DATA_DIRS+=:$out/share ''; - nativeBuildInputs = attrs.nativeBuildInputs or [] - # TODO: Remove the darwin check once - # https://github.com/NixOS/nixpkgs/pull/291814 is available - ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear + ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; }); in @@ -412,9 +417,8 @@ (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName})); in (makeShells "native" nixpkgsFor.${system}.native) // - (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/package.nix b/package.nix index 20796a386..1f895e301 100644 --- a/package.nix +++ b/package.nix @@ -154,7 +154,7 @@ in { in fileset.toSource { root = ./.; - fileset = fileset.intersection baseFiles (fileset.unions ([ + fileset = fileset.intersect baseFiles (fileset.unions ([ # For configure ./.version ./configure.ac diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a7c34ab4a..0d9faa7c6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1208,7 +1208,7 @@ StorePath LocalStore::addToStoreFromDump( Path tempDir; AutoCloseFD tempDirFd; - bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod; + bool methodsMatch = (FileIngestionMethod) dumpMethod == hashMethod; /* If the methods don't match, our streaming hash of the dump is the wrong sort, and we need to rehash. */ diff --git a/tests/functional/git-hashing/common.sh b/tests/functional/git-hashing/common.sh index 572cea438..5de96e74f 100644 --- a/tests/functional/git-hashing/common.sh +++ b/tests/functional/git-hashing/common.sh @@ -4,7 +4,7 @@ clearStore clearCache # Need backend to support git-hashing too -requireDaemonNewerThan "2.19" +requireDaemonNewerThan "2.18.0pre20230908" enableFeatures "git-hashing" diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh index cfea4cae9..d9e4a34a2 100644 --- a/tests/functional/impure-env.sh +++ b/tests/functional/impure-env.sh @@ -1,7 +1,7 @@ source common.sh # Needs the config option 'impure-env' to work -requireDaemonNewerThan "2.19.0" +requireDaemonNewerThan "2.18.0pre20230816" enableFeatures "configurable-impure-env" restartDaemon From 5a2985431c8b03be7d434a39ea62a8b3d26e4f51 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 29 Feb 2024 14:52:31 -0500 Subject: [PATCH 129/211] Revert "Revert "Merge pull request #9546 from NixOS/nixos-23.11"" This reverts commit d6d7d2cb4617c571e1ee7c152818f3b7a07a5f81. --- Makefile | 1 + flake.lock | 8 ++++---- flake.nix | 26 +++++++++++--------------- package.nix | 2 +- src/libstore/local-store.cc | 2 +- tests/functional/git-hashing/common.sh | 2 +- tests/functional/impure-env.sh | 2 +- 7 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index a33614d26..af00e58b2 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,7 @@ ifeq ($(OPTIMIZE), 1) GLOBAL_LDFLAGS += $(CXXLTO) else GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE + unexport NIX_HARDENING_ENABLE endif include mk/platform.mk diff --git a/flake.lock b/flake.lock index f0efb4036..bb2e400c0 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1705033721, - "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=", + "lastModified": 1709083642, + "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea", + "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "release-23.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 0bc70768e..42aaace67 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,9 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + # TODO switch to nixos-23.11-small + # https://nixpk.gs/pr-tracker.html?pr=291954 + inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; @@ -10,20 +12,10 @@ let inherit (nixpkgs) lib; - - # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 - # Not an "idiomatic" flake input because: - # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 - # - Subflake would download redundant and huge parent flake - # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 - inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) - fileset; + inherit (lib) fileset; officialRelease = false; - # Set to true to build the release notes for the next release. - buildUnreleasedNotes = false; - version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease @@ -404,8 +396,11 @@ # Make bash completion work. XDG_DATA_DIRS+=:$out/share ''; + nativeBuildInputs = attrs.nativeBuildInputs or [] - ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + # TODO: Remove the darwin check once + # https://github.com/NixOS/nixpkgs/pull/291814 is available + ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; }); in @@ -417,8 +412,9 @@ (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName})); in (makeShells "native" nixpkgsFor.${system}.native) // - (makeShells "static" nixpkgsFor.${system}.static) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) + (makeShells "static" nixpkgsFor.${system}.static)) // + (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/package.nix b/package.nix index 1f895e301..20796a386 100644 --- a/package.nix +++ b/package.nix @@ -154,7 +154,7 @@ in { in fileset.toSource { root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions ([ + fileset = fileset.intersection baseFiles (fileset.unions ([ # For configure ./.version ./configure.ac diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0d9faa7c6..a7c34ab4a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1208,7 +1208,7 @@ StorePath LocalStore::addToStoreFromDump( Path tempDir; AutoCloseFD tempDirFd; - bool methodsMatch = (FileIngestionMethod) dumpMethod == hashMethod; + bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod; /* If the methods don't match, our streaming hash of the dump is the wrong sort, and we need to rehash. */ diff --git a/tests/functional/git-hashing/common.sh b/tests/functional/git-hashing/common.sh index 5de96e74f..572cea438 100644 --- a/tests/functional/git-hashing/common.sh +++ b/tests/functional/git-hashing/common.sh @@ -4,7 +4,7 @@ clearStore clearCache # Need backend to support git-hashing too -requireDaemonNewerThan "2.18.0pre20230908" +requireDaemonNewerThan "2.19" enableFeatures "git-hashing" diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh index d9e4a34a2..cfea4cae9 100644 --- a/tests/functional/impure-env.sh +++ b/tests/functional/impure-env.sh @@ -1,7 +1,7 @@ source common.sh # Needs the config option 'impure-env' to work -requireDaemonNewerThan "2.18.0pre20230816" +requireDaemonNewerThan "2.19.0" enableFeatures "configurable-impure-env" restartDaemon From fe42a0ead70db8f5b82dec42a14682bbe4414577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:10:32 +0100 Subject: [PATCH 130/211] Documentation typo --- src/libstore/local-overlay-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index b85f69205..cc310bc7f 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -23,7 +23,7 @@ The parts of a local overlay store are as follows: Something else could modify the lower store, but there are restrictions on this Nix itself requires that this store only grow, and not change in other ways. For example, new store objects can be added, but deleting or modifying store objects is not allowed in general, because that will confuse and corrupt any local overlay store using those objects. - (In addition, the underlying filesystem overlay mechanism may imposed additional restrictions, see below.) + (In addition, the underlying filesystem overlay mechanism may impose additional restrictions, see below.) The lower store must not change while it is mounted as part of an overlay store. To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`). From cd35e0010322bbf30b967cb7991e84dd740e43df Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Mar 2024 16:41:16 -0400 Subject: [PATCH 131/211] Adding missing tracking URL for local overlay store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libutil/experimental-features.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 595dd4301..e1a8b5b9d 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -260,6 +260,7 @@ constexpr std::array xpFeatureDetails .description = R"( Allow the use of [local overlay store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-overlay-store). )", + .trackingUrl = ""https://github.com/NixOS/nix/milestone/50", }, { .tag = Xp::ConfigurableImpureEnv, From 8c0590fa32395e8f66bad11049fb40458d94424b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Mar 2024 18:20:18 +0100 Subject: [PATCH 132/211] Never update values after setting the type Thunks are now overwritten by a helper function `Value::finishValue(newType, payload)` (where `payload` is the original anonymous union inside `Value`). This helps to ensure we never update a value elsewhere, since that would be incompatible with parallel evaluation (i.e. after a value has transitioned from being a thunk to being a non-thunk, it should be immutable). There were two places where this happened: `Value::mkString()` and `ExprAttrs::eval()`. This PR also adds a bunch of accessor functions for value contents, like `Value::integer()` to access the integer field in the union. --- src/libcmd/installable-flake.cc | 2 +- src/libcmd/installables.cc | 4 +- src/libcmd/repl.cc | 10 +- src/libexpr/attr-path.cc | 6 +- src/libexpr/attr-set.cc | 17 -- src/libexpr/attr-set.hh | 34 ++- src/libexpr/eval-cache.cc | 14 +- src/libexpr/eval-inline.hh | 6 +- src/libexpr/eval.cc | 303 ++++++++++---------- src/libexpr/eval.hh | 5 +- src/libexpr/flake/flake.cc | 34 +-- src/libexpr/get-drvs.cc | 68 +++-- src/libexpr/get-drvs.hh | 6 +- src/libexpr/nixexpr.cc | 4 +- src/libexpr/primops.cc | 125 ++++---- src/libexpr/primops/context.cc | 24 +- src/libexpr/primops/fetchClosure.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/fetchTree.cc | 10 +- src/libexpr/print-ambiguous.cc | 12 +- src/libexpr/print.cc | 27 +- src/libexpr/value-to-json.cc | 28 +- src/libexpr/value-to-xml.cc | 47 ++- src/libexpr/value.hh | 129 ++++----- src/nix-build/nix-build.cc | 4 +- src/nix-env/nix-env.cc | 14 +- src/nix-env/user-env.cc | 4 +- src/nix/bundle.cc | 6 +- src/nix/eval.cc | 2 +- src/nix/flake.cc | 54 ++-- src/nix/main.cc | 7 +- src/nix/prefetch.cc | 10 +- tests/unit/libexpr-support/tests/libexpr.hh | 12 +- tests/unit/libexpr/primops.cc | 42 +-- tests/unit/libexpr/trivial.cc | 12 +- 35 files changed, 530 insertions(+), 556 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index ddec7537b..6ff837ddc 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -49,7 +49,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked callFlake(state, lockedFlake, *vFlake); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 6db9bf9a1..ed6772377 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -289,7 +289,7 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { + for (auto & i : *v2.attrs()) { std::string name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") @@ -461,7 +461,7 @@ ref openEvalCache( state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 7cecc60b7..a97d8aaf4 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -290,7 +290,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) e->eval(*state, *env, v); state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -490,7 +490,7 @@ ProcessLineResult NixRepl::processLine(std::string line) auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; + auto pos = state->positions[v.payload.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else @@ -742,17 +742,17 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); - if (displ + attrs.attrs->size() >= envSize) + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index d6befd362..9ad201b63 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -72,10 +72,10 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin if (attr.empty()) throw Error("empty attribute name in selection path '%1%'", attrPath); - Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); - if (a == v->attrs->end()) { + auto a = v->attrs()->get(state.symbols.create(attr)); + if (!a) { std::set attrNames; - for (auto & attr : *v->attrs) + for (auto & attr : *v->attrs()) attrNames.insert(state.symbols[attr.name]); auto suggestions = Suggestions::bestMatches(attrNames, attr); diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 877116f1f..866ef817a 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -23,23 +23,6 @@ Bindings * EvalState::allocBindings(size_t capacity) } -/* Create a new attribute named 'name' on an existing attribute set stored - in 'vAttrs' and return the newly allocated Value which is associated with - this attribute. */ -Value * EvalState::allocAttr(Value & vAttrs, Symbol name) -{ - Value * v = allocValue(); - vAttrs.attrs->push_back(Attr(name, v)); - return v; -} - - -Value * EvalState::allocAttr(Value & vAttrs, std::string_view name) -{ - return allocAttr(vAttrs, symbols.create(name)); -} - - Value & BindingsBuilder::alloc(Symbol name, PosIdx pos) { auto value = state.allocValue(); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 31215f880..c90bb0633 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -65,24 +65,26 @@ public: typedef Attr * iterator; + typedef const Attr * const_iterator; + void push_back(const Attr & attr) { assert(size_ < capacity_); attrs[size_++] = attr; } - iterator find(Symbol name) + const_iterator find(Symbol name) const { Attr key(name, 0); - iterator i = std::lower_bound(begin(), end(), key); + const_iterator i = std::lower_bound(begin(), end(), key); if (i != end() && i->name == name) return i; return end(); } - Attr * get(Symbol name) + const Attr * get(Symbol name) const { Attr key(name, 0); - iterator i = std::lower_bound(begin(), end(), key); + const_iterator i = std::lower_bound(begin(), end(), key); if (i != end() && i->name == name) return &*i; return nullptr; } @@ -90,14 +92,22 @@ public: iterator begin() { return &attrs[0]; } iterator end() { return &attrs[size_]; } + const_iterator begin() const { return &attrs[0]; } + const_iterator end() const { return &attrs[size_]; } + Attr & operator[](size_t pos) { return attrs[pos]; } + const Attr & operator[](size_t pos) const + { + return attrs[pos]; + } + void sort(); - size_t capacity() { return capacity_; } + size_t capacity() const { return capacity_; } /** * Returns the attributes in lexicographically sorted order. @@ -166,6 +176,20 @@ public: { return bindings; } + + size_t capacity() + { + return bindings->capacity(); + } + + void grow(Bindings * newBindings) + { + for (auto & i : *bindings) + newBindings->push_back(i); + bindings = newBindings; + } + + friend class ExprAttrs; }; } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 1538eb056..d60967a14 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -387,7 +387,7 @@ Value & AttrCursor::getValue() if (parent) { auto & vParent = parent->first->getValue(); root->state.forceAttrs(vParent, noPos, "while searching for an attribute"); - auto attr = vParent.attrs->get(parent->second); + auto attr = vParent.attrs()->get(parent->second); if (!attr) throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); _value = allocRootValue(attr->value); @@ -448,9 +448,9 @@ Value & AttrCursor::forceValue() cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; } else if (v.type() == nBool) - cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; + cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()}; else if (v.type() == nInt) - cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}}; + cachedValue = {root->db->setInt(getKey(), v.integer()), int_t{v.integer()}}; else if (v.type() == nAttrs) ; // FIXME: do something? else @@ -510,7 +510,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro return nullptr; //error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); - auto attr = v.attrs->get(name); + auto attr = v.attrs()->get(name); if (!attr) { if (root->db) { @@ -652,7 +652,7 @@ bool AttrCursor::getBool() if (v.type() != nBool) root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); - return v.boolean; + return v.boolean(); } NixInt AttrCursor::getInt() @@ -674,7 +674,7 @@ NixInt AttrCursor::getInt() if (v.type() != nInt) root->state.error("'%s' is not an integer", getAttrPathStr()).debugThrow(); - return v.integer; + return v.integer(); } std::vector AttrCursor::getListOfStrings() @@ -730,7 +730,7 @@ std::vector AttrCursor::getAttrs() root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); std::vector attrs; - for (auto & attr : *getValue().attrs) + for (auto & attr : *getValue().attrs()) attrs.push_back(attr.name); std::sort(attrs.begin(), attrs.end(), [&](Symbol a, Symbol b) { std::string_view sa = root->state.symbols[a], sb = root->state.symbols[b]; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 03320c7c9..6fa34b062 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -85,8 +85,8 @@ Env & EvalState::allocEnv(size_t size) void EvalState::forceValue(Value & v, const PosIdx pos) { if (v.isThunk()) { - Env * env = v.thunk.env; - Expr * expr = v.thunk.expr; + Env * env = v.payload.thunk.env; + Expr * expr = v.payload.thunk.expr; try { v.mkBlackhole(); //checkInterrupt(); @@ -98,7 +98,7 @@ void EvalState::forceValue(Value & v, const PosIdx pos) } } else if (v.isApp()) - callFunction(*v.app.left, *v.app.right, v, pos); + callFunction(*v.payload.app.left, *v.payload.app.right, v, pos); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5e2f71649..7147e7997 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -131,7 +131,7 @@ void Value::print(EvalState & state, std::ostream & str, PrintOptions options) const Value * getPrimOp(const Value &v) { const Value * primOp = &v; while (primOp->isPrimOpApp()) { - primOp = primOp->primOpApp.left; + primOp = primOp->payload.primOpApp.left; } assert(primOp->isPrimOp()); return primOp; @@ -163,12 +163,12 @@ std::string showType(const Value & v) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v.internalType) { - case tString: return v.string.context ? "a string with context" : "a string"; + case tString: return v.payload.string.context ? "a string with context" : "a string"; case tPrimOp: - return fmt("the built-in function '%s'", std::string(v.primOp->name)); + return fmt("the built-in function '%s'", std::string(v.payload.primOp->name)); case tPrimOpApp: - return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name)); - case tExternal: return v.external->showType(); + return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->payload.primOp->name)); + case tExternal: return v.external()->showType(); case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk"; case tApp: return "a function application"; default: @@ -183,9 +183,9 @@ PosIdx Value::determinePos(const PosIdx pos) const #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" switch (internalType) { - case tAttrs: return attrs->pos; - case tLambda: return lambda.fun->pos; - case tApp: return app.left->determinePos(pos); + case tAttrs: return attrs()->pos; + case tLambda: return payload.lambda.fun->pos; + case tApp: return payload.app.left->determinePos(pos); default: return pos; } #pragma GCC diagnostic pop @@ -197,10 +197,10 @@ bool Value::isTrivial() const internalType != tApp && internalType != tPrimOpApp && (internalType != tThunk - || (dynamic_cast(thunk.expr) - && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) - || dynamic_cast(thunk.expr) - || dynamic_cast(thunk.expr)); + || (dynamic_cast(payload.thunk.expr) + && ((ExprAttrs *) payload.thunk.expr)->dynamicAttrs.empty()) + || dynamic_cast(payload.thunk.expr) + || dynamic_cast(payload.thunk.expr)); } @@ -584,7 +584,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) /* Install value the base environment. */ staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); + baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(name2), v)); } } @@ -597,32 +597,30 @@ void PrimOp::check() } -std::ostream & operator<<(std::ostream & output, PrimOp & primOp) +std::ostream & operator<<(std::ostream & output, const PrimOp & primOp) { output << "primop " << primOp.name; return output; } -PrimOp * Value::primOpAppPrimOp() const +const PrimOp * Value::primOpAppPrimOp() const { - Value * left = primOpApp.left; + Value * left = payload.primOpApp.left; while (left && !left->isPrimOp()) { - left = left->primOpApp.left; + left = left->payload.primOpApp.left; } if (!left) return nullptr; - return left->primOp; + return left->primOp(); } void Value::mkPrimOp(PrimOp * p) { p->check(); - clearValue(); - internalType = tPrimOp; - primOp = p; + finishValue(tPrimOp, { .primOp = p }); } @@ -650,14 +648,14 @@ Value * EvalState::addPrimOp(PrimOp && primOp) v->mkPrimOp(new PrimOp(primOp)); staticBaseEnv->vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(primOp.name), v)); + baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(primOp.name), v)); return v; } Value & EvalState::getBuiltin(const std::string & name) { - return *baseEnv.values[0]->attrs->find(symbols.create(name))->value; + return *baseEnv.values[0]->attrs()->find(symbols.create(name))->value; } @@ -665,12 +663,12 @@ std::optional EvalState::getDoc(Value & v) { if (v.isPrimOp()) { auto v2 = &v; - if (auto * doc = v2->primOp->doc) + if (auto * doc = v2->primOp()->doc) return Doc { .pos = {}, - .name = v2->primOp->name, - .arity = v2->primOp->arity, - .args = v2->primOp->args, + .name = v2->primOp()->name, + .arity = v2->primOp()->arity, + .args = v2->primOp()->args, .doc = doc, }; } @@ -694,8 +692,8 @@ void printWithBindings(const SymbolTable & st, const Env & env) if (!env.values[0]->isThunk()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; - Bindings::iterator j = env.values[0]->attrs->begin(); - while (j != env.values[0]->attrs->end()) { + auto j = env.values[0]->attrs()->begin(); + while (j != env.values[0]->attrs()->end()) { std::cout << st[j->name] << " "; ++j; } @@ -749,11 +747,8 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (se.isWith && !env.values[0]->isThunk()) { // add 'with' bindings. - Bindings::iterator j = env.values[0]->attrs->begin(); - while (j != env.values[0]->attrs->end()) { - vm[st[j->name]] = j->value; - ++j; - } + for (auto & j : *env.values[0]->attrs()) + vm[st[j.name]] = j.value; } else { // iterate through staticenv bindings and add them. for (auto & i : se.vars) @@ -873,28 +868,28 @@ void Value::mkString(std::string_view s) } -static void copyContextToValue(Value & v, const NixStringContext & context) +static const char * * encodeContext(const NixStringContext & context) { if (!context.empty()) { size_t n = 0; - v.string.context = (const char * *) + auto ctx = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); for (auto & i : context) - v.string.context[n++] = dupString(i.to_string().c_str()); - v.string.context[n] = 0; - } + ctx[n++] = dupString(i.to_string().c_str()); + ctx[n] = 0; + return ctx; + } else + return nullptr; } void Value::mkString(std::string_view s, const NixStringContext & context) { - mkString(s); - copyContextToValue(*this, context); + mkString(makeImmutableString(s), encodeContext(context)); } void Value::mkStringMove(const char * s, const NixStringContext & context) { - mkString(s); - copyContextToValue(*this, context); + mkString(s, encodeContext(context)); } @@ -918,8 +913,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) auto * fromWith = var.fromWith; while (1) { forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression"); - Bindings::iterator j = env->values[0]->attrs->find(var.name); - if (j != env->values[0]->attrs->end()) { + if (auto j = env->values[0]->attrs()->get(var.name)) { if (countCalls) attrSelects[j->pos]++; return j->value; } @@ -1167,7 +1161,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::stri showType(v), ValuePrinter(*this, v, errorPrintOptions) ).atPos(pos).withFrame(env, *e).debugThrow(); - return v.boolean; + return v.boolean(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -1235,8 +1229,9 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up) void ExprAttrs::eval(EvalState & state, Env & env, Value & v) { - v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish()); + auto bindings = state.buildBindings(attrs.size() + dynamicAttrs.size()); auto dynamicEnv = &env; + bool sort = false; if (recursive) { /* Create a new environment that contains the attributes in @@ -1261,7 +1256,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) } else vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv)); env2.values[displ++] = vAttr; - v.attrs->push_back(Attr(i.first, vAttr, i.second.pos)); + bindings.insert(i.first, vAttr, i.second.pos); } /* If the rec contains an attribute called `__overrides', then @@ -1273,32 +1268,28 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) been substituted into the bodies of the other attributes. Hence we need __overrides.) */ if (hasOverrides) { - Value * vOverrides = (*v.attrs)[overrides->second.displ].value; + Value * vOverrides = (*bindings.bindings)[overrides->second.displ].value; state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }, "while evaluating the `__overrides` attribute"); - Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); - for (auto & i : *v.attrs) - newBnds->push_back(i); - for (auto & i : *vOverrides->attrs) { + bindings.grow(state.allocBindings(bindings.capacity() + vOverrides->attrs()->size())); + for (auto & i : *vOverrides->attrs()) { AttrDefs::iterator j = attrs.find(i.name); if (j != attrs.end()) { - (*newBnds)[j->second.displ] = i; + (*bindings.bindings)[j->second.displ] = i; env2.values[j->second.displ] = i.value; } else - newBnds->push_back(i); + bindings.push_back(i); } - newBnds->sort(); - v.attrs = newBnds; + sort = true; } } else { Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr; - for (auto & i : attrs) { - v.attrs->push_back(Attr( - i.first, - i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), - i.second.pos)); - } + for (auto & i : attrs) + bindings.insert( + i.first, + i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), + i.second.pos); } /* Dynamic attrs apply *after* rec and __overrides. */ @@ -1310,17 +1301,21 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) continue; state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute"); auto nameSym = state.symbols.create(nameVal.string_view()); - Bindings::iterator j = v.attrs->find(nameSym); - if (j != v.attrs->end()) + if (sort) + // FIXME: inefficient + bindings.bindings->sort(); + if (auto j = bindings.bindings->get(nameSym)) state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ - v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), i.pos)); - v.attrs->sort(); // FIXME: inefficient + bindings.insert(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), i.pos); + sort = true; } - v.attrs->pos = pos; + bindings.bindings->pos = pos; + + v.mkAttrs(sort ? bindings.finish() : bindings.alreadySorted()); } @@ -1426,21 +1421,21 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) for (auto & i : attrPath) { state.nrLookups++; - Bindings::iterator j; + const Attr * j; auto name = getName(i, state, env); if (def) { state.forceValue(*vAttrs, pos); if (vAttrs->type() != nAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + !(j = vAttrs->attrs()->get(name))) { def->eval(state, env, v); return; } } else { state.forceAttrs(*vAttrs, pos, "while selecting an attribute"); - if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + if (!(j = vAttrs->attrs()->get(name))) { std::set allAttrNames; - for (auto & attr : *vAttrs->attrs) + for (auto & attr : *vAttrs->attrs()) allAttrNames.insert(state.symbols[attr.name]); auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); state.error("attribute '%1%' missing", state.symbols[name]) @@ -1478,15 +1473,15 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) for (auto & i : attrPath) { state.forceValue(*vAttrs, getPos()); - Bindings::iterator j; + const Attr * j; auto name = getName(i, state, env); - if (vAttrs->type() != nAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + if (vAttrs->type() == nAttrs && + (j = vAttrs->attrs()->get(name))) { + vAttrs = j->value; + } else { v.mkBool(false); return; - } else { - vAttrs = j->value; } } @@ -1538,19 +1533,19 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } }; - Attr * functor; + const Attr * functor; while (nrArgs > 0) { if (vCur.isLambda()) { - ExprLambda & lambda(*vCur.lambda.fun); + ExprLambda & lambda(*vCur.payload.lambda.fun); auto size = (!lambda.arg ? 0 : 1) + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); Env & env2(allocEnv(size)); - env2.up = vCur.lambda.env; + env2.up = vCur.payload.lambda.env; Displacement displ = 0; @@ -1572,7 +1567,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & argument has a default, use the default. */ size_t attrsUsed = 0; for (auto & i : lambda.formals->formals) { - auto j = args[0]->attrs->get(i.name); + auto j = args[0]->attrs()->get(i.name); if (!j) { if (!i.def) { error("function '%1%' called without required argument '%2%'", @@ -1580,7 +1575,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & symbols[i.name]) .atPos(lambda.pos) .withTrace(pos, "from call site") - .withFrame(*fun.lambda.env, lambda) + .withFrame(*fun.payload.lambda.env, lambda) .debugThrow(); } env2.values[displ++] = i.def->maybeThunk(*this, env2); @@ -1592,10 +1587,10 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* Check that each actual argument is listed as a formal argument (unless the attribute match specifies a `...'). */ - if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) { + if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs()->size()) { /* Nope, so show the first unexpected argument to the user. */ - for (auto & i : *args[0]->attrs) + for (auto & i : *args[0]->attrs()) if (!lambda.formals->has(i.name)) { std::set formalNames; for (auto & formal : lambda.formals->formals) @@ -1607,7 +1602,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & .atPos(lambda.pos) .withTrace(pos, "from call site") .withSuggestions(suggestions) - .withFrame(*fun.lambda.env, lambda) + .withFrame(*fun.payload.lambda.env, lambda) .debugThrow(); } abort(); // can't happen @@ -1649,7 +1644,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & else if (vCur.isPrimOp()) { - size_t argsLeft = vCur.primOp->arity; + size_t argsLeft = vCur.primOp()->arity; if (nrArgs < argsLeft) { /* We don't have enough arguments, so create a tPrimOpApp chain. */ @@ -1657,7 +1652,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & return; } else { /* We have all the arguments, so call the primop. */ - auto * fn = vCur.primOp; + auto * fn = vCur.primOp(); nrPrimOpCalls++; if (countCalls) primOpCalls[fn->name]++; @@ -1680,10 +1675,10 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & Value * primOp = &vCur; while (primOp->isPrimOpApp()) { argsDone++; - primOp = primOp->primOpApp.left; + primOp = primOp->payload.primOpApp.left; } assert(primOp->isPrimOp()); - auto arity = primOp->primOp->arity; + auto arity = primOp->primOp()->arity; auto argsLeft = arity - argsDone; if (nrArgs < argsLeft) { @@ -1696,13 +1691,13 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & Value * vArgs[maxPrimOpArity]; auto n = argsDone; - for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) - vArgs[--n] = arg->primOpApp.right; + for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->payload.primOpApp.left) + vArgs[--n] = arg->payload.primOpApp.right; for (size_t i = 0; i < argsLeft; ++i) vArgs[argsDone + i] = args[i]; - auto fn = primOp->primOp; + auto fn = primOp->primOp(); nrPrimOpCalls++; if (countCalls) primOpCalls[fn->name]++; @@ -1722,7 +1717,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } } - else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) { + else if (vCur.type() == nAttrs && (functor = vCur.attrs()->get(sFunctor))) { /* 'vCur' may be allocated on the stack of the calling function, but for functors we may keep a reference, so heap-allocate a copy and use that instead. */ @@ -1797,8 +1792,8 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) forceValue(fun, pos); if (fun.type() == nAttrs) { - auto found = fun.attrs->find(sFunctor); - if (found != fun.attrs->end()) { + auto found = fun.attrs()->find(sFunctor); + if (found != fun.attrs()->end()) { Value * v = allocValue(); callFunction(*found->value, fun, *v, pos); forceValue(*v, pos); @@ -1806,14 +1801,14 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } } - if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) { + if (!fun.isLambda() || !fun.payload.lambda.fun->hasFormals()) { res = fun; return; } - auto attrs = buildBindings(std::max(static_cast(fun.lambda.fun->formals->formals.size()), args.size())); + auto attrs = buildBindings(std::max(static_cast(fun.payload.lambda.fun->formals->formals.size()), args.size())); - if (fun.lambda.fun->formals->ellipsis) { + if (fun.payload.lambda.fun->formals->ellipsis) { // If the formals have an ellipsis (eg the function accepts extra args) pass // all available automatic arguments (which includes arguments specified on // the command line via --arg/--argstr) @@ -1821,9 +1816,9 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) attrs.insert(v); } else { // Otherwise, only pass the arguments that the function accepts - for (auto & i : fun.lambda.fun->formals->formals) { - Bindings::iterator j = args.find(i.name); - if (j != args.end()) { + for (auto & i : fun.payload.lambda.fun->formals->formals) { + auto j = args.get(i.name); + if (j) { attrs.insert(*j); } else if (!i.def) { error(R"(cannot evaluate a function that has an argument without a value ('%1%') @@ -1831,7 +1826,7 @@ Nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with '--arg' or '--argstr'. See https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name]) - .atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow(); + .atPos(i.pos).withFrame(*fun.payload.lambda.env, *fun.payload.lambda.fun).debugThrow(); } } } @@ -1916,17 +1911,17 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) state.nrOpUpdates++; - if (v1.attrs->size() == 0) { v = v2; return; } - if (v2.attrs->size() == 0) { v = v1; return; } + if (v1.attrs()->size() == 0) { v = v2; return; } + if (v2.attrs()->size() == 0) { v = v1; return; } - auto attrs = state.buildBindings(v1.attrs->size() + v2.attrs->size()); + auto attrs = state.buildBindings(v1.attrs()->size() + v2.attrs()->size()); /* Merge the sets, preferring values from the second set. Make sure to keep the resulting vector in sorted order. */ - Bindings::iterator i = v1.attrs->begin(); - Bindings::iterator j = v2.attrs->begin(); + auto i = v1.attrs()->begin(); + auto j = v2.attrs()->begin(); - while (i != v1.attrs->end() && j != v2.attrs->end()) { + while (i != v1.attrs()->end() && j != v2.attrs()->end()) { if (i->name == j->name) { attrs.insert(*j); ++i; ++j; @@ -1937,12 +1932,12 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) attrs.insert(*j++); } - while (i != v1.attrs->end()) attrs.insert(*i++); - while (j != v2.attrs->end()) attrs.insert(*j++); + while (i != v1.attrs()->end()) attrs.insert(*i++); + while (j != v2.attrs()->end()) attrs.insert(*j++); v.mkAttrs(attrs.alreadySorted()); - state.nrOpUpdateValuesCopied += v.attrs->size(); + state.nrOpUpdateValuesCopied += v.attrs()->size(); } @@ -2034,19 +2029,19 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) if (firstType == nInt) { if (vTmp.type() == nInt) { - n += vTmp.integer; + n += vTmp.integer(); } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; firstType = nFloat; nf = n; - nf += vTmp.fpoint; + nf += vTmp.fpoint(); } else state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { - nf += vTmp.integer; + nf += vTmp.integer(); } else if (vTmp.type() == nFloat) { - nf += vTmp.fpoint; + nf += vTmp.fpoint(); } else state.error("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else { @@ -2119,11 +2114,11 @@ void EvalState::forceValueDeep(Value & v) forceValue(v, v.determinePos(noPos)); if (v.type() == nAttrs) { - for (auto & i : *v.attrs) + for (auto & i : *v.attrs()) try { // If the value is a thunk, we're evaling. Otherwise no trace necessary. auto dts = debugRepl && i.value->isThunk() - ? makeDebugTraceStacker(*this, *i.value->thunk.expr, *i.value->thunk.env, positions[i.pos], + ? makeDebugTraceStacker(*this, *i.value->payload.thunk.expr, *i.value->payload.thunk.env, positions[i.pos], "while evaluating the attribute '%1%'", symbols[i.name]) : nullptr; @@ -2154,13 +2149,13 @@ NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCt showType(v), ValuePrinter(*this, v, errorPrintOptions) ).atPos(pos).debugThrow(); - return v.integer; + return v.integer(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; } - return v.integer; + return v.integer(); } @@ -2169,14 +2164,14 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err try { forceValue(v, pos); if (v.type() == nInt) - return v.integer; + return v.integer(); else if (v.type() != nFloat) error( "expected a float but found %1%: %2%", showType(v), ValuePrinter(*this, v, errorPrintOptions) ).atPos(pos).debugThrow(); - return v.fpoint; + return v.fpoint(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -2194,19 +2189,19 @@ bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx showType(v), ValuePrinter(*this, v, errorPrintOptions) ).atPos(pos).debugThrow(); - return v.boolean; + return v.boolean(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; } - return v.boolean; + return v.boolean(); } bool EvalState::isFunctor(Value & fun) { - return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); + return fun.type() == nAttrs && fun.attrs()->find(sFunctor) != fun.attrs()->end(); } @@ -2247,8 +2242,8 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string void copyContext(const Value & v, NixStringContext & context) { - if (v.string.context) - for (const char * * p = v.string.context; *p; ++p) + if (v.payload.string.context) + for (const char * * p = v.payload.string.context; *p; ++p) context.insert(NixStringContextElem::parse(*p)); } @@ -2274,8 +2269,8 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s bool EvalState::isDerivation(Value & v) { if (v.type() != nAttrs) return false; - Bindings::iterator i = v.attrs->find(sType); - if (i == v.attrs->end()) return false; + auto i = v.attrs()->get(sType); + if (!i) return false; forceValue(*i->value, i->pos); if (i->value->type() != nString) return false; return i->value->string_view().compare("derivation") == 0; @@ -2285,8 +2280,8 @@ bool EvalState::isDerivation(Value & v) std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore, bool copyToStore) { - auto i = v.attrs->find(sToString); - if (i != v.attrs->end()) { + auto i = v.attrs()->find(sToString); + if (i != v.attrs()->end()) { Value v1; callFunction(*i->value, v, v1, pos); return coerceToString(pos, v1, context, @@ -2318,7 +2313,7 @@ BackedStringView EvalState::coerceToString( !canonicalizePath && !copyToStore ? // FIXME: hack to preserve path literals that end in a // slash, as in /foo/${x}. - v._path.path + v.payload.path.path : copyToStore ? store->printStorePath(copyPathToStore(context, v.path())) : std::string(v.path().path.abs()); @@ -2328,8 +2323,8 @@ BackedStringView EvalState::coerceToString( auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); if (maybeString) return std::move(*maybeString); - auto i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) { + auto i = v.attrs()->find(sOutPath); + if (i == v.attrs()->end()) { error( "cannot coerce %1% to a string: %2%", showType(v), @@ -2344,7 +2339,7 @@ BackedStringView EvalState::coerceToString( if (v.type() == nExternal) { try { - return v.external->coerceToString(*this, pos, context, coerceMore, copyToStore); + return v.external()->coerceToString(*this, pos, context, coerceMore, copyToStore); } catch (Error & e) { e.addTrace(nullptr, errorCtx); throw; @@ -2354,10 +2349,10 @@ BackedStringView EvalState::coerceToString( if (coerceMore) { /* Note that `false' is represented as an empty string for shell scripting convenience, just like `null'. */ - if (v.type() == nBool && v.boolean) return "1"; - if (v.type() == nBool && !v.boolean) return ""; - if (v.type() == nInt) return std::to_string(v.integer); - if (v.type() == nFloat) return std::to_string(v.fpoint); + if (v.type() == nBool && v.boolean()) return "1"; + if (v.type() == nBool && !v.boolean()) return ""; + if (v.type() == nInt) return std::to_string(v.integer()); + if (v.type() == nFloat) return std::to_string(v.fpoint()); if (v.type() == nNull) return ""; if (v.isList()) { @@ -2436,8 +2431,8 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext /* Similarly, handle __toString where the result may be a path value. */ if (v.type() == nAttrs) { - auto i = v.attrs->find(sToString); - if (i != v.attrs->end()) { + auto i = v.attrs()->find(sToString); + if (i != v.attrs()->end()) { Value v1; callFunction(*i->value, v, v1, pos); return coerceToPath(pos, v1, context, errorCtx); @@ -2531,19 +2526,19 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v // Special case type-compatibility between float and int if (v1.type() == nInt && v2.type() == nFloat) - return v1.integer == v2.fpoint; + return v1.integer() == v2.fpoint(); if (v1.type() == nFloat && v2.type() == nInt) - return v1.fpoint == v2.integer; + return v1.fpoint() == v2.integer(); // All other types are not compatible with each other. if (v1.type() != v2.type()) return false; switch (v1.type()) { case nInt: - return v1.integer == v2.integer; + return v1.integer() == v2.integer(); case nBool: - return v1.boolean == v2.boolean; + return v1.boolean() == v2.boolean(); case nString: return strcmp(v1.c_str(), v2.c_str()) == 0; @@ -2551,8 +2546,8 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nPath: return // FIXME: compare accessors by their fingerprint. - v1._path.accessor == v2._path.accessor - && strcmp(v1._path.path, v2._path.path) == 0; + v1.payload.path.accessor == v2.payload.path.accessor + && strcmp(v1.payload.path.path, v2.payload.path.path) == 0; case nNull: return true; @@ -2567,17 +2562,17 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v /* If both sets denote a derivation (type = "derivation"), then compare their outPaths. */ if (isDerivation(v1) && isDerivation(v2)) { - Bindings::iterator i = v1.attrs->find(sOutPath); - Bindings::iterator j = v2.attrs->find(sOutPath); - if (i != v1.attrs->end() && j != v2.attrs->end()) + auto i = v1.attrs()->get(sOutPath); + auto j = v2.attrs()->get(sOutPath); + if (i && j) return eqValues(*i->value, *j->value, pos, errorCtx); } - if (v1.attrs->size() != v2.attrs->size()) return false; + if (v1.attrs()->size() != v2.attrs()->size()) return false; /* Otherwise, compare the attributes one by one. */ - Bindings::iterator i, j; - for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j) + Bindings::const_iterator i, j; + for (i = v1.attrs()->begin(), j = v2.attrs()->begin(); i != v1.attrs()->end(); ++i, ++j) if (i->name != j->name || !eqValues(*i->value, *j->value, pos, errorCtx)) return false; @@ -2589,10 +2584,10 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return false; case nExternal: - return *v1.external == *v2.external; + return *v1.external() == *v2.external(); case nFloat: - return v1.fpoint == v2.fpoint; + return v1.fpoint() == v2.fpoint(); case nThunk: // Must not be left by forceValue default: diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f15d19653..0e569de26 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -86,7 +86,7 @@ struct PrimOp void check(); }; -std::ostream & operator<<(std::ostream & output, PrimOp & primOp); +std::ostream & operator<<(std::ostream & output, const PrimOp & primOp); /** * Info about a constant @@ -635,9 +635,6 @@ public: inline Value * allocValue(); inline Env & allocEnv(size_t size); - Value * allocAttr(Value & vAttrs, Symbol name); - Value * allocAttr(Value & vAttrs, std::string_view name); - Bindings * allocBindings(size_t capacity); BindingsBuilder buildBindings(size_t capacity) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index bca473453..1f0326eb5 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -111,7 +111,7 @@ static FlakeInput parseFlakeInput(EvalState & state, fetchers::Attrs attrs; std::optional url; - for (nix::Attr attr : *(value->attrs)) { + for (auto & attr : *value->attrs()) { try { if (attr.name == sUrl) { expectType(state, nString, *attr.value, attr.pos); @@ -119,7 +119,7 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); - input.isFlake = attr.value->boolean; + input.isFlake = attr.value->boolean(); } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { @@ -136,10 +136,10 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(state.symbols[attr.name], attr.value->c_str()); break; case nBool: - attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean }); + attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); break; case nInt: - attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer); + attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer()); break; default: if (attr.name == state.symbols.create("publicKeys")) { @@ -189,7 +189,7 @@ static std::map parseFlakeInputs( expectType(state, nAttrs, *value, pos); - for (nix::Attr & inputAttr : *(*value).attrs) { + for (auto & inputAttr : *value->attrs()) { inputs.emplace(state.symbols[inputAttr.name], parseFlakeInput(state, state.symbols[inputAttr.name], @@ -223,23 +223,23 @@ static Flake readFlake( .path = flakePath, }; - if (auto description = vInfo.attrs->get(state.sDescription)) { + if (auto description = vInfo.attrs()->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); flake.description = description->value->c_str(); } auto sInputs = state.symbols.create("inputs"); - if (auto inputs = vInfo.attrs->get(sInputs)) + if (auto inputs = vInfo.attrs()->get(sInputs)) flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME auto sOutputs = state.symbols.create("outputs"); - if (auto outputs = vInfo.attrs->get(sOutputs)) { + if (auto outputs = vInfo.attrs()->get(sOutputs)) { expectType(state, nFunction, *outputs->value, outputs->pos); - if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) { - for (auto & formal : outputs->value->lambda.fun->formals->formals) { + if (outputs->value->isLambda() && outputs->value->payload.lambda.fun->hasFormals()) { + for (auto & formal : outputs->value->payload.lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(state.symbols[formal.name], FlakeInput { .ref = parseFlakeRef(state.symbols[formal.name]) @@ -252,10 +252,10 @@ static Flake readFlake( auto sNixConfig = state.symbols.create("nixConfig"); - if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { + if (auto nixConfig = vInfo.attrs()->get(sNixConfig)) { expectType(state, nAttrs, *nixConfig->value, nixConfig->pos); - for (auto & setting : *nixConfig->value->attrs) { + for (auto & setting : *nixConfig->value->attrs()) { forceTrivialValue(state, *setting.value, setting.pos); if (setting.value->type() == nString) flake.config.settings.emplace( @@ -291,7 +291,7 @@ static Flake readFlake( } } - for (auto & attr : *vInfo.attrs) { + for (auto & attr : *vInfo.attrs()) { if (attr.name != state.sDescription && attr.name != sInputs && attr.name != sOutputs && @@ -879,17 +879,17 @@ static void prim_flakeRefToString( state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.flakeRefToString"); fetchers::Attrs attrs; - for (const auto & attr : *args[0]->attrs) { + for (const auto & attr : *args[0]->attrs()) { auto t = attr.value->type(); if (t == nInt) { attrs.emplace(state.symbols[attr.name], - (uint64_t) attr.value->integer); + (uint64_t) attr.value->integer()); } else if (t == nBool) { attrs.emplace(state.symbols[attr.name], - Explicit { attr.value->boolean }); + Explicit { attr.value->boolean() }); } else if (t == nString) { attrs.emplace(state.symbols[attr.name], - std::string(attr.value->string_view())); + std::string(attr.value->string_view())); } else { state.error( "flake reference attribute sets may only contain integers, Booleans, " diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index e9ed1ef08..cf10ed84a 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -11,7 +11,7 @@ namespace nix { -PackageInfo::PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs) +PackageInfo::PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs) : state(&state), attrs(attrs), attrPath(std::move(attrPath)) { } @@ -69,12 +69,11 @@ std::string PackageInfo::querySystem() const std::optional PackageInfo::queryDrvPath() const { if (!drvPath && attrs) { - Bindings::iterator i = attrs->find(state->sDrvPath); NixStringContext context; - if (i == attrs->end()) - drvPath = {std::nullopt}; - else + if (auto i = attrs->get(state->sDrvPath)) drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")}; + else + drvPath = {std::nullopt}; } return drvPath.value_or(std::nullopt); } @@ -91,7 +90,7 @@ StorePath PackageInfo::requireDrvPath() const StorePath PackageInfo::queryOutPath() const { if (!outPath && attrs) { - Bindings::iterator i = attrs->find(state->sOutPath); + auto i = attrs->find(state->sOutPath); NixStringContext context; if (i != attrs->end()) outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation"); @@ -106,8 +105,8 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT { if (outputs.empty()) { /* Get the ‘outputs’ list. */ - Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { + const Attr * i; + if (attrs && (i = attrs->get(state->sOutputs))) { state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation"); /* For each output... */ @@ -116,13 +115,13 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT if (withPaths) { /* Evaluate the corresponding set. */ - Bindings::iterator out = attrs->find(state->symbols.create(output)); - if (out == attrs->end()) continue; // FIXME: throw error? + auto out = attrs->get(state->symbols.create(output)); + if (!out) continue; // FIXME: throw error? state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation"); /* And evaluate its ‘outPath’ attribute. */ - Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); - if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? + auto outPath = out->value->attrs()->get(state->sOutPath); + if (!outPath) continue; // FIXME: throw error? NixStringContext context; outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation")); } else @@ -135,8 +134,8 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT if (!onlyOutputsToInstall || !attrs) return outputs; - Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) { + const Attr * i; + if (attrs && (i = attrs->get(state->sOutputSpecified)) && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) { Outputs result; auto out = outputs.find(queryOutputName()); if (out == outputs.end()) @@ -167,21 +166,21 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT std::string PackageInfo::queryOutputName() const { if (outputName == "" && attrs) { - Bindings::iterator i = attrs->find(state->sOutputName); - outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : ""; + auto i = attrs->get(state->sOutputName); + outputName = i ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : ""; } return outputName; } -Bindings * PackageInfo::getMeta() +const Bindings * PackageInfo::getMeta() { if (meta) return meta; if (!attrs) return 0; - Bindings::iterator a = attrs->find(state->sMeta); - if (a == attrs->end()) return 0; + auto a = attrs->get(state->sMeta); + if (!a) return 0; state->forceAttrs(*a->value, a->pos, "while evaluating the 'meta' attribute of a derivation"); - meta = a->value->attrs; + meta = a->value->attrs(); return meta; } @@ -205,9 +204,8 @@ bool PackageInfo::checkMeta(Value & v) return true; } else if (v.type() == nAttrs) { - Bindings::iterator i = v.attrs->find(state->sOutPath); - if (i != v.attrs->end()) return false; - for (auto & i : *v.attrs) + if (v.attrs()->get(state->sOutPath)) return false; + for (auto & i : *v.attrs()) if (!checkMeta(*i.value)) return false; return true; } @@ -219,8 +217,8 @@ bool PackageInfo::checkMeta(Value & v) Value * PackageInfo::queryMeta(const std::string & name) { if (!getMeta()) return 0; - Bindings::iterator a = meta->find(state->symbols.create(name)); - if (a == meta->end() || !checkMeta(*a->value)) return 0; + auto a = meta->get(state->symbols.create(name)); + if (!a || !checkMeta(*a->value)) return 0; return a->value; } @@ -237,7 +235,7 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type() == nInt) return v->integer; + if (v->type() == nInt) return v->integer(); if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ @@ -251,7 +249,7 @@ NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type() == nFloat) return v->fpoint; + if (v->type() == nFloat) return v->fpoint(); if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ @@ -266,7 +264,7 @@ bool PackageInfo::queryMetaBool(const std::string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type() == nBool) return v->boolean; + if (v->type() == nBool) return v->boolean(); if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ @@ -292,7 +290,7 @@ void PackageInfo::setMeta(const std::string & name, Value * v) /* Cache for already considered attrsets. */ -typedef std::set Done; +typedef std::set Done; /* Evaluate value `v'. If it evaluates to a set of type `derivation', @@ -309,9 +307,9 @@ static bool getDerivation(EvalState & state, Value & v, /* Remove spurious duplicates (e.g., a set like `rec { x = derivation {...}; y = x;}'. */ - if (!done.insert(v.attrs).second) return false; + if (!done.insert(v.attrs()).second) return false; - PackageInfo drv(state, attrPath, v.attrs); + PackageInfo drv(state, attrPath, v.attrs()); drv.queryName(); @@ -361,14 +359,14 @@ static void getDerivations(EvalState & state, Value & vIn, /* !!! undocumented hackery to support combining channels in nix-env.cc. */ - bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); + bool combineChannels = v.attrs()->get(state.symbols.create("_combineChannels")); /* Consider the attributes in sorted order to get more deterministic behaviour in nix-env operations (e.g. when there are names clashes between derivations, the derivation bound to the attribute with the "lower" name should take precedence). */ - for (auto & i : v.attrs->lexicographicOrder(state.symbols)) { + for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) { debug("evaluating attribute '%1%'", state.symbols[i->name]); if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex)) continue; @@ -380,8 +378,8 @@ static void getDerivations(EvalState & state, Value & vIn, should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ if (i->value->type() == nAttrs) { - Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); - if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) + auto j = i->value->attrs()->get(state.sRecurseForDerivations); + if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index e8c1190f7..db3eedb05 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -33,9 +33,9 @@ private: */ bool failed = false; - Bindings * attrs = nullptr, * meta = nullptr; + const Bindings * attrs = nullptr, * meta = nullptr; - Bindings * getMeta(); + const Bindings * getMeta(); bool checkMeta(Value & v); @@ -46,7 +46,7 @@ public: std::string attrPath; PackageInfo(EvalState & state) : state(&state) { }; - PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs); + PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs); PackageInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs); std::string queryName() const; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 5bdc466eb..c1e2b0448 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -28,12 +28,12 @@ void Expr::show(const SymbolTable & symbols, std::ostream & str) const void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const { - str << v.integer; + str << v.integer(); } void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const { - str << v.fpoint; + str << v.fpoint(); } void ExprString::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d0fcfd194..630ee8071 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -216,13 +216,13 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v else { state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport"); - Env * env = &state.allocEnv(vScope->attrs->size()); + Env * env = &state.allocEnv(vScope->attrs()->size()); env->up = &state.baseEnv; - auto staticEnv = std::make_shared(nullptr, state.staticBaseEnv.get(), vScope->attrs->size()); + auto staticEnv = std::make_shared(nullptr, state.staticBaseEnv.get(), vScope->attrs()->size()); unsigned int displ = 0; - for (auto & attr : *vScope->attrs) { + for (auto & attr : *vScope->attrs()) { staticEnv->vars.emplace_back(attr.name, displ); env->values[displ++] = attr.value; } @@ -411,7 +411,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Val case nList: t = "list"; break; case nFunction: t = "lambda"; break; case nExternal: - t = args[0]->external->typeOf(); + t = args[0]->external()->typeOf(); break; case nFloat: t = "float"; break; case nThunk: abort(); @@ -575,9 +575,9 @@ struct CompareValues { try { if (v1->type() == nFloat && v2->type() == nInt) - return v1->fpoint < v2->integer; + return v1->fpoint() < v2->integer(); if (v1->type() == nInt && v2->type() == nFloat) - return v1->integer < v2->fpoint; + return v1->integer() < v2->fpoint(); if (v1->type() != v2->type()) state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); // Allow selecting a subset of enum values @@ -585,16 +585,16 @@ struct CompareValues #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v1->type()) { case nInt: - return v1->integer < v2->integer; + return v1->integer() < v2->integer(); case nFloat: - return v1->fpoint < v2->fpoint; + return v1->fpoint() < v2->fpoint(); case nString: return strcmp(v1->c_str(), v2->c_str()) < 0; case nPath: // Note: we don't take the accessor into account // since it's not obvious how to compare them in a // reproducible way. - return strcmp(v1->_path.path, v2->_path.path) < 0; + return strcmp(v1->payload.path.path, v2->payload.path.path) < 0; case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -626,13 +626,13 @@ typedef std::list ValueList; #endif -static Bindings::iterator getAttr( +static Bindings::const_iterator getAttr( EvalState & state, Symbol attrSym, - Bindings * attrSet, + const Bindings * attrSet, std::string_view errorCtx) { - Bindings::iterator value = attrSet->find(attrSym); + auto value = attrSet->find(attrSym); if (value == attrSet->end()) { state.error("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow(); } @@ -644,7 +644,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a state.forceAttrs(*args[0], noPos, "while evaluating the first argument passed to builtins.genericClosure"); /* Get the start set. */ - Bindings::iterator startSet = getAttr(state, state.sStartSet, args[0]->attrs, "in the attrset passed as argument to builtins.genericClosure"); + auto startSet = getAttr(state, state.sStartSet, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure"); state.forceList(*startSet->value, noPos, "while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure"); @@ -658,7 +658,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a } /* Get the operator. */ - Bindings::iterator op = getAttr(state, state.sOperator, args[0]->attrs, "in the attrset passed as argument to builtins.genericClosure"); + auto op = getAttr(state, state.sOperator, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure"); state.forceFunction(*op->value, noPos, "while evaluating the 'operator' attribute passed as argument to builtins.genericClosure"); /* Construct the closure by applying the operator to elements of @@ -675,7 +675,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a state.forceAttrs(*e, noPos, "while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure"); - Bindings::iterator key = getAttr(state, state.sKey, e->attrs, "in one of the attrsets generated by (or initially passed to) builtins.genericClosure"); + auto key = getAttr(state, state.sKey, e->attrs(), "in one of the attrsets generated by (or initially passed to) builtins.genericClosure"); state.forceValue(*key->value, noPos); if (!doneKeys.insert(key->value).second) continue; @@ -1041,7 +1041,11 @@ static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Val * Derivations *************************************************************/ -static void derivationStrictInternal(EvalState & state, const std::string & name, Bindings * attrs, Value & v); +static void derivationStrictInternal( + EvalState & state, + const std::string & name, + const Bindings * attrs, + Value & v); /* Construct (as a unobservable side effect) a Nix derivation expression that performs the derivation described by the argument @@ -1054,10 +1058,10 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * { state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.derivationStrict"); - Bindings * attrs = args[0]->attrs; + auto attrs = args[0]->attrs(); /* Figure out the name first (for stack backtraces). */ - Bindings::iterator nameAttr = getAttr(state, state.sName, attrs, "in the attrset passed as argument to builtins.derivationStrict"); + auto nameAttr = getAttr(state, state.sName, attrs, "in the attrset passed as argument to builtins.derivationStrict"); std::string drvName; try { @@ -1096,8 +1100,11 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * } } -static void derivationStrictInternal(EvalState & state, const std::string & -drvName, Bindings * attrs, Value & v) +static void derivationStrictInternal( + EvalState & state, + const std::string & drvName, + const Bindings * attrs, + Value & v) { /* Check whether attributes should be passed as a JSON file. */ using nlohmann::json; @@ -1670,11 +1677,11 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.findFile"); std::string prefix; - Bindings::iterator i = v2->attrs->find(state.sPrefix); - if (i != v2->attrs->end()) + auto i = v2->attrs()->find(state.sPrefix); + if (i != v2->attrs()->end()) prefix = state.forceStringNoCtx(*i->value, pos, "while evaluating the `prefix` attribute of an element of the list passed to builtins.findFile"); - i = getAttr(state, state.sPath, v2->attrs, "in an element of the __nixPath"); + i = getAttr(state, state.sPath, v2->attrs(), "in an element of the __nixPath"); NixStringContext context; auto path = state.coerceToString(pos, *i->value, context, @@ -2346,7 +2353,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value state.forceAttrs(*args[0], pos, "while evaluating the argument passed to 'builtins.path'"); - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *args[0]->attrs()) { auto n = state.symbols[attr.name]; if (n == "path") path.emplace(state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the 'path' attribute passed to 'builtins.path'")); @@ -2421,9 +2428,9 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, { state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.attrNames"); - auto list = state.buildList(args[0]->attrs->size()); + auto list = state.buildList(args[0]->attrs()->size()); - for (const auto & [n, i] : enumerate(*args[0]->attrs)) + for (const auto & [n, i] : enumerate(*args[0]->attrs())) (list[n] = state.allocValue())->mkString(state.symbols[i.name]); std::sort(list.begin(), list.end(), @@ -2449,9 +2456,9 @@ static void prim_attrValues(EvalState & state, const PosIdx pos, Value * * args, { state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.attrValues"); - auto list = state.buildList(args[0]->attrs->size()); + auto list = state.buildList(args[0]->attrs()->size()); - for (const auto & [n, i] : enumerate(*args[0]->attrs)) + for (const auto & [n, i] : enumerate(*args[0]->attrs())) list[n] = (Value *) &i; std::sort(list.begin(), list.end(), @@ -2482,10 +2489,10 @@ void prim_getAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v { auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getAttr"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.getAttr"); - Bindings::iterator i = getAttr( + auto i = getAttr( state, state.symbols.create(attr), - args[1]->attrs, + args[1]->attrs(), "in the attribute set under consideration" ); // !!! add to stack trace? @@ -2511,8 +2518,8 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * * { auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.unsafeGetAttrPos"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.unsafeGetAttrPos"); - Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); - if (i == args[1]->attrs->end()) + auto i = args[1]->attrs()->find(state.symbols.create(attr)); + if (i == args[1]->attrs()->end()) v.mkNull(); else state.mkPos(v, i->pos); @@ -2540,13 +2547,13 @@ static struct LazyPosAcessors { PrimOp primop_lineOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer)].line); + v.mkInt(state.positions[PosIdx(args[0]->integer())].line); } }; PrimOp primop_columnOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer)].column); + v.mkInt(state.positions[PosIdx(args[0]->integer())].column); } }; @@ -2577,7 +2584,7 @@ static void prim_hasAttr(EvalState & state, const PosIdx pos, Value * * args, Va { auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hasAttr"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.hasAttr"); - v.mkBool(args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); + v.mkBool(args[1]->attrs()->find(state.symbols.create(attr)) != args[1]->attrs()->end()); } static RegisterPrimOp primop_hasAttr({ @@ -2627,9 +2634,9 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args /* Copy all attributes not in that set. Note that we don't need to sort v.attrs because it's a subset of an already sorted vector. */ - auto attrs = state.buildBindings(args[0]->attrs->size()); + auto attrs = state.buildBindings(args[0]->attrs()->size()); std::set_difference( - args[0]->attrs->begin(), args[0]->attrs->end(), + args[0]->attrs()->begin(), args[0]->attrs()->end(), names.begin(), names.end(), std::back_inserter(attrs)); v.mkAttrs(attrs.alreadySorted()); @@ -2667,13 +2674,13 @@ static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value * * args for (auto v2 : args[0]->listItems()) { state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.listToAttrs"); - Bindings::iterator j = getAttr(state, state.sName, v2->attrs, "in a {name=...; value=...;} pair"); + auto j = getAttr(state, state.sName, v2->attrs(), "in a {name=...; value=...;} pair"); auto name = state.forceStringNoCtx(*j->value, j->pos, "while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs"); auto sym = state.symbols.create(name); if (seen.insert(sym).second) { - Bindings::iterator j2 = getAttr(state, state.sValue, v2->attrs, "in a {name=...; value=...;} pair"); + auto j2 = getAttr(state, state.sValue, v2->attrs(), "in a {name=...; value=...;} pair"); attrs.insert(sym, j2->value, j2->pos); } } @@ -2717,8 +2724,8 @@ static void prim_intersectAttrs(EvalState & state, const PosIdx pos, Value * * a state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.intersectAttrs"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.intersectAttrs"); - Bindings &left = *args[0]->attrs; - Bindings &right = *args[1]->attrs; + auto & left = *args[0]->attrs(); + auto & right = *args[1]->attrs(); auto attrs = state.buildBindings(std::min(left.size(), right.size())); @@ -2762,14 +2769,14 @@ static void prim_intersectAttrs(EvalState & state, const PosIdx pos, Value * * a if (left.size() < right.size()) { for (auto & l : left) { - Bindings::iterator r = right.find(l.name); + auto r = right.find(l.name); if (r != right.end()) attrs.insert(*r); } } else { for (auto & r : right) { - Bindings::iterator l = left.find(r.name); + auto l = left.find(r.name); if (l != left.end()) attrs.insert(r); } @@ -2800,8 +2807,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V for (auto v2 : args[1]->listItems()) { state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs"); - Bindings::iterator i = v2->attrs->find(attrName); - if (i != v2->attrs->end()) + if (auto i = v2->attrs()->get(attrName)) res[found++] = i->value; } @@ -2838,13 +2844,13 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value * * arg if (!args[0]->isLambda()) state.error("'functionArgs' requires a function").atPos(pos).debugThrow(); - if (!args[0]->lambda.fun->hasFormals()) { + if (!args[0]->payload.lambda.fun->hasFormals()) { v.mkAttrs(&state.emptyBindings); return; } - auto attrs = state.buildBindings(args[0]->lambda.fun->formals->formals.size()); - for (auto & i : args[0]->lambda.fun->formals->formals) + auto attrs = state.buildBindings(args[0]->payload.lambda.fun->formals->formals.size()); + for (auto & i : args[0]->payload.lambda.fun->formals->formals) attrs.insert(i.name, state.getBool(i.def), i.pos); v.mkAttrs(attrs); } @@ -2871,9 +2877,9 @@ static void prim_mapAttrs(EvalState & state, const PosIdx pos, Value * * args, V { state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.mapAttrs"); - auto attrs = state.buildBindings(args[1]->attrs->size()); + auto attrs = state.buildBindings(args[1]->attrs()->size()); - for (auto & i : *args[1]->attrs) { + for (auto & i : *args[1]->attrs()) { Value * vName = state.allocValue(); Value * vFun2 = state.allocValue(); vName->mkString(state.symbols[i.name]); @@ -2923,7 +2929,7 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg for (auto & vElem : listItems) { state.forceAttrs(*vElem, noPos, "while evaluating a value of the list passed as second argument to builtins.zipAttrsWith"); - for (auto & attr : *vElem->attrs) + for (auto & attr : *vElem->attrs()) attrsSeen.try_emplace(attr.name).first->second.size++; } @@ -2931,7 +2937,7 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg elem.list.emplace(state.buildList(elem.size)); for (auto & vElem : listItems) { - for (auto & attr : *vElem->attrs) { + for (auto & attr : *vElem->attrs()) { auto & item = attrsSeen.at(attr.name); (*item.list)[item.pos++] = attr.value; } @@ -3375,7 +3381,7 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value callFunction. */ /* TODO: (layus) this is absurd. An optimisation like this should be outside the lambda creation */ - if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) + if (args[0]->isPrimOp() && args[0]->primOp()->fun == prim_lessThan) return CompareValues(state, noPos, "while evaluating the ordering function passed to builtins.sort")(a, b); Value * vs[] = {a, b}; @@ -3866,18 +3872,17 @@ static RegisterPrimOp primop_hashString({ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash"); - auto &inputAttrs = args[0]->attrs; + auto inputAttrs = args[0]->attrs(); - Bindings::iterator iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'"); + auto iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'"); auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); - Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo")); + auto iteratorHashAlgo = inputAttrs->get(state.symbols.create("hashAlgo")); std::optional ha = std::nullopt; - if (iteratorHashAlgo != inputAttrs->end()) { + if (iteratorHashAlgo) ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); - } - Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'"); + auto iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs(), "while locating the attribute 'toHashFormat'"); HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI)); @@ -4623,7 +4628,7 @@ void EvalState::createBaseEnv() /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ - baseEnv.values[0]->attrs->sort(); + baseEnv.values[0]->payload.attrs->sort(); staticBaseEnv->sort(); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 75d9e147d..2d3013132 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -258,7 +258,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar auto sPath = state.symbols.create("path"); auto sAllOutputs = state.symbols.create("allOutputs"); - for (auto & i : *args[1]->attrs) { + for (auto & i : *args[1]->attrs()) { const auto & name = state.symbols[i.name]; if (!state.store->isStorePath(name)) state.error( @@ -269,17 +269,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar if (!settings.readOnlyMode) state.store->ensurePath(namePath); state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); - auto iter = i.value->attrs->find(sPath); - if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context")) + + if (auto attr = i.value->attrs()->get(sPath)) { + if (state.forceBool(*attr->value, attr->pos, "while evaluating the `path` attribute of a string context")) context.emplace(NixStringContextElem::Opaque { .path = namePath, }); } - iter = i.value->attrs->find(sAllOutputs); - if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, iter->pos, "while evaluating the `allOutputs` attribute of a string context")) { + if (auto attr = i.value->attrs()->get(sAllOutputs)) { + if (state.forceBool(*attr->value, attr->pos, "while evaluating the `allOutputs` attribute of a string context")) { if (!isDerivation(name)) { state.error( "tried to add all-outputs context of %s, which is not a derivation, to a string", @@ -292,17 +291,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar } } - iter = i.value->attrs->find(state.sOutputs); - if (iter != i.value->attrs->end()) { - state.forceList(*iter->value, iter->pos, "while evaluating the `outputs` attribute of a string context"); - if (iter->value->listSize() && !isDerivation(name)) { + if (auto attr = i.value->attrs()->get(state.sOutputs)) { + state.forceList(*attr->value, attr->pos, "while evaluating the `outputs` attribute of a string context"); + if (attr->value->listSize() && !isDerivation(name)) { state.error( "tried to add derivation output context of %s, which is not a derivation, to a string", name ).atPos(i.pos).debugThrow(); } - for (auto elem : iter->value->listItems()) { - auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); + for (auto elem : attr->value->listItems()) { + auto outputName = state.forceStringNoCtx(*elem, attr->pos, "while evaluating an output name within a string context"); context.emplace(NixStringContextElem::Built { .drvPath = makeConstantStorePathRef(namePath), .output = std::string { outputName }, diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index f51a6465d..fc5bb3145 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -121,7 +121,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg std::optional toPath; std::optional inputAddressedMaybe; - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *args[0]->attrs()) { const auto & attrName = state.symbols[attr.name]; auto attrHint = [&]() -> std::string { return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure"; diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index bfc19115a..d9ba6aa97 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -20,7 +20,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a if (args[0]->type() == nAttrs) { - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *args[0]->attrs()) { std::string_view n(state.symbols[attr.name]); if (n == "url") url = state.coerceToString(attr.pos, *attr.value, context, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 5061e40fd..409b36d81 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -97,7 +97,7 @@ static void fetchTree( fetchers::Attrs attrs; - if (auto aType = args[0]->attrs->get(state.sType)) { + if (auto aType = args[0]->attrs()->get(state.sType)) { if (type) state.error( "unexpected attribute 'type'" @@ -110,7 +110,7 @@ static void fetchTree( attrs.emplace("type", type.value()); - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *args[0]->attrs()) { if (attr.name == state.sType) continue; state.forceValue(*attr.value, attr.pos); if (attr.value->type() == nPath || attr.value->type() == nString) { @@ -121,9 +121,9 @@ static void fetchTree( : s); } else if (attr.value->type() == nBool) - attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean}); + attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); else if (attr.value->type() == nInt) - attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer)); + attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer())); else if (state.symbols[attr.name] == "publicKeys") { experimentalFeatureSettings.require(Xp::VerifiedFetches); attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); @@ -422,7 +422,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (args[0]->type() == nAttrs) { - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *args[0]->attrs()) { std::string_view n(state.symbols[attr.name]); if (n == "url") url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index 521250cec..5d55b45da 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -21,10 +21,10 @@ void printAmbiguous( } switch (v.type()) { case nInt: - str << v.integer; + str << v.integer(); break; case nBool: - printLiteralBool(str, v.boolean); + printLiteralBool(str, v.boolean()); break; case nString: printLiteralString(str, v.string_view()); @@ -36,11 +36,11 @@ void printAmbiguous( str << "null"; break; case nAttrs: { - if (seen && !v.attrs->empty() && !seen->insert(v.attrs).second) + if (seen && !v.attrs()->empty() && !seen->insert(v.attrs()).second) str << "«repeated»"; else { str << "{ "; - for (auto & i : v.attrs->lexicographicOrder(symbols)) { + for (auto & i : v.attrs()->lexicographicOrder(symbols)) { str << symbols[i->name] << " = "; printAmbiguous(*i->value, symbols, str, seen, depth - 1); str << "; "; @@ -87,10 +87,10 @@ void printAmbiguous( } break; case nExternal: - str << *v.external; + str << *v.external(); break; case nFloat: - str << v.fpoint; + str << v.fpoint(); break; default: printError("Nix evaluator internal error: printAmbiguous: invalid value type"); diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index f67e94750..7799a0bbe 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -223,7 +223,7 @@ private: { if (options.ansiColors) output << ANSI_CYAN; - output << v.integer; + output << v.integer(); if (options.ansiColors) output << ANSI_NORMAL; } @@ -232,7 +232,7 @@ private: { if (options.ansiColors) output << ANSI_CYAN; - output << v.fpoint; + output << v.fpoint(); if (options.ansiColors) output << ANSI_NORMAL; } @@ -241,7 +241,7 @@ private: { if (options.ansiColors) output << ANSI_CYAN; - printLiteralBool(output, v.boolean); + printLiteralBool(output, v.boolean()); if (options.ansiColors) output << ANSI_NORMAL; } @@ -271,10 +271,9 @@ private: void printDerivation(Value & v) { - Bindings::iterator i = v.attrs->find(state.sDrvPath); NixStringContext context; std::string storePath; - if (i != v.attrs->end()) + if (auto i = v.attrs()->get(state.sDrvPath)) storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); if (options.ansiColors) @@ -312,7 +311,7 @@ private: void printAttrs(Value & v, size_t depth) { - if (seen && !seen->insert(v.attrs).second) { + if (seen && !seen->insert(v.attrs()).second) { printRepeated(); return; } @@ -324,7 +323,7 @@ private: output << "{"; AttrVec sorted; - for (auto & i : *v.attrs) + for (auto & i : *v.attrs()) sorted.emplace_back(std::pair(state.symbols[i.name], i.value)); if (options.maxAttrs == std::numeric_limits::max()) @@ -423,18 +422,18 @@ private: if (v.isLambda()) { output << "lambda"; - if (v.lambda.fun) { - if (v.lambda.fun->name) { - output << " " << state.symbols[v.lambda.fun->name]; + if (v.payload.lambda.fun) { + if (v.payload.lambda.fun->name) { + output << " " << state.symbols[v.payload.lambda.fun->name]; } std::ostringstream s; - s << state.positions[v.lambda.fun->pos]; + s << state.positions[v.payload.lambda.fun->pos]; output << " @ " << filterANSIEscapes(s.str()); } } else if (v.isPrimOp()) { - if (v.primOp) - output << *v.primOp; + if (v.primOp()) + output << *v.primOp(); else output << "primop"; } else if (v.isPrimOpApp()) { @@ -480,7 +479,7 @@ private: void printExternal(Value & v) { - v.external->print(output); + v.external()->print(output); } void printUnknown() diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 3f877a7fd..936ecf078 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -22,11 +22,11 @@ json printValueAsJSON(EvalState & state, bool strict, switch (v.type()) { case nInt: - out = v.integer; + out = v.integer(); break; case nBool: - out = v.boolean; + out = v.boolean(); break; case nString: @@ -52,24 +52,20 @@ json printValueAsJSON(EvalState & state, bool strict, out = *maybeString; break; } - auto i = v.attrs->find(state.sOutPath); - if (i == v.attrs->end()) { + if (auto i = v.attrs()->get(state.sOutPath)) + return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); + else { out = json::object(); - StringSet names; - for (auto & j : *v.attrs) - names.emplace(state.symbols[j.name]); - for (auto & j : names) { - Attr & a(*v.attrs->find(state.symbols.create(j))); + for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { try { - out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + out[state.symbols[a->name]] = printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore); } catch (Error & e) { - e.addTrace(state.positions[a.pos], - HintFmt("while evaluating attribute '%1%'", j)); + e.addTrace(state.positions[a->pos], + HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); throw; } } - } else - return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); + } break; } @@ -90,11 +86,11 @@ json printValueAsJSON(EvalState & state, bool strict, } case nExternal: - return v.external->printValueAsJSON(state, strict, context, copyToStore); + return v.external()->printValueAsJSON(state, strict, context, copyToStore); break; case nFloat: - out = v.fpoint; + out = v.fpoint(); break; case nThunk: diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 5032115bb..1de8cdf84 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -32,23 +32,18 @@ static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen) + const Bindings & attrs, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen) { StringSet names; - for (auto & i : attrs) - names.emplace(state.symbols[i.name]); - - for (auto & i : names) { - Attr & a(*attrs.find(state.symbols.create(i))); - + for (auto & a : attrs.lexicographicOrder(state.symbols)) { XMLAttrs xmlAttrs; - xmlAttrs["name"] = i; - if (location && a.pos) posToXML(state, xmlAttrs, state.positions[a.pos]); + xmlAttrs["name"] = state.symbols[a->name]; + if (location && a->pos) posToXML(state, xmlAttrs, state.positions[a->pos]); XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen, a.pos); + *a->value, doc, context, drvsSeen, a->pos); } } @@ -64,11 +59,11 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, switch (v.type()) { case nInt: - doc.writeEmptyElement("int", singletonAttrs("value", fmt("%1%", v.integer))); + doc.writeEmptyElement("int", singletonAttrs("value", fmt("%1%", v.integer()))); break; case nBool: - doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); + doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean() ? "true" : "false")); break; case nString: @@ -89,18 +84,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (state.isDerivation(v)) { XMLAttrs xmlAttrs; - Bindings::iterator a = v.attrs->find(state.symbols.create("derivation")); - Path drvPath; - a = v.attrs->find(state.sDrvPath); - if (a != v.attrs->end()) { + if (auto a = v.attrs()->get(state.sDrvPath)) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->c_str(); } - a = v.attrs->find(state.sOutPath); - if (a != v.attrs->end()) { + if (auto a = v.attrs()->get(state.sOutPath)) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->c_str(); @@ -109,14 +100,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "derivation", xmlAttrs); if (drvPath != "" && drvsSeen.insert(drvPath).second) - showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + showAttrs(state, strict, location, *v.attrs(), doc, context, drvsSeen); else doc.writeEmptyElement("repeated"); } else { XMLOpenElement _(doc, "attrs"); - showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + showAttrs(state, strict, location, *v.attrs(), doc, context, drvsSeen); } break; @@ -135,28 +126,28 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; } XMLAttrs xmlAttrs; - if (location) posToXML(state, xmlAttrs, state.positions[v.lambda.fun->pos]); + if (location) posToXML(state, xmlAttrs, state.positions[v.payload.lambda.fun->pos]); XMLOpenElement _(doc, "function", xmlAttrs); - if (v.lambda.fun->hasFormals()) { + if (v.payload.lambda.fun->hasFormals()) { XMLAttrs attrs; - if (v.lambda.fun->arg) attrs["name"] = state.symbols[v.lambda.fun->arg]; - if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; + if (v.payload.lambda.fun->arg) attrs["name"] = state.symbols[v.payload.lambda.fun->arg]; + if (v.payload.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; XMLOpenElement _(doc, "attrspat", attrs); - for (auto & i : v.lambda.fun->formals->lexicographicOrder(state.symbols)) + for (auto & i : v.payload.lambda.fun->formals->lexicographicOrder(state.symbols)) doc.writeEmptyElement("attr", singletonAttrs("name", state.symbols[i.name])); } else - doc.writeEmptyElement("varpat", singletonAttrs("name", state.symbols[v.lambda.fun->arg])); + doc.writeEmptyElement("varpat", singletonAttrs("name", state.symbols[v.payload.lambda.fun->arg])); break; } case nExternal: - v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); + v.external()->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); break; case nFloat: - doc.writeEmptyElement("float", singletonAttrs("value", fmt("%1%", v.fpoint))); + doc.writeEmptyElement("float", singletonAttrs("value", fmt("%1%", v.fpoint()))); break; case nThunk: diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 335801b34..eb71765c5 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -234,14 +234,14 @@ public: ExprLambda * fun; }; - union + using Payload = union { NixInt integer; bool boolean; StringWithContext string; - Path _path; + Path path; Bindings * attrs; struct { @@ -258,6 +258,8 @@ public: NixFloat fpoint; }; + Payload payload; + /** * Returns the normal type of a Value. This only returns nThunk if * the Value hasn't been forceValue'd @@ -286,34 +288,31 @@ public: abort(); } - /** - * After overwriting an app node, be sure to clear pointers in the - * Value to ensure that the target isn't kept alive unnecessarily. - */ - inline void clearValue() + inline void finishValue(InternalType newType, Payload newPayload) { - app.left = app.right = 0; + /* After overwriting thunk/app values, be sure to clear + pointers in the Value to ensure that the target isn't kept + alive unnecessarily. */ + payload.app.left = payload.app.right = 0; + + payload = newPayload; + + internalType = newType; } inline void mkInt(NixInt n) { - clearValue(); - internalType = tInt; - integer = n; + finishValue(tInt, { .integer = n }); } inline void mkBool(bool b) { - clearValue(); - internalType = tBool; - boolean = b; + finishValue(tBool, { .boolean = b }); } inline void mkString(const char * s, const char * * context = 0) { - internalType = tString; - string.c_str = s; - string.context = context; + finishValue(tString, { .string = { .c_str = s, .context = context } }); } void mkString(std::string_view s); @@ -331,63 +330,44 @@ public: inline void mkPath(InputAccessor * accessor, const char * path) { - clearValue(); - internalType = tPath; - _path.accessor = accessor; - _path.path = path; + finishValue(tPath, { .path = { .accessor = accessor, .path = path } }); } inline void mkNull() { - clearValue(); - internalType = tNull; + finishValue(tNull, {}); } inline void mkAttrs(Bindings * a) { - clearValue(); - internalType = tAttrs; - attrs = a; + finishValue(tAttrs, { .attrs = a }); } Value & mkAttrs(BindingsBuilder & bindings); void mkList(const ListBuilder & builder) { - clearValue(); - if (builder.size == 1) { - smallList[0] = builder.inlineElems[0]; - internalType = tList1; - } else if (builder.size == 2) { - smallList[0] = builder.inlineElems[0]; - smallList[1] = builder.inlineElems[1]; - internalType = tList2; - } else { - bigList.size = builder.size; - bigList.elems = builder.elems; - internalType = tListN; - } + if (builder.size == 1) + finishValue(tList1, { .smallList = { builder.inlineElems[0] } }); + else if (builder.size == 2) + finishValue(tList2, { .smallList = { builder.inlineElems[0], builder.inlineElems[1] } }); + else + finishValue(tListN, { .bigList = { .size = builder.size, .elems = builder.elems } }); } inline void mkThunk(Env * e, Expr * ex) { - internalType = tThunk; - thunk.env = e; - thunk.expr = ex; + finishValue(tThunk, { .thunk = { .env = e, .expr = ex } }); } inline void mkApp(Value * l, Value * r) { - internalType = tApp; - app.left = l; - app.right = r; + finishValue(tApp, { .app = { .left = l, .right = r } }); } inline void mkLambda(Env * e, ExprLambda * f) { - internalType = tLambda; - lambda.env = e; - lambda.fun = f; + finishValue(tLambda, { .lambda = { .env = e, .fun = f } }); } inline void mkBlackhole(); @@ -396,28 +376,22 @@ public: inline void mkPrimOpApp(Value * l, Value * r) { - internalType = tPrimOpApp; - primOpApp.left = l; - primOpApp.right = r; + finishValue(tPrimOpApp, { .primOpApp = { .left = l, .right = r } }); } /** * For a `tPrimOpApp` value, get the original `PrimOp` value. */ - PrimOp * primOpAppPrimOp() const; + const PrimOp * primOpAppPrimOp() const; inline void mkExternal(ExternalValueBase * e) { - clearValue(); - internalType = tExternal; - external = e; + finishValue(tExternal, { .external = e }); } inline void mkFloat(NixFloat n) { - clearValue(); - internalType = tFloat; - fpoint = n; + finishValue(tFloat, { .fpoint = n }); } bool isList() const @@ -427,7 +401,7 @@ public: Value * const * listElems() { - return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? payload.smallList : payload.bigList.elems; } std::span listItems() const @@ -438,12 +412,12 @@ public: Value * const * listElems() const { - return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? payload.smallList : payload.bigList.elems; } size_t listSize() const { - return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size; + return internalType == tList1 ? 1 : internalType == tList2 ? 2 : payload.bigList.size; } PosIdx determinePos(const PosIdx pos) const; @@ -459,26 +433,44 @@ public: { assert(internalType == tPath); return SourcePath( - ref(_path.accessor->shared_from_this()), - CanonPath(CanonPath::unchecked_t(), _path.path)); + ref(payload.path.accessor->shared_from_this()), + CanonPath(CanonPath::unchecked_t(), payload.path.path)); } std::string_view string_view() const { assert(internalType == tString); - return std::string_view(string.c_str); + return std::string_view(payload.string.c_str); } const char * const c_str() const { assert(internalType == tString); - return string.c_str; + return payload.string.c_str; } const char * * context() const { - return string.context; + return payload.string.context; } + + ExternalValueBase * external() const + { return payload.external; } + + const Bindings * attrs() const + { return payload.attrs; } + + const PrimOp * primOp() const + { return payload.primOp; } + + bool boolean() const + { return payload.boolean; } + + NixInt integer() const + { return payload.integer; } + + NixFloat fpoint() const + { return payload.fpoint; } }; @@ -486,13 +478,12 @@ extern ExprBlackHole eBlackHole; bool Value::isBlackhole() const { - return internalType == tThunk && thunk.expr == (Expr*) &eBlackHole; + return internalType == tThunk && payload.thunk.expr == (Expr*) &eBlackHole; } void Value::mkBlackhole() { - internalType = tThunk; - thunk.expr = (Expr*) &eBlackHole; + mkThunk(nullptr, (Expr *) &eBlackHole); } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 35eef5b83..35bc03926 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -332,8 +332,8 @@ static void main_nix_build(int argc, char * * argv) return false; } bool add = false; - if (v.type() == nFunction && v.lambda.fun->hasFormals()) { - for (auto & i : v.lambda.fun->formals->formals) { + if (v.type() == nFunction && v.payload.lambda.fun->hasFormals()) { + for (auto & i : v.payload.lambda.fun->formals->formals) { if (state->symbols[i.name] == "inNixShell") { add = true; break; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index f79755375..9aff83bb0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1237,15 +1237,15 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; - attrs2["value"] = fmt("%1%", v->integer); + attrs2["value"] = fmt("%1%", v->integer()); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nFloat) { attrs2["type"] = "float"; - attrs2["value"] = fmt("%1%", v->fpoint); + attrs2["value"] = fmt("%1%", v->fpoint()); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nBool) { attrs2["type"] = "bool"; - attrs2["value"] = v->boolean ? "true" : "false"; + attrs2["value"] = v->boolean() ? "true" : "false"; xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nList) { attrs2["type"] = "strings"; @@ -1259,13 +1259,11 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } else if (v->type() == nAttrs) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); - Bindings & attrs = *v->attrs; - for (auto &i : attrs) { - Attr & a(*attrs.find(i.name)); - if(a.value->type() != nString) continue; + for (auto & i : *v->attrs()) { + if (i.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = globals.state->symbols[i.name]; - attrs3["value"] = a.value->c_str(); + attrs3["value"] = i.value->c_str(); xml.writeEmptyElement("string", attrs3); } } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index dd27344aa..6cbbacb15 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -138,9 +138,9 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, debug("evaluating user environment builder"); state.forceValue(topLevel, topLevel.determinePos(noPos)); NixStringContext context; - Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); + auto & aDrvPath(*topLevel.attrs()->find(state.sDrvPath)); auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); - Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); + auto & aOutPath(*topLevel.attrs()->find(state.sOutPath)); auto topLevelOut = state.coerceToStorePath(aOutPath.pos, *aOutPath.value, context, ""); /* Realise the resulting store expression. */ diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 54cc6a17f..2e50392f7 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -93,14 +93,14 @@ struct CmdBundle : InstallableValueCommand if (!evalState->isDerivation(*vRes)) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - auto attr1 = vRes->attrs->get(evalState->sDrvPath); + auto attr1 = vRes->attrs()->get(evalState->sDrvPath); if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); NixStringContext context2; auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2, ""); - auto attr2 = vRes->attrs->get(evalState->sOutPath); + auto attr2 = vRes->attrs()->get(evalState->sOutPath); if (!attr2) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); @@ -116,7 +116,7 @@ struct CmdBundle : InstallableValueCommand auto outPathS = store->printStorePath(outPath); if (!outLink) { - auto * attr = vRes->attrs->get(evalState->sName); + auto * attr = vRes->attrs()->get(evalState->sName); if (!attr) throw Error("attribute 'name' missing"); outLink = evalState->forceStringNoCtx(*attr->value, attr->pos, ""); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 088be3b17..7b9bf7cb1 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -89,7 +89,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); - for (auto & attr : *v.attrs) { + for (auto & attr : *v.attrs()) { std::string_view name = state->symbols[attr.name]; try { if (name == "." || name == "..") diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a846f6371..4825ab0e1 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -169,7 +169,7 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, auto pos = vFlake.determinePos(noPos); state.forceAttrs(vFlake, pos, "while evaluating a flake to get its outputs"); - auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake.attrs()->get(state.symbols.create("outputs")); assert(aOutputs); state.forceAttrs(*aOutputs->value, pos, "while evaluating the outputs of a flake"); @@ -179,10 +179,10 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, /* Hack: ensure that hydraJobs is evaluated before anything else. This way we can disable IFD for hydraJobs and then enable it for other outputs. */ - if (auto attr = aOutputs->value->attrs->get(sHydraJobs)) + if (auto attr = aOutputs->value->attrs()->get(sHydraJobs)) callback(state.symbols[attr->name], *attr->value, attr->pos); - for (auto & attr : *aOutputs->value->attrs) { + for (auto & attr : *aOutputs->value->attrs()) { if (attr.name != sHydraJobs) callback(state.symbols[attr.name], *attr.value, attr.pos); } @@ -451,10 +451,10 @@ struct CmdFlakeCheck : FlakeCommand if (!v.isLambda()) { throw Error("overlay is not a function, but %s instead", showType(v)); } - if (v.lambda.fun->hasFormals() - || !argHasName(v.lambda.fun->arg, "final")) + if (v.payload.lambda.fun->hasFormals() + || !argHasName(v.payload.lambda.fun->arg, "final")) throw Error("overlay does not take an argument named 'final'"); - auto body = dynamic_cast(v.lambda.fun->body); + auto body = dynamic_cast(v.payload.lambda.fun->body); if (!body || body->hasFormals() || !argHasName(body->arg, "prev")) @@ -489,7 +489,7 @@ struct CmdFlakeCheck : FlakeCommand if (state->isDerivation(v)) throw Error("jobset should not be a derivation at top-level"); - for (auto & attr : *v.attrs) { + for (auto & attr : *v.attrs()) { state->forceAttrs(*attr.value, attr.pos, ""); auto attrPath2 = concatStrings(attrPath, ".", state->symbols[attr.name]); if (state->isDerivation(*attr.value)) { @@ -528,7 +528,7 @@ struct CmdFlakeCheck : FlakeCommand state->forceAttrs(v, pos, ""); - if (auto attr = v.attrs->get(state->symbols.create("path"))) { + if (auto attr = v.attrs()->get(state->symbols.create("path"))) { if (attr->name == state->symbols.create("path")) { NixStringContext context; auto path = state->coerceToPath(attr->pos, *attr->value, context, ""); @@ -539,12 +539,12 @@ struct CmdFlakeCheck : FlakeCommand } else throw Error("template '%s' lacks attribute 'path'", attrPath); - if (auto attr = v.attrs->get(state->symbols.create("description"))) + if (auto attr = v.attrs()->get(state->symbols.create("description"))) state->forceStringNoCtx(*attr->value, attr->pos, ""); else throw Error("template '%s' lacks attribute 'description'", attrPath); - for (auto & attr : *v.attrs) { + for (auto & attr : *v.attrs()) { std::string_view name(state->symbols[attr.name]); if (name != "path" && name != "description" && name != "welcomeText") throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); @@ -600,12 +600,12 @@ struct CmdFlakeCheck : FlakeCommand if (name == "checks") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) { + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) { + for (auto & attr2 : *attr.value->attrs()) { auto drvPath = checkDerivation( fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), *attr2.value, attr2.pos); @@ -622,7 +622,7 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "formatter") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) { + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { @@ -635,12 +635,12 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "packages" || name == "devShells") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) { + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) + for (auto & attr2 : *attr.value->attrs()) checkDerivation( fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), *attr2.value, attr2.pos); @@ -650,12 +650,12 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "apps") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) { + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) + for (auto & attr2 : *attr.value->attrs()) checkApp( fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), *attr2.value, attr2.pos); @@ -665,7 +665,7 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "defaultPackage" || name == "devShell") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) { + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { @@ -678,7 +678,7 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "defaultApp") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) { + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos) ) { @@ -691,7 +691,7 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "legacyPackages") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) { + for (auto & attr : *vOutput.attrs()) { checkSystemName(state->symbols[attr.name], attr.pos); checkSystemType(state->symbols[attr.name], attr.pos); // FIXME: do getDerivations? @@ -703,7 +703,7 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "overlays") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) + for (auto & attr : *vOutput.attrs()) checkOverlay(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } @@ -713,14 +713,14 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "nixosModules") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) + for (auto & attr : *vOutput.attrs()) checkModule(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } else if (name == "nixosConfigurations") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) + for (auto & attr : *vOutput.attrs()) checkNixOSConfiguration(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } @@ -733,14 +733,14 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "templates") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) + for (auto & attr : *vOutput.attrs()) checkTemplate(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } else if (name == "defaultBundler") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) { + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { @@ -753,12 +753,12 @@ struct CmdFlakeCheck : FlakeCommand else if (name == "bundlers") { state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs) { + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) { + for (auto & attr2 : *attr.value->attrs()) { checkBundler( fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), *attr2.value, attr2.pos); diff --git a/src/nix/main.cc b/src/nix/main.cc index 5af5f2e41..1ee9e4198 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -256,7 +256,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos); state.callFunction(*vRes, *vDump, *vRes, noPos); - auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); + auto attr = vRes->attrs()->get(state.symbols.create(mdName + ".md")); if (!attr) throw UsageError("Nix has no subcommand '%s'", concatStringsSep("", subcommand)); @@ -400,11 +400,10 @@ void mainWrapped(int argc, char * * argv) auto res = nlohmann::json::object(); res["builtins"] = ({ auto builtinsJson = nlohmann::json::object(); - auto builtins = state.baseEnv.values[0]->attrs; - for (auto & builtin : *builtins) { + for (auto & builtin : *state.baseEnv.values[0]->attrs()) { auto b = nlohmann::json::object(); if (!builtin.value->isPrimOp()) continue; - auto primOp = builtin.value->primOp; + auto primOp = builtin.value->primOp(); if (!primOp->doc) continue; b["arity"] = primOp->arity; b["args"] = primOp->args; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index fabec5d88..17178c84f 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -35,8 +35,8 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url) vMirrors); state.forceAttrs(vMirrors, noPos, "while evaluating the set of all mirrors"); - auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); - if (mirrorList == vMirrors.attrs->end()) + auto mirrorList = vMirrors.attrs()->get(state.symbols.create(mirrorName)); + if (!mirrorList) throw Error("unknown mirror name '%s'", mirrorName); state.forceList(*mirrorList->value, noPos, "while evaluating one mirror configuration"); @@ -213,7 +213,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) state->forceAttrs(v, noPos, "while evaluating the source attribute to prefetch"); /* Extract the URL. */ - auto * attr = v.attrs->get(state->symbols.create("urls")); + auto * attr = v.attrs()->get(state->symbols.create("urls")); if (!attr) throw Error("attribute 'urls' missing"); state->forceList(*attr->value, noPos, "while evaluating the urls to prefetch"); @@ -222,7 +222,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) url = state->forceString(*attr->value->listElems()[0], noPos, "while evaluating the first url from the urls list"); /* Extract the hash mode. */ - auto attr2 = v.attrs->get(state->symbols.create("outputHashMode")); + auto attr2 = v.attrs()->get(state->symbols.create("outputHashMode")); if (!attr2) printInfo("warning: this does not look like a fetchurl call"); else @@ -230,7 +230,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) /* Extract the name. */ if (!name) { - auto attr3 = v.attrs->get(state->symbols.create("name")); + auto attr3 = v.attrs()->get(state->symbols.create("name")); if (!attr3) name = state->forceString(*attr3->value, noPos, "while evaluating the name of the source to prefetch"); } diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index d720cedde..1a4313990 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -80,28 +80,28 @@ namespace nix { if (arg.type() != nInt) { return false; } - return arg.integer == v; + return arg.integer() == v; } MATCHER_P(IsFloatEq, v, fmt("The float is equal to \"%1%\"", v)) { if (arg.type() != nFloat) { return false; } - return arg.fpoint == v; + return arg.fpoint() == v; } MATCHER(IsTrue, "") { if (arg.type() != nBool) { return false; } - return arg.boolean == true; + return arg.boolean() == true; } MATCHER(IsFalse, "") { if (arg.type() != nBool) { return false; } - return arg.boolean == false; + return arg.boolean() == false; } MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) { @@ -134,8 +134,8 @@ namespace nix { if (arg.type() != nAttrs) { *result_listener << "Expected set got " << arg.type(); return false; - } else if (arg.attrs->size() != (size_t)n) { - *result_listener << "Expected a set with " << n << " attributes but got " << arg.attrs->size(); + } else if (arg.attrs()->size() != (size_t) n) { + *result_listener << "Expected a set with " << n << " attributes but got " << arg.attrs()->size(); return false; } return true; diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc index b1426edae..7988eac70 100644 --- a/tests/unit/libexpr/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -72,7 +72,7 @@ namespace nix { auto v = eval("builtins.tryEval (throw \"\")"); ASSERT_THAT(v, IsAttrsOfSize(2)); auto s = createSymbol("success"); - auto p = v.attrs->get(s); + auto p = v.attrs()->get(s); ASSERT_NE(p, nullptr); ASSERT_THAT(*p->value, IsFalse()); } @@ -81,11 +81,11 @@ namespace nix { auto v = eval("builtins.tryEval 123"); ASSERT_THAT(v, IsAttrs()); auto s = createSymbol("success"); - auto p = v.attrs->get(s); + auto p = v.attrs()->get(s); ASSERT_NE(p, nullptr); ASSERT_THAT(*p->value, IsTrue()); s = createSymbol("value"); - p = v.attrs->get(s); + p = v.attrs()->get(s); ASSERT_NE(p, nullptr); ASSERT_THAT(*p->value, IsIntEq(123)); } @@ -157,18 +157,18 @@ namespace nix { auto v = eval(expr); ASSERT_THAT(v, IsAttrsOfSize(3)); - auto file = v.attrs->find(createSymbol("file")); + auto file = v.attrs()->find(createSymbol("file")); ASSERT_NE(file, nullptr); ASSERT_THAT(*file->value, IsString()); auto s = baseNameOf(file->value->string_view()); ASSERT_EQ(s, "foo.nix"); - auto line = v.attrs->find(createSymbol("line")); + auto line = v.attrs()->find(createSymbol("line")); ASSERT_NE(line, nullptr); state.forceValue(*line->value, noPos); ASSERT_THAT(*line->value, IsIntEq(4)); - auto column = v.attrs->find(createSymbol("column")); + auto column = v.attrs()->find(createSymbol("column")); ASSERT_NE(column, nullptr); state.forceValue(*column->value, noPos); ASSERT_THAT(*column->value, IsIntEq(3)); @@ -202,14 +202,14 @@ namespace nix { TEST_F(PrimOpTest, removeAttrsRetains) { auto v = eval("builtins.removeAttrs { x = 1; y = 2; } [\"x\"]"); ASSERT_THAT(v, IsAttrsOfSize(1)); - ASSERT_NE(v.attrs->find(createSymbol("y")), nullptr); + ASSERT_NE(v.attrs()->find(createSymbol("y")), nullptr); } TEST_F(PrimOpTest, listToAttrsEmptyList) { auto v = eval("builtins.listToAttrs []"); ASSERT_THAT(v, IsAttrsOfSize(0)); ASSERT_EQ(v.type(), nAttrs); - ASSERT_EQ(v.attrs->size(), 0); + ASSERT_EQ(v.attrs()->size(), 0); } TEST_F(PrimOpTest, listToAttrsNotFieldName) { @@ -219,7 +219,7 @@ namespace nix { TEST_F(PrimOpTest, listToAttrs) { auto v = eval("builtins.listToAttrs [ { name = \"key\"; value = 123; } ]"); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto key = v.attrs->find(createSymbol("key")); + auto key = v.attrs()->find(createSymbol("key")); ASSERT_NE(key, nullptr); ASSERT_THAT(*key->value, IsIntEq(123)); } @@ -227,7 +227,7 @@ namespace nix { TEST_F(PrimOpTest, intersectAttrs) { auto v = eval("builtins.intersectAttrs { a = 1; b = 2; } { b = 3; c = 4; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto b = v.attrs->find(createSymbol("b")); + auto b = v.attrs()->find(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(3)); } @@ -243,11 +243,11 @@ namespace nix { auto v = eval("builtins.functionArgs ({ x, y ? 123}: 1)"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto x = v.attrs->find(createSymbol("x")); + auto x = v.attrs()->find(createSymbol("x")); ASSERT_NE(x, nullptr); ASSERT_THAT(*x->value, IsFalse()); - auto y = v.attrs->find(createSymbol("y")); + auto y = v.attrs()->find(createSymbol("y")); ASSERT_NE(y, nullptr); ASSERT_THAT(*y->value, IsTrue()); } @@ -256,13 +256,13 @@ namespace nix { auto v = eval("builtins.mapAttrs (name: value: value * 10) { a = 1; b = 2; }"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto a = v.attrs->find(createSymbol("a")); + auto a = v.attrs()->find(createSymbol("a")); ASSERT_NE(a, nullptr); ASSERT_THAT(*a->value, IsThunk()); state.forceValue(*a->value, noPos); ASSERT_THAT(*a->value, IsIntEq(10)); - auto b = v.attrs->find(createSymbol("b")); + auto b = v.attrs()->find(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsThunk()); state.forceValue(*b->value, noPos); @@ -410,13 +410,13 @@ namespace nix { auto v = eval("builtins.partition (x: x > 10) [1 23 9 3 42]"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto right = v.attrs->get(createSymbol("right")); + auto right = v.attrs()->get(createSymbol("right")); ASSERT_NE(right, nullptr); ASSERT_THAT(*right->value, IsListOfSize(2)); ASSERT_THAT(*right->value->listElems()[0], IsIntEq(23)); ASSERT_THAT(*right->value->listElems()[1], IsIntEq(42)); - auto wrong = v.attrs->get(createSymbol("wrong")); + auto wrong = v.attrs()->get(createSymbol("wrong")); ASSERT_NE(wrong, nullptr); ASSERT_EQ(wrong->value->type(), nList); ASSERT_EQ(wrong->value->listSize(), 3); @@ -641,14 +641,14 @@ namespace nix { auto v = eval("derivation"); ASSERT_EQ(v.type(), nFunction); ASSERT_TRUE(v.isLambda()); - ASSERT_NE(v.lambda.fun, nullptr); - ASSERT_TRUE(v.lambda.fun->hasFormals()); + ASSERT_NE(v.payload.lambda.fun, nullptr); + ASSERT_TRUE(v.payload.lambda.fun->hasFormals()); } TEST_F(PrimOpTest, currentTime) { auto v = eval("builtins.currentTime"); ASSERT_EQ(v.type(), nInt); - ASSERT_TRUE(v.integer > 0); + ASSERT_TRUE(v.integer() > 0); } TEST_F(PrimOpTest, splitVersion) { @@ -709,11 +709,11 @@ namespace nix { auto v = eval(expr); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto name = v.attrs->find(createSymbol("name")); + auto name = v.attrs()->find(createSymbol("name")); ASSERT_TRUE(name); ASSERT_THAT(*name->value, IsStringEq(expectedName)); - auto version = v.attrs->find(createSymbol("version")); + auto version = v.attrs()->find(createSymbol("version")); ASSERT_TRUE(version); ASSERT_THAT(*version->value, IsStringEq(expectedVersion)); } diff --git a/tests/unit/libexpr/trivial.cc b/tests/unit/libexpr/trivial.cc index 171727ac7..61ea71a0f 100644 --- a/tests/unit/libexpr/trivial.cc +++ b/tests/unit/libexpr/trivial.cc @@ -62,11 +62,11 @@ namespace nix { TEST_F(TrivialExpressionTest, updateAttrs) { auto v = eval("{ a = 1; } // { b = 2; a = 3; }"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto a = v.attrs->find(createSymbol("a")); + auto a = v.attrs()->find(createSymbol("a")); ASSERT_NE(a, nullptr); ASSERT_THAT(*a->value, IsIntEq(3)); - auto b = v.attrs->find(createSymbol("b")); + auto b = v.attrs()->find(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(2)); } @@ -151,7 +151,7 @@ namespace nix { auto v = eval(expr); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto a = v.attrs->find(createSymbol("a")); + auto a = v.attrs()->find(createSymbol("a")); ASSERT_NE(a, nullptr); ASSERT_THAT(*a->value, IsThunk()); @@ -159,11 +159,11 @@ namespace nix { ASSERT_THAT(*a->value, IsAttrsOfSize(2)); - auto b = a->value->attrs->find(createSymbol("b")); + auto b = a->value->attrs()->find(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(1)); - auto c = a->value->attrs->find(createSymbol("c")); + auto c = a->value->attrs()->find(createSymbol("c")); ASSERT_NE(c, nullptr); ASSERT_THAT(*c->value, IsIntEq(2)); } @@ -185,7 +185,7 @@ namespace nix { TEST_F(TrivialExpressionTest, bindOr) { auto v = eval("{ or = 1; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto b = v.attrs->find(createSymbol("or")); + auto b = v.attrs()->find(createSymbol("or")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(1)); } From c82623a6cc918fb7657621250baf62726e94c4ea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Mar 2024 18:35:36 +0100 Subject: [PATCH 133/211] Remove value clearing since it no longer has an effect --- src/libexpr/value.hh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index eb71765c5..21eb71d5c 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -290,13 +290,7 @@ public: inline void finishValue(InternalType newType, Payload newPayload) { - /* After overwriting thunk/app values, be sure to clear - pointers in the Value to ensure that the target isn't kept - alive unnecessarily. */ - payload.app.left = payload.app.right = 0; - payload = newPayload; - internalType = newType; } From 02c41aba5b5b135bae0de4961fcf4d3a888a9524 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 5 Apr 2024 16:08:18 +0200 Subject: [PATCH 134/211] libexpr-c: Add nix_string_realise --- src/libexpr-c/nix_api_expr_internal.h | 6 ++ src/libexpr-c/nix_api_value.cc | 55 ++++++++++++ src/libexpr-c/nix_api_value.h | 60 ++++++++++++- src/libexpr/eval.hh | 8 +- src/libexpr/primops.cc | 21 +++-- tests/unit/libexpr/nix_api_expr.cc | 85 +++++++++++++++++++ .../libutil-support/tests/nix_api_util.hh | 10 +++ 7 files changed, 235 insertions(+), 10 deletions(-) diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index b50a51347..7743849fd 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -35,4 +35,10 @@ struct nix_string_context nix::NixStringContext & ctx; }; +struct nix_realised_string +{ + std::string str; + std::vector storePaths; +}; + #endif // NIX_API_EXPR_INTERNAL_H diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 02bd154b3..fd1bfc165 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -2,6 +2,7 @@ #include "config.hh" #include "eval.hh" #include "globals.hh" +#include "path.hh" #include "primops.hh" #include "value.hh" @@ -9,7 +10,9 @@ #include "nix_api_expr_internal.h" #include "nix_api_util.h" #include "nix_api_util_internal.h" +#include "nix_api_store_internal.h" #include "nix_api_value.h" +#include "value/context.hh" #ifdef HAVE_BOEHMGC # include "gc/gc.h" @@ -528,3 +531,55 @@ void nix_bindings_builder_free(BindingsBuilder * bb) delete (nix::BindingsBuilder *) bb; #endif } + +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto &v = check_value_not_null(value); + nix::NixStringContext stringContext; + auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned(); + nix::StorePathSet storePaths; + auto rewrites = state->state.realiseContext(stringContext, &storePaths); + + auto s = nix::rewriteStrings(rawStr, rewrites); + + // Convert to the C API StorePath type and convert to vector for index-based access + std::vector vec; + for (auto &sp : storePaths) { + vec.push_back(StorePath{sp}); + } + + return new nix_realised_string { + .str = s, + .storePaths = vec + }; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_realised_string_free(nix_realised_string * s) +{ + delete s; +} + +size_t nix_realised_string_get_buffer_size(nix_realised_string * s) +{ + return s->str.size(); +} + +const char * nix_realised_string_get_buffer_start(nix_realised_string * s) +{ + return s->str.data(); +} + +size_t nix_realised_string_get_store_path_count(nix_realised_string * s) +{ + return s->storePaths.size(); +} + +const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, size_t i) +{ + return &s->storePaths[i]; +} diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 42218188c..c8e85d181 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -9,6 +9,7 @@ */ #include "nix_api_util.h" +#include "nix_api_store.h" #include "stdbool.h" #include "stddef.h" #include "stdint.h" @@ -69,6 +70,10 @@ typedef struct PrimOp PrimOp; */ typedef struct ExternalValue ExternalValue; +/** @brief String without placeholders, and realised store paths + */ +typedef struct nix_realised_string nix_realised_string; + /** @defgroup primops * @brief Create your own primops * @{ @@ -167,7 +172,10 @@ const char * nix_get_typename(nix_c_context * context, const Value * value); */ bool nix_get_bool(nix_c_context * context, const Value * value); -/** @brief Get string +/** @brief Get the raw string + * + * This may contain placeholders. + * * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return string @@ -425,6 +433,56 @@ nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, void nix_bindings_builder_free(BindingsBuilder * builder); /**@}*/ +/** @brief Realise a string context. + * + * This will + * - realise the store paths referenced by the string's context, and + * - perform the replacement of placeholders. + * - create temporary garbage collection roots for the store paths, for + * the lifetime of the current process. + * - log to stderr + * + * @param[out] context Optional, stores error information + * @param[in] value Nix value, which must be a string + * @param[in] state Nix evaluator state + * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false. + You should set this to true when this call is part of a primop. + You should set this to false when building for your application's purpose. + * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free + */ +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD); + +/** @brief Start of the string + * @param[in] realised_string + * @return pointer to the start of the string. It may not be null-terminated. + */ +const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string); + +/** @brief Length of the string + * @param[in] realised_string + * @return length of the string in bytes + */ +size_t nix_realised_string_get_buffer_size(nix_realised_string * realised_string); + +/** @brief Number of realised store paths + * @param[in] realised_string + * @return number of realised store paths that were referenced by the string via its context + */ +size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_string); + +/** @brief Get a store path. The store paths are stored in an arbitrary order. + * @param[in] realised_string + * @param[in] index index of the store path, must be less than the count + * @return store path + */ +const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index); + +/** @brief Free a realised string + * @param[in] realised_string + */ +void nix_realised_string_free(nix_realised_string * realised_string); + + // cffi end #ifdef __cplusplus } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 4d53bcde6..2741220f1 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -733,10 +733,12 @@ public: bool fullGC(); /** - * Realise the given context, and return a mapping from the placeholders - * used to construct the associated value to their final store path + * Realise the given context + * @param[in] context the context to realise + * @param[out] maybePaths if not nullptr, all built or referenced store paths will be added to this set + * @return a mapping from the placeholders used to construct the associated value to their final store path. */ - [[nodiscard]] StringMap realiseContext(const NixStringContext & context); + [[nodiscard]] StringMap realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true); /* Call the binary path filter predicate used builtins.path etc. */ bool callPathFilter( diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 752176178..e446199ae 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -39,7 +39,7 @@ namespace nix { * Miscellaneous *************************************************************/ -StringMap EvalState::realiseContext(const NixStringContext & context) +StringMap EvalState::realiseContext(const NixStringContext & context, StorePathSet * maybePathsOut, bool isIFD) { std::vector drvs; StringMap res; @@ -61,19 +61,23 @@ StringMap EvalState::realiseContext(const NixStringContext & context) auto ctxS = store->printStorePath(o.path); res.insert_or_assign(ctxS, ctxS); ensureValid(o.path); + if (maybePathsOut) + maybePathsOut->emplace(o.path); }, [&](const NixStringContextElem::DrvDeep & d) { /* Treat same as Opaque */ auto ctxS = store->printStorePath(d.drvPath); res.insert_or_assign(ctxS, ctxS); ensureValid(d.drvPath); + if (maybePathsOut) + maybePathsOut->emplace(d.drvPath); }, }, c.raw); } if (drvs.empty()) return {}; - if (!evalSettings.enableImportFromDerivation) + if (isIFD && !evalSettings.enableImportFromDerivation) error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", drvs.begin()->to_string(*store) @@ -90,6 +94,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context) auto outputs = resolveDerivedPath(*buildStore, drv, &*store); for (auto & [outputName, outputPath] : outputs) { outputsToCopyAndAllow.insert(outputPath); + if (maybePathsOut) + maybePathsOut->emplace(outputPath); /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { @@ -106,10 +112,13 @@ StringMap EvalState::realiseContext(const NixStringContext & context) } if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); - for (auto & outputPath : outputsToCopyAndAllow) { - /* Add the output of this derivations to the allowed - paths. */ - allowPath(outputPath); + + if (isIFD) { + for (auto & outputPath : outputsToCopyAndAllow) { + /* Add the output of this derivations to the allowed + paths. */ + allowPath(outputPath); + } } return res; diff --git a/tests/unit/libexpr/nix_api_expr.cc b/tests/unit/libexpr/nix_api_expr.cc index 9d54a62f8..3808bf0eb 100644 --- a/tests/unit/libexpr/nix_api_expr.cc +++ b/tests/unit/libexpr/nix_api_expr.cc @@ -7,6 +7,7 @@ #include "tests/nix_api_expr.hh" +#include "gmock/gmock.h" #include namespace nixC { @@ -97,4 +98,88 @@ TEST_F(nix_api_expr_test, nix_build_drv) nix_store_path_free(drvStorePath); nix_store_path_free(outStorePath); } + +TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_value) +{ + auto expr = "true"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + auto r = nix_string_realise(ctx, state, value, false); + ASSERT_EQ(nullptr, r); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("cannot coerce"))); } + +TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_build) +{ + auto expr = R"( + derivation { name = "letsbuild"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo failing a build for testing purposes; exit 1;" ]; + } + )"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + auto r = nix_string_realise(ctx, state, value, false); + ASSERT_EQ(nullptr, r); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("failed with exit code 1"))); +} + +TEST_F(nix_api_expr_test, nix_expr_realise_context) +{ + // TODO (ca-derivations): add a content-addressed derivation output, which produces a placeholder + auto expr = R"( + '' + a derivation output: ${ + derivation { name = "letsbuild"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo foo > $out" ]; + }} + a path: ${builtins.toFile "just-a-file" "ooh file good"} + a derivation path by itself: ${ + builtins.unsafeDiscardOutputDependency + (derivation { + name = "not-actually-built-yet"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo foo > $out" ]; + }).drvPath} + '' + )"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + auto r = nix_string_realise(ctx, state, value, false); + assert_ctx_ok(); + ASSERT_NE(nullptr, r); + + auto s = std::string(nix_realised_string_get_buffer_start(r), nix_realised_string_get_buffer_size(r)); + + EXPECT_THAT(s, testing::StartsWith("a derivation output:")); + EXPECT_THAT(s, testing::HasSubstr("-letsbuild\n")); + EXPECT_THAT(s, testing::Not(testing::HasSubstr("-letsbuild.drv"))); + EXPECT_THAT(s, testing::HasSubstr("a path:")); + EXPECT_THAT(s, testing::HasSubstr("-just-a-file")); + EXPECT_THAT(s, testing::Not(testing::HasSubstr("-just-a-file.drv"))); + EXPECT_THAT(s, testing::Not(testing::HasSubstr("ooh file good"))); + EXPECT_THAT(s, testing::HasSubstr("a derivation path by itself:")); + EXPECT_THAT(s, testing::EndsWith("-not-actually-built-yet.drv\n")); + + std::vector names; + size_t n = nix_realised_string_get_store_path_count(r); + for (size_t i = 0; i < n; ++i) { + const StorePath * p = nix_realised_string_get_store_path(r, i); + names.push_back(p->path.name()); + } + std::sort(names.begin(), names.end()); + ASSERT_EQ(3, names.size()); + EXPECT_THAT(names[0], testing::StrEq("just-a-file")); + EXPECT_THAT(names[1], testing::StrEq("letsbuild")); + EXPECT_THAT(names[2], testing::StrEq("not-actually-built-yet.drv")); + + nix_realised_string_free(r); +} + +} // namespace nixC diff --git a/tests/unit/libutil-support/tests/nix_api_util.hh b/tests/unit/libutil-support/tests/nix_api_util.hh index 0dfb38f7b..75d302bd6 100644 --- a/tests/unit/libutil-support/tests/nix_api_util.hh +++ b/tests/unit/libutil-support/tests/nix_api_util.hh @@ -23,5 +23,15 @@ protected: } nix_c_context * ctx; + + inline void assert_ctx_ok() { + if (nix_err_code(ctx) == NIX_OK) { + return; + } + unsigned int n; + const char * p = nix_err_msg(nullptr, ctx, &n); + std::string msg(p, n); + FAIL() << "nix_err_code(ctx) != NIX_OK, message: " << msg; + } }; } From c145ce0e1a01f54f7ddf7d38909cf6bd8ec32a88 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 5 Apr 2024 16:15:43 +0200 Subject: [PATCH 135/211] realiseContext: Remove no-op replacements A possible use of them might have been to figure out the paths (which can now be retrieved with maybePathsOut), but I have not found evidence that it was used this way, and it would have been broken, because non-CA outputs weren't recorded in the map. --- src/libexpr/primops.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e446199ae..5fd3256a5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -59,7 +59,6 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS }, [&](const NixStringContextElem::Opaque & o) { auto ctxS = store->printStorePath(o.path); - res.insert_or_assign(ctxS, ctxS); ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); @@ -67,7 +66,6 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS [&](const NixStringContextElem::DrvDeep & d) { /* Treat same as Opaque */ auto ctxS = store->printStorePath(d.drvPath); - res.insert_or_assign(ctxS, ctxS); ensureValid(d.drvPath); if (maybePathsOut) maybePathsOut->emplace(d.drvPath); From 513634ab5bf97c32256733d12e7381c4e83e1adf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 5 Apr 2024 12:25:43 -0400 Subject: [PATCH 136/211] Make `cgroup.{cc,hh}` linux-only files Forcing a conditional include, vs making the headers content conditional, I think is more maintainable. It is also how the other platform-specific headers (like `namespaces.hh`) have been adapted. --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libutil/{ => linux}/cgroup.cc | 4 ---- src/libutil/{ => linux}/cgroup.hh | 4 ---- 3 files changed, 1 insertion(+), 9 deletions(-) rename src/libutil/{ => linux}/cgroup.cc (99%) rename src/libutil/{ => linux}/cgroup.hh (96%) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index f8794e783..ab66195b8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -14,7 +14,6 @@ #include "topo-sort.hh" #include "callback.hh" #include "json-utils.hh" -#include "cgroup.hh" #include "personality.hh" #include "current-process.hh" #include "child.hh" @@ -52,6 +51,7 @@ # include # endif # define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +# include "cgroup.hh" #endif #if __APPLE__ diff --git a/src/libutil/cgroup.cc b/src/libutil/linux/cgroup.cc similarity index 99% rename from src/libutil/cgroup.cc rename to src/libutil/linux/cgroup.cc index de83b5ad1..8b8942643 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/linux/cgroup.cc @@ -1,5 +1,3 @@ -#if __linux__ - #include "cgroup.hh" #include "util.hh" #include "file-system.hh" @@ -145,5 +143,3 @@ CgroupStats destroyCgroup(const Path & cgroup) } } - -#endif diff --git a/src/libutil/cgroup.hh b/src/libutil/linux/cgroup.hh similarity index 96% rename from src/libutil/cgroup.hh rename to src/libutil/linux/cgroup.hh index 574ae8e5b..783a0ab87 100644 --- a/src/libutil/cgroup.hh +++ b/src/libutil/linux/cgroup.hh @@ -1,8 +1,6 @@ #pragma once ///@file -#if __linux__ - #include #include @@ -28,5 +26,3 @@ struct CgroupStats CgroupStats destroyCgroup(const Path & cgroup); } - -#endif From c80cd6bb0684c265f09cf0b17908859a306626fd Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 5 Apr 2024 16:09:20 +0200 Subject: [PATCH 137/211] path-info: print correct path when using `nix path-info --store file://... --all --json` When querying all paths in a binary cache store, the path's representation is `-x` (where `x` is the value of `MissingName`) because the .narinfo filenames only contain the hash. Before cc46ea163024254d0b74646e1b38b19896d40040 this worked correctly, because the entire path info was read and the path from this representation was printed, i.e. in the form `-`. Since then however, the direct result from `queryAllValidPaths()` was used as `path`. Added a regression test to make sure the behavior remains correct. --- src/nix/path-info.cc | 8 +++++++- tests/functional/binary-cache.sh | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 5f10cfb61..921b25d7f 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -43,10 +43,16 @@ static json pathInfoToJSON( for (auto & storePath : storePaths) { json jsonObject; + auto printedStorePath = store.printStorePath(storePath); try { auto info = store.queryPathInfo(storePath); + // `storePath` has the representation `-x` rather than + // `-` in case of binary-cache stores & `--all` because we don't + // know the name yet until we've read the NAR info. + printedStorePath = store.printStorePath(info->path); + jsonObject = info->toJSON(store, true, HashFormat::SRI); if (showClosureSize) { @@ -74,7 +80,7 @@ static json pathInfoToJSON( jsonObject = nullptr; } - jsonAllObjects[store.printStorePath(storePath)] = std::move(jsonObject); + jsonAllObjects[printedStorePath] = std::move(jsonObject); } return jsonAllObjects; } diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 7c64a115c..2a8d5ccdb 100644 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -14,6 +14,14 @@ outPath=$(nix-build dependencies.nix --no-out-link) nix copy --to file://$cacheDir $outPath +readarray -t paths < <(nix path-info --all --json --store file://$cacheDir | jq 'keys|sort|.[]' -r) +[[ "${#paths[@]}" -eq 3 ]] +for path in "${paths[@]}"; do + [[ "$path" =~ -dependencies-input-0$ ]] \ + || [[ "$path" =~ -dependencies-input-2$ ]] \ + || [[ "$path" =~ -dependencies-top$ ]] +done + # Test copying build logs to the binary cache. expect 1 nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available' nix store copy-log --to file://$cacheDir $outPath From 910211f9ffba41f5fa269217192638773a0bf30c Mon Sep 17 00:00:00 2001 From: stuebinm Date: Fri, 5 Apr 2024 23:19:32 +0200 Subject: [PATCH 138/211] avoid markdown which the repl's :doc cannot handle code blocks, if not surrounded by empty lines, have the language tags (in these cases, always `nix`) show up in the output of :doc. for example: nix-repl> :doc builtins.parseFlakeRef Synopsis: builtins.parseFlakeRef flake-ref Parse a flake reference, and return its exploded form. For example: nix builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" evaluates to: nix { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } is now instead: nix-repl> :doc builtins.parseFlakeRef Synopsis: builtins.parseFlakeRef flake-ref Parse a flake reference, and return its exploded form. For example: | builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" evaluates to: | { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } --- src/libexpr/flake/flake.cc | 6 ++++++ src/libexpr/primops.cc | 2 ++ src/libexpr/primops/fetchTree.cc | 2 ++ 3 files changed, 10 insertions(+) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 4a781beb8..8fe3a1d74 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -868,10 +868,13 @@ static RegisterPrimOp r3({ Parse a flake reference, and return its exploded form. For example: + ```nix builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" ``` + evaluates to: + ```nix { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } ``` @@ -920,12 +923,15 @@ static RegisterPrimOp r4({ Convert a flake reference from attribute set format to URL format. For example: + ```nix builtins.flakeRefToString { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } ``` + evaluates to + ```nix "github:NixOS/nixpkgs/23.05?dir=lib" ``` diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8e79f4953..8a2b3512b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1915,11 +1915,13 @@ static RegisterPrimOp primop_outputOf({ *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`. This primop can be chained arbitrarily deeply. For instance, + ```nix builtins.outputOf (builtins.outputOf myDrv "out") "out" ``` + will return a placeholder for the output of the output of `myDrv`. This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line. diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 5061e40fd..7906f8ac3 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -650,12 +650,14 @@ static RegisterPrimOp primop_fetchGit({ The public keys against which `rev` is verified if `verifyCommit` is enabled. Must be given as a list of attribute sets with the following form: + ```nix { key = ""; type = ""; # optional, default: "ssh-ed25519" } ``` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). From e73dc0e938c59b071eb24252980935b51cb17106 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Fri, 5 Apr 2024 16:43:14 -0500 Subject: [PATCH 139/211] Use LIBMOUNT_FORCE_MOUNT2=always to workaround new mount API issues --- tests/functional/local-overlay-store/common.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 13d921c5e..1c70f947f 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -1,5 +1,7 @@ source ../common.sh +export LIBMOUNT_FORCE_MOUNT2=always + requireEnvironment () { requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" From a2a633d332f4ca3a38b47f5a524d8d07cead5917 Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Sat, 6 Apr 2024 10:26:29 -0400 Subject: [PATCH 140/211] Prevent nix-daemon.sh from leaking variable into user environment The script at `/nix/store/...-nix-2.21.0/etc/profile.d/nix-daemon.sh` was leaving behind a variable, which was visible in the user's shell environment, but not used outside the script. --- scripts/nix-profile-daemon.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..0ec72e797 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW From bd7c26bc7bb7af37c21d46faf28aa0e88606b98f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 7 Apr 2024 21:37:30 -0400 Subject: [PATCH 141/211] Add comment explaining `LIBMOUNT_FORCE_MOUNT2=always` --- .../functional/local-overlay-store/common.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 1c70f947f..2634f8c8f 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -1,5 +1,26 @@ source ../common.sh +# The new Linux mount interface does not seem to support remounting +# OverlayFS mount points. +# +# It is not clear whether this is intentional or not: +# +# The kernel source code [1] would seem to indicate merely remounting +# while *changing* mount options is now an error because it erroneously +# succeeded (by ignoring those new options) before. However, we are +# *not* trying to remount with changed options, and are still hitting +# the failure when using the new interface. +# +# For further details, see these `util-linux` issues: +# +# - https://github.com/util-linux/util-linux/issues/2528 +# - https://github.com/util-linux/util-linux/issues/2576 +# +# In the meantime, setting this environment variable to "always" will +# force the use of the old mount interface, keeping the remounting +# working and these tests passing. +# +# [1]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/fs/overlayfs/params.c?id=3006adf3be79cde4d14b1800b963b82b6e5572e0#n549 export LIBMOUNT_FORCE_MOUNT2=always requireEnvironment () { From dea23c3c9b0f0770e915bef3b1006f8f6cb225d8 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Sun, 7 Apr 2024 22:43:02 -0700 Subject: [PATCH 142/211] "but doctor, I AM the untrusted store": nix doctor had wrong trustedness This probably snuck in in a refactor using truthiness or so. The trustedness flag was having the optional fullness checked, rather than the actual contained trust level. Also adds some tests. ``` m1@6876551b-255d-4cb0-af02-8a4f17b27e2e ~ % nix store ping warning: 'nix store ping' is a deprecated alias for 'nix store info' Store URL: daemon Version: 2.20.4 Trusted: 0 m1@6876551b-255d-4cb0-af02-8a4f17b27e2e ~ % nix doctor warning: 'doctor' is a deprecated alias for 'config check' [PASS] PATH contains only one nix version. [PASS] All profiles are gcroots. [PASS] Client protocol matches store protocol. [INFO] You are trusted by store uri: daemon ``` --- src/nix/config-check.cc | 7 +++---- tests/functional/legacy-ssh-store.sh | 7 ++++++- tests/functional/remote-store.sh | 2 ++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 8d4717e15..661e1377b 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -145,10 +145,9 @@ struct CmdConfigCheck : StoreCommand void checkTrustedUser(ref store) { - std::string_view trusted = store->isTrustedClient() - ? "trusted" - : "not trusted"; - checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + auto trustedMay = store->isTrustedClient(); + std::string_view trustedness = trustedMay ? (*trustedMay ? "trusted" : "not trusted") : "unknown trust"; + checkInfo(fmt("You are %s by store uri: %s", trustedness, store->getUri())); } }; diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh index 894efccd4..79c3fba82 100644 --- a/tests/functional/legacy-ssh-store.sh +++ b/tests/functional/legacy-ssh-store.sh @@ -1,4 +1,9 @@ source common.sh +store_uri="ssh://localhost?remote-store=$TEST_ROOT/other-store" + # Check that store info trusted doesn't yet work with ssh:// -nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e 'has("trusted") | not' +nix --store "$store_uri" store info --json | jq -e 'has("trusted") | not' + +# Suppress grumpiness about multiple nixes on PATH +(nix --store "$store_uri" doctor || true) 2>&1 | grep 'You are unknown trust' diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index dc80f8b55..cc5dd1833 100644 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -13,6 +13,8 @@ startDaemon if isDaemonNewer "2.15pre0"; then # Ensure that ping works trusted with new daemon nix store info --json | jq -e '.trusted' + # Suppress grumpiness about multiple nixes on PATH + (nix doctor || true) 2>&1 | grep 'You are trusted by' else # And the the field is absent with the old daemon nix store info --json | jq -e 'has("trusted") | not' From bd8c276ddbb980be2f9024c071cd304ef8a91b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 8 Apr 2024 11:02:39 +0200 Subject: [PATCH 143/211] Improve the `config check` output for stores that don't know about trust Make it proper english --- src/nix/config-check.cc | 11 ++++++++--- tests/functional/legacy-ssh-store.sh | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 661e1377b..f7c4cebec 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -145,9 +145,14 @@ struct CmdConfigCheck : StoreCommand void checkTrustedUser(ref store) { - auto trustedMay = store->isTrustedClient(); - std::string_view trustedness = trustedMay ? (*trustedMay ? "trusted" : "not trusted") : "unknown trust"; - checkInfo(fmt("You are %s by store uri: %s", trustedness, store->getUri())); + if (auto trustedMay = store->isTrustedClient()) { + std::string_view trusted = trustedMay.value() + ? "trusted" + : "not trusted"; + checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + } else { + checkInfo(fmt("Store uri: %s doesn't have a notion of trusted user", store->getUri())); + } } }; diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh index 79c3fba82..56b4c2d20 100644 --- a/tests/functional/legacy-ssh-store.sh +++ b/tests/functional/legacy-ssh-store.sh @@ -6,4 +6,4 @@ store_uri="ssh://localhost?remote-store=$TEST_ROOT/other-store" nix --store "$store_uri" store info --json | jq -e 'has("trusted") | not' # Suppress grumpiness about multiple nixes on PATH -(nix --store "$store_uri" doctor || true) 2>&1 | grep 'You are unknown trust' +(nix --store "$store_uri" doctor || true) 2>&1 | grep "doesn't have a notion of trusted user" From 737ce5e81f4eae292d193e38fec961f105909f0b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2024 15:20:04 +0200 Subject: [PATCH 144/211] Actually run the Mercurial tests --- tests/functional/fetchMercurial.sh | 2 +- tests/functional/flakes/mercurial.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/fetchMercurial.sh b/tests/functional/fetchMercurial.sh index e6f8525c6..e133df1f8 100644 --- a/tests/functional/fetchMercurial.sh +++ b/tests/functional/fetchMercurial.sh @@ -1,6 +1,6 @@ source common.sh -[[ $(type -p hq) ]] || skipTest "Mercurial not installed" +[[ $(type -p hg) ]] || skipTest "Mercurial not installed" clearStore diff --git a/tests/functional/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh index 0622c79b7..7074af6f7 100644 --- a/tests/functional/flakes/mercurial.sh +++ b/tests/functional/flakes/mercurial.sh @@ -1,6 +1,6 @@ source ./common.sh -[[ $(type -p hq) ]] || skipTest "Mercurial not installed" +[[ $(type -p hg) ]] || skipTest "Mercurial not installed" flake1Dir=$TEST_ROOT/flake-hg1 mkdir -p $flake1Dir From e68f24f1e0d81d3d367057f149d43dfa883579cc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 Apr 2024 09:06:00 -0400 Subject: [PATCH 145/211] Remove `resolve-system-dependencies` Fix #9769 As Abathur reports, it seems to be unused since #3429 in 2020. --- .gitignore | 2 - Makefile | 1 - src/resolve-system-dependencies/local.mk | 13 -- .../resolve-system-dependencies.cc | 190 ------------------ 4 files changed, 206 deletions(-) delete mode 100644 src/resolve-system-dependencies/local.mk delete mode 100644 src/resolve-system-dependencies/resolve-system-dependencies.cc diff --git a/.gitignore b/.gitignore index 5a33c00ea..6996ca484 100644 --- a/.gitignore +++ b/.gitignore @@ -118,8 +118,6 @@ perl/Makefile.config /misc/systemd/nix-daemon.conf /misc/upstart/nix-daemon.conf -/src/resolve-system-dependencies/resolve-system-dependencies - outputs/ *.a diff --git a/Makefile b/Makefile index 5c11c4a1d..0028c957a 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,6 @@ makefiles = \ src/libutil-c/local.mk \ src/libstore-c/local.mk \ src/libexpr-c/local.mk \ - src/resolve-system-dependencies/local.mk \ scripts/local.mk \ misc/bash/local.mk \ misc/fish/local.mk \ diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk deleted file mode 100644 index e138c4080..000000000 --- a/src/resolve-system-dependencies/local.mk +++ /dev/null @@ -1,13 +0,0 @@ -ifdef HOST_DARWIN - programs += resolve-system-dependencies -endif - -resolve-system-dependencies_DIR := $(d) - -resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix - -resolve-system-dependencies_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libmain) - -resolve-system-dependencies_LIBS := libstore libmain libutil - -resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc deleted file mode 100644 index 4ea268d24..000000000 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ /dev/null @@ -1,190 +0,0 @@ -#include "derivations.hh" -#include "globals.hh" -#include "shared.hh" -#include "store-api.hh" -#include -#include -#include -#include -#include -#include -#include -#include - -#define DO_SWAP(x, y) ((x) ? OSSwapInt32(y) : (y)) - -using namespace nix; - -static auto cacheDir = Path{}; - -Path resolveCacheFile(Path lib) -{ - std::replace(lib.begin(), lib.end(), '/', '%'); - return cacheDir + "/" + lib; -} - -std::set readCacheFile(const Path & file) -{ - return tokenizeString>(readFile(file), "\n"); -} - -std::set runResolver(const Path & filename) -{ - AutoCloseFD fd = open(filename.c_str(), O_RDONLY); - if (!fd) - throw SysError("opening '%s'", filename); - - struct stat st; - if (fstat(fd.get(), &st)) - throw SysError("statting '%s'", filename); - - if (!S_ISREG(st.st_mode)) { - printError("file '%s' is not a regular MACH binary", filename); - return {}; - } - - if (st.st_size < sizeof(mach_header_64)) { - printError("file '%s' is too short for a MACH binary", filename); - return {}; - } - - char* obj = (char*) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0); - if (!obj) - throw SysError("mmapping '%s'", filename); - - ptrdiff_t mach64_offset = 0; - - uint32_t magic = ((mach_header_64*) obj)->magic; - if (magic == FAT_CIGAM || magic == FAT_MAGIC) { - bool should_swap = magic == FAT_CIGAM; - uint32_t narches = DO_SWAP(should_swap, ((fat_header *) obj)->nfat_arch); - for (uint32_t i = 0; i < narches; i++) { - fat_arch* arch = (fat_arch*) (obj + sizeof(fat_header) + sizeof(fat_arch) * i); - if (DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { - mach64_offset = (ptrdiff_t) DO_SWAP(should_swap, arch->offset); - break; - } - } - if (mach64_offset == 0) { - printError("could not find any mach64 blobs in file '%1%', continuing...", filename); - return {}; - } - } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { - mach64_offset = 0; - } else { - printError("Object file has unknown magic number '%1%', skipping it...", magic); - return {}; - } - - mach_header_64 * m_header = (mach_header_64 *) (obj + mach64_offset); - - bool should_swap = magic == MH_CIGAM_64; - ptrdiff_t cmd_offset = mach64_offset + sizeof(mach_header_64); - - std::set libs; - for (uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { - load_command * cmd = (load_command *) (obj + cmd_offset); - switch(DO_SWAP(should_swap, cmd->cmd)) { - case LC_LOAD_UPWARD_DYLIB: - case LC_LOAD_DYLIB: - case LC_REEXPORT_DYLIB: - libs.insert(std::string((char *) cmd + ((dylib_command*) cmd)->dylib.name.offset)); - break; - } - cmd_offset += DO_SWAP(should_swap, cmd->cmdsize); - } - - return libs; -} - -bool isSymlink(const Path & path) -{ - return S_ISLNK(lstat(path).st_mode); -} - -Path resolveSymlink(const Path & path) -{ - auto target = readLink(path); - return hasPrefix(target, "/") - ? target - : concatStrings(dirOf(path), "/", target); -} - -std::set resolveTree(const Path & path, PathSet & deps) -{ - std::set results; - if (!deps.insert(path).second) return {}; - for (auto & lib : runResolver(path)) { - results.insert(lib); - for (auto & p : resolveTree(lib, deps)) { - results.insert(p); - } - } - return results; -} - -std::set getPath(const Path & path) -{ - if (hasPrefix(path, "/dev")) return {}; - - Path cacheFile = resolveCacheFile(path); - if (pathExists(cacheFile)) - return readCacheFile(cacheFile); - - std::set deps, paths; - paths.insert(path); - - Path nextPath(path); - while (isSymlink(nextPath)) { - nextPath = resolveSymlink(nextPath); - paths.insert(nextPath); - } - - for (auto & t : resolveTree(nextPath, deps)) - paths.insert(t); - - writeFile(cacheFile, concatStringsSep("\n", paths)); - - return paths; -} - -int main(int argc, char ** argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); - - struct utsname _uname; - - uname(&_uname); - - auto cacheParentDir = fmt("%1%/dependency-maps", settings.nixStateDir); - - cacheDir = fmt("%1%/%2%-%3%-%4%", cacheParentDir, _uname.machine, _uname.sysname, _uname.release); - - mkdir(cacheParentDir.c_str(), 0755); - mkdir(cacheDir.c_str(), 0755); - - auto store = openStore(); - - StringSet impurePaths; - - if (std::string(argv[1]) == "--test") - impurePaths.insert(argv[2]); - else { - auto drv = store->derivationFromPath(store->parseStorePath(argv[1])); - impurePaths = tokenizeString(getOr(drv.env, "__impureHostDeps", "")); - impurePaths.insert("/usr/lib/libSystem.dylib"); - } - - std::set allPaths; - - for (auto & path : impurePaths) - for (auto & p : getPath(path)) - allPaths.insert(p); - - std::cout << "extra-chroot-dirs" << std::endl; - for (auto & path : allPaths) - std::cout << path << std::endl; - std::cout << std::endl; - }); -} From d29786f25854c910e08d9a8438d589f02adc9569 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Apr 2024 16:35:12 +0200 Subject: [PATCH 146/211] downloadFile(): Remove the "locked" (aka "immutable") flag This was used in only one place, namely builtins.fetchurl with an expected hash. Since this can cause similar issues as described in #9814 and #9905 with the "locked" flag for fetchTarball and fetchTree, let's just remove it. Note that if an expected hash is given and the hash algorithm is SHA-256, then we will never do a download anyway if the resulting store path already exists. So removing the "locked" flag will only cause potentially unnecessary HTTP requests (subject to the tarball TTL) for non-SHA-256 hashes. --- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/github.cc | 8 ++++---- src/libfetchers/registry.cc | 2 +- src/libfetchers/tarball.cc | 5 ++--- src/libfetchers/tarball.hh | 1 - src/nix-channel/nix-channel.cc | 6 +++--- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 7906f8ac3..0190aca3d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -473,7 +473,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto storePath = unpack ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name) - : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + : fetchers::downloadFile(state.store, *url, name).storePath; if (expectedHash) { auto hash = unpack diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 60e323464..985f2e479 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -357,7 +357,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + downloadFile(store, url, "source", headers).storePath))); return RefInfo { .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1), @@ -431,7 +431,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + downloadFile(store, url, "source", headers).storePath))); return RefInfo { .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) @@ -495,7 +495,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( - downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); + downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; getline(is, line); @@ -511,7 +511,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::regex refRegex(refUri); auto file = store->toRealPath( - downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); + downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 9c7bc0cfe..e00b9de46 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -158,7 +158,7 @@ static std::shared_ptr getGlobalRegistry(ref store) } if (!hasPrefix(path, "/")) { - auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + auto storePath = downloadFile(store, path, "flake-registry.json").storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json"); path = store->toRealPath(storePath); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index f08509cb7..a1f934c35 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -19,7 +19,6 @@ DownloadFileResult downloadFile( ref store, const std::string & url, const std::string & name, - bool locked, const Headers & headers) { // FIXME: check store @@ -101,7 +100,7 @@ DownloadFileResult downloadFile( inAttrs, infoAttrs, *storePath, - locked); + false); } return { @@ -306,7 +305,7 @@ struct FileInputScheme : CurlInputScheme the Nix store directly, since there is little deduplication benefit in using the Git cache for single big files like tarballs. */ - auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); + auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName()); auto narHash = store->queryPathInfo(file.storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh index 77ad3bf09..bcb5dcc5e 100644 --- a/src/libfetchers/tarball.hh +++ b/src/libfetchers/tarball.hh @@ -25,7 +25,6 @@ DownloadFileResult downloadFile( ref store, const std::string & url, const std::string & name, - bool locked, const Headers & headers = {}); struct DownloadTarballResult diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 48553fa31..9f7f557b5 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -112,7 +112,7 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url))); auto filename = store->toRealPath(result.storePath); url = result.effectiveUrl; @@ -126,9 +126,9 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); } } // Regardless of where it came from, add the expression representing this channel to accumulated expression From f34b8de5b2ba9f5ed92924e30661b70ed427a123 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 9 Apr 2024 21:27:00 +0200 Subject: [PATCH 147/211] doc/rl-2.20: add missing entry about `nix copy --to ssh-ng://...` This requires `--substitute-on-destination` if you want the remote side to substitute instead of copying if possible. For completeness sake, document it here. Also, the stable Nix from nixpkgs is still 2.18, so more folks may stumble upon this when this is bumped, so I'd expect this to be actually useful. Closes #10182 --- doc/manual/src/release-notes/rl-2.20.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..d36dd4784 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,8 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by `--builders-use-substitutes`. From 93d68e18e588aa82e144fb6ab0cd1be0eae2e413 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 13 Feb 2024 15:22:31 -0500 Subject: [PATCH 148/211] Make `outputHashAlgo` accept `"nar"`, stay in sync Now that we have a few things identifying content address methods by name, we should be consistent about it. Move up the `parseHashAlgoOpt` for tidiness too. Discussed this change for consistency's sake as part of #8876 Co-authored-by: Eelco Dolstra --- .../src/language/advanced-attributes.md | 9 +++++++-- src/libexpr/primops.cc | 20 ++++++++++--------- tests/functional/fixed.nix | 2 ++ tests/functional/fixed.sh | 4 ++++ 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index b3e3afe3b..16dcc6ba9 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -207,12 +207,17 @@ Derivations can declare some infrequently used optional attributes. This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output + - `"recursive"` or `"nar"`\ + The hash is computed over the [NAR archive](@docroot@/glossary.md#gloss-nar) dump of the output (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In this case, the output can be anything, including a directory tree. + `"recursive"` is the traditional way of indicating this, + and is supported since 2005 (virtually the entire history of Nix). + `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + however support for it is only added in Nix version 2.21. + - [`__contentAddressed`]{#adv-attr-__contentAddressed} > **Warning** > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a2b3512b..0911ce117 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1139,18 +1139,20 @@ drvName, Bindings * attrs, Value & v) vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string_view s) { - if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; - else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; - else if (s == "git") { - experimentalFeatureSettings.require(Xp::GitHashing); - ingestionMethod = FileIngestionMethod::Git; - } else if (s == "text") { - experimentalFeatureSettings.require(Xp::DynamicDerivations); - ingestionMethod = TextIngestionMethod {}; - } else + if (s == "recursive") { + // back compat, new name is "nar" + ingestionMethod = FileIngestionMethod::Recursive; + } else try { + ingestionMethod = ContentAddressMethod::parse(s); + } catch (UsageError &) { state.error( "invalid value '%s' for 'outputHashMode' attribute", s ).atPos(v).debugThrow(); + } + if (ingestionMethod == TextIngestionMethod {}) + experimentalFeatureSettings.require(Xp::DynamicDerivations); + if (ingestionMethod == FileIngestionMethod::Git) + experimentalFeatureSettings.require(Xp::GitHashing); }; auto handleOutputs = [&](const Strings & ss) { diff --git a/tests/functional/fixed.nix b/tests/functional/fixed.nix index 5bdf79333..a920a2167 100644 --- a/tests/functional/fixed.nix +++ b/tests/functional/fixed.nix @@ -64,4 +64,6 @@ rec { (f2 "bar" ./fixed.builder2.sh "recursive" "md5" "3670af73070fa14077ad74e0f5ea4e42") ]; + # Can use "nar" instead of "recursive" now. + nar-not-recursive = f2 "foo" ./fixed.builder2.sh "nar" "md5" "3670af73070fa14077ad74e0f5ea4e42"; } diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index d98d4cd15..7bbecda91 100644 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -61,3 +61,7 @@ out3=$(nix-store --add-fixed --recursive sha256 $TEST_ROOT/fixed) out4=$(nix-store --print-fixed-path --recursive sha256 "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik" fixed) [ "$out" = "$out4" ] + +# Can use `outputHashMode = "nar";` instead of `"recursive"` now. +clearStore +nix-build fixed.nix -A nar-not-recursive --no-out-link From 872d93eb13f22e8705e03903b65c7eba8b26a99b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 10 Apr 2024 15:17:39 +0200 Subject: [PATCH 149/211] Add a test for depending on a symlink store path Regression test for https://github.com/NixOS/nix/issues/9579 --- tests/functional/linux-sandbox.sh | 3 +++ tests/functional/symlink-derivation.nix | 36 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/functional/symlink-derivation.nix diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index ff7d257bd..04209277b 100644 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -73,3 +73,6 @@ testCert missing fixed-output "$nocert" # Cert in sandbox when ssl-cert-file is set to an existing file testCert present fixed-output "$cert" + +# Symlinks should be added in the sandbox directly and not followed +nix-sandbox-build symlink-derivation.nix diff --git a/tests/functional/symlink-derivation.nix b/tests/functional/symlink-derivation.nix new file mode 100644 index 000000000..17ba37424 --- /dev/null +++ b/tests/functional/symlink-derivation.nix @@ -0,0 +1,36 @@ +with import ./config.nix; + +let + foo_in_store = builtins.toFile "foo" "foo"; + foo_symlink = mkDerivation { + name = "foo-symlink"; + buildCommand = '' + ln -s ${foo_in_store} $out + ''; + }; + symlink_to_not_in_store = mkDerivation { + name = "symlink-to-not-in-store"; + buildCommand = '' + ln -s ${builtins.toString ./.} $out + ''; + }; +in +mkDerivation { + name = "depends-on-symlink"; + buildCommand = '' + ( + set -x + + # `foo_symlink` should be a symlink pointing to `foo_in_store` + [[ -L ${foo_symlink} ]] + [[ $(readlink ${foo_symlink}) == ${foo_in_store} ]] + + # `symlink_to_not_in_store` should be a symlink pointing to `./.`, which + # is not available in the sandbox + [[ -L ${symlink_to_not_in_store} ]] + [[ $(readlink ${symlink_to_not_in_store}) == ${builtins.toString ./.} ]] + (! ls ${symlink_to_not_in_store}/) + ) + echo "Success!" > $out + ''; +} From 913db9f7385b8717d9eaf6269e9f319e78e4c564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 10 Apr 2024 15:19:18 +0200 Subject: [PATCH 150/211] Fix permission denied when building symlink derivation which points to a symlink out of the store Bind-mounting symlinks is apparently not possible, which is why the thing was failing. Fortunately, symlinks are small, so we can fallback to copy them at no cost. Fix https://github.com/NixOS/nix/issues/9579 Co-authored-by: Artturin --- src/libstore/build/local-derivation-goal.cc | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ab66195b8..68c387a9d 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -396,20 +397,30 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() static void doBind(const Path & source, const Path & target, bool optional = false) { debug("bind mounting '%1%' to '%2%'", source, target); struct stat st; - if (stat(source.c_str(), &st) == -1) { + + auto bindMount = [&]() { + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + if (lstat(source.c_str(), &st) == -1) { if (optional && errno == ENOENT) return; else throw SysError("getting attributes of path '%1%'", source); } - if (S_ISDIR(st.st_mode)) + if (S_ISDIR(st.st_mode)) { createDirs(target); - else { + bindMount(); + } else if (S_ISLNK(st.st_mode)) { + // Symlinks can (apparently) not be bind-mounted, so just copy it + createDirs(dirOf(target)); + copyFile(source, target, /* andDelete */ false); + } else { createDirs(dirOf(target)); writeFile(target, ""); + bindMount(); } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); }; #endif From ae4737294e91ab93526612b17950e1bc4f0b47f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 10 Apr 2024 15:17:56 +0200 Subject: [PATCH 151/211] doBind: Use our own lstat wrapper Doesn't change much, but brings a bit more consistency to the code --- src/libstore/build/local-derivation-goal.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 68c387a9d..db12af810 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -24,7 +24,6 @@ #include #include -#include #include #include #include @@ -396,19 +395,21 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() #if __linux__ static void doBind(const Path & source, const Path & target, bool optional = false) { debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; auto bindMount = [&]() { if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) throw SysError("bind mount from '%1%' to '%2%' failed", source, target); }; - if (lstat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) + auto maybeSt = maybeLstat(source); + if (!maybeSt) { + if (optional) return; else throw SysError("getting attributes of path '%1%'", source); } + auto st = *maybeSt; + if (S_ISDIR(st.st_mode)) { createDirs(target); bindMount(); From 50557adb3b58445f1ca176bc6f653afa5151567c Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 10 Apr 2024 17:26:58 +0200 Subject: [PATCH 152/211] doc/rl-2.20: clarify builders-use-substitutes vs. substitute-on-destination ...as this lead to confusion before. --- doc/manual/src/release-notes/rl-2.20.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index d36dd4784..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -204,4 +204,5 @@ - `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) in order to substitute paths on the remote store instead of copying them. The behavior is consistent with `nix copy` to a different kind of remote store. - Previously this behavior was controlled by `--builders-use-substitutes`. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. From 664532c533457fb7bda113ea432fb53710fd7d92 Mon Sep 17 00:00:00 2001 From: Ivan Trubach Date: Wed, 10 Apr 2024 13:34:17 +0300 Subject: [PATCH 153/211] Do not rely on $stdenv/setup to set output variables Instead of relying on setup script to set output variables when structured attributes are enabled, iterate over the values of an outputs associative array. See also https://github.com/NixOS/nixpkgs/blob/374fa3532ee7d7496165d3b5a6652a3dcad1ebc2/pkgs/stdenv/generic/setup.sh#L23-L26 --- src/nix/develop.cc | 24 +++++++++++++++----- src/nix/get-env.sh | 33 ++++++++++++++++------------ tests/functional/shell.nix | 8 ------- tests/functional/structured-attrs.sh | 2 +- 4 files changed, 38 insertions(+), 29 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index bb96f7786..b654dc52f 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -177,6 +177,14 @@ struct BuildEnvironment throw Error("bash variable is not a string"); } + static Associative getAssociative(const Value & value) + { + if (auto assoc = std::get_if(&value)) + return *assoc; + else + throw Error("bash variable is not an associative array"); + } + static Array getStrings(const Value & value) { if (auto str = std::get_if(&value)) @@ -362,13 +370,17 @@ struct Common : InstallableCommand, MixProfile auto outputs = buildEnvironment.vars.find("outputs"); assert(outputs != buildEnvironment.vars.end()); - // FIXME: properly unquote 'outputs'. StringMap rewrites; - for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { - auto from = buildEnvironment.vars.find(outputName); - assert(from != buildEnvironment.vars.end()); - // FIXME: unquote - rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); + if (buildEnvironment.providesStructuredAttrs()) { + for (auto & [outputName, from] : BuildEnvironment::getAssociative(outputs->second)) { + rewrites.insert({from, outputsDir + "/" + outputName}); + } + } else { + for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { + auto from = buildEnvironment.vars.find(outputName); + assert(from != buildEnvironment.vars.end()); + rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); + } } /* Substitute redirects. */ diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 832cc2f11..071edf9b9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -128,20 +128,25 @@ __escapeString() { printf '"%s"' "$__s" } -# In case of `__structuredAttrs = true;` the list of outputs is an associative -# array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` -# must contain the array's keys (hence `${!...[@]}`) in this case. -if [ -e "$NIX_ATTRS_SH_FILE" ]; then - __olist="${!outputs[@]}" -else - __olist=$outputs -fi - -for __output in $__olist; do - if [[ -z $__done ]]; then - __dumpEnv > ${!__output} +__dumpEnvToOutput() { + local __output="$1" + if [[ -z ${__done-} ]]; then + __dumpEnv > "$__output" __done=1 else - echo -n >> "${!__output}" + echo -n >> "$__output" fi -done +} + +# In case of `__structuredAttrs = true;` the list of outputs is an associative +# array with a format like `outname => /nix/store/hash-drvname-outname`. +# Otherwise it is a space-separated list of output variable names. +if [ -e "$NIX_ATTRS_SH_FILE" ]; then + for __output in "${outputs[@]}"; do + __dumpEnvToOutput "$__output" + done +else + for __outname in $outputs; do + __dumpEnvToOutput "${!__outname}" + done +fi diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix index 92d94fbc2..6a7dd7ad1 100644 --- a/tests/functional/shell.nix +++ b/tests/functional/shell.nix @@ -21,14 +21,6 @@ let pkgs = rec { export PATH=$PATH:$pkg/bin done - # mimic behavior of stdenv for `$out` etc. for structured attrs. - if [ -n "''${NIX_ATTRS_SH_FILE}" ]; then - for o in "''${!outputs[@]}"; do - eval "''${o}=''${outputs[$o]}" - export "''${o}" - done - fi - declare -a arr1=(1 2 "3 4" 5) declare -a arr2=(x $'\n' $'x\ny') fun() { diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index f11992dcd..6711efbb4 100644 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -32,4 +32,4 @@ jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")" -test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')" +test "$(<<<"$jsonOut" jq '.variables.outputs.value.out' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')" From 3e5797e97fe73f0468e2b3faa0ce6b1860617137 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 10 Apr 2024 15:21:22 -0400 Subject: [PATCH 154/211] Document the Nix Archive format This is adopted from Eelco's PhD thesis. --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/protocols/nix-archive.md | 42 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 doc/manual/src/protocols/nix-archive.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 43b9e925f..d9044fbda 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -110,6 +110,7 @@ - [Derivation](protocols/json/derivation.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) + - [Nix Archive (NAR) Format](protocols/nix-archive.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) - [Contributing](contributing/index.md) diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..4fb6282ee --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,42 @@ +# Nix Archive (NAR) format + +This is the complete specification of the Nix Archive format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = str("type"), str("directory") { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte From 19c8867d2a916fd5af46d8bc6b9449875145abb0 Mon Sep 17 00:00:00 2001 From: Nikhil Dhiman Date: Thu, 11 Apr 2024 02:41:57 +0530 Subject: [PATCH 155/211] Fix store-path.md (#10457) Co-authored-by: John Ericson Co-authored-by: Cole Helbling --- doc/manual/src/store/store-path.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..085aead51 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -46,7 +46,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > From 85b9f4ef4fcc7b7fc03ff9503f280d736b5c65c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Apr 2024 23:46:19 +0200 Subject: [PATCH 156/211] nix shell: Handle output paths that are symlinks This requires moving resolveSymlinks() into SourceAccessor. Also, it requires LocalStoreAccessor::maybeLstat() to work on parents of the store (to avoid an error like "/nix is not in the store"). Fixes #10375. --- src/libstore/local-fs-store.cc | 4 +++ src/libutil/source-accessor.cc | 44 +++++++++++++++++++++++++++++--- src/libutil/source-accessor.hh | 37 ++++++++++++++++++++++++--- src/libutil/source-path.cc | 38 --------------------------- src/libutil/source-path.hh | 31 ++++------------------ src/nix/unix/run.cc | 3 ++- tests/functional/shell-hello.nix | 10 +++++++- tests/functional/shell.sh | 2 ++ 8 files changed, 97 insertions(+), 72 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 81c385ddb..843c0d288 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -33,6 +33,10 @@ struct LocalStoreAccessor : PosixSourceAccessor std::optional maybeLstat(const CanonPath & path) override { + /* Handle the case where `path` is (a parent of) the store. */ + if (isDirOrInDir(store->storeDir, path.abs())) + return Stat{ .type = tDirectory }; + return PosixSourceAccessor::maybeLstat(toRealPath(path)); } diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index afbbbe1a9..66093d2cc 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -39,9 +39,9 @@ void SourceAccessor::readFile( } Hash SourceAccessor::hashPath( - const CanonPath & path, - PathFilter & filter, - HashAlgorithm ha) + const CanonPath & path, + PathFilter & filter, + HashAlgorithm ha) { HashSink sink(ha); dumpPath(path, sink, filter); @@ -67,4 +67,42 @@ std::string SourceAccessor::showPath(const CanonPath & path) return displayPrefix + path.abs() + displaySuffix; } +CanonPath SourceAccessor::resolveSymlinks( + const CanonPath & path, + SymlinkResolution mode) +{ + auto res = CanonPath::root; + + int linksAllowed = 1024; + + std::list todo; + for (auto & c : path) + todo.push_back(std::string(c)); + + while (!todo.empty()) { + auto c = *todo.begin(); + todo.pop_front(); + if (c == "" || c == ".") + ; + else if (c == "..") + res.pop(); + else { + res.push(c); + if (mode == SymlinkResolution::Full || !todo.empty()) { + if (auto st = maybeLstat(res); st && st->type == SourceAccessor::tSymlink) { + if (!linksAllowed--) + throw Error("infinite symlink recursion in path '%s'", showPath(path)); + auto target = readLink(res); + res.pop(); + if (hasPrefix(target, "/")) + res = CanonPath::root; + todo.splice(todo.begin(), tokenizeString>(target, "/")); + } + } + } + } + + return res; +} + } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index aff7da09c..1f272327f 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -9,6 +9,26 @@ namespace nix { struct Sink; +/** + * Note there is a decent chance this type soon goes away because the problem is solved another way. + * See the discussion in https://github.com/NixOS/nix/pull/9985. + */ +enum class SymlinkResolution { + /** + * Resolve symlinks in the ancestors only. + * + * Only the last component of the result is possibly a symlink. + */ + Ancestors, + + /** + * Resolve symlinks fully, realpath(3)-style. + * + * No component of the result will be a symlink. + */ + Full, +}; + /** * A read-only filesystem abstraction. This is used by the Nix * evaluator and elsewhere for accessing sources in various @@ -112,9 +132,9 @@ struct SourceAccessor PathFilter & filter = defaultPathFilter); Hash hashPath( - const CanonPath & path, - PathFilter & filter = defaultPathFilter, - HashAlgorithm ha = HashAlgorithm::SHA256); + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashAlgorithm ha = HashAlgorithm::SHA256); /** * Return a corresponding path in the root filesystem, if @@ -137,6 +157,17 @@ struct SourceAccessor void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); virtual std::string showPath(const CanonPath & path); + + /** + * Resolve any symlinks in `path` according to the given + * resolution mode. + * + * @param mode might only be a temporary solution for this. + * See the discussion in https://github.com/NixOS/nix/pull/9985. + */ + CanonPath resolveSymlinks( + const CanonPath & path, + SymlinkResolution mode = SymlinkResolution::Full); }; } diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc index 56ae1d699..2a5b20858 100644 --- a/src/libutil/source-path.cc +++ b/src/libutil/source-path.cc @@ -62,44 +62,6 @@ bool SourcePath::operator<(const SourcePath & x) const return std::tie(*accessor, path) < std::tie(*x.accessor, x.path); } -SourcePath SourcePath::resolveSymlinks(SymlinkResolution mode) const -{ - auto res = SourcePath(accessor); - - int linksAllowed = 1024; - - std::list todo; - for (auto & c : path) - todo.push_back(std::string(c)); - - bool resolve_last = mode == SymlinkResolution::Full; - - while (!todo.empty()) { - auto c = *todo.begin(); - todo.pop_front(); - if (c == "" || c == ".") - ; - else if (c == "..") - res.path.pop(); - else { - res.path.push(c); - if (resolve_last || !todo.empty()) { - if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { - if (!linksAllowed--) - throw Error("infinite symlink recursion in path '%s'", path); - auto target = res.readLink(); - res.path.pop(); - if (hasPrefix(target, "/")) - res.path = CanonPath::root; - todo.splice(todo.begin(), tokenizeString>(target, "/")); - } - } - } - } - - return res; -} - std::ostream & operator<<(std::ostream & str, const SourcePath & path) { str << path.to_string(); diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 59991c640..b8f69af12 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -11,26 +11,6 @@ namespace nix { -/** - * Note there is a decent chance this type soon goes away because the problem is solved another way. - * See the discussion in https://github.com/NixOS/nix/pull/9985. - */ -enum class SymlinkResolution { - /** - * Resolve symlinks in the ancestors only. - * - * Only the last component of the result is possibly a symlink. - */ - Ancestors, - - /** - * Resolve symlinks fully, realpath(3)-style. - * - * No component of the result will be a symlink. - */ - Full, -}; - /** * An abstraction for accessing source files during * evaluation. Currently, it's just a wrapper around `CanonPath` that @@ -123,14 +103,13 @@ struct SourcePath bool operator<(const SourcePath & x) const; /** - * Resolve any symlinks in this `SourcePath` according to the - * given resolution mode. - * - * @param mode might only be a temporary solution for this. - * See the discussion in https://github.com/NixOS/nix/pull/9985. + * Convenience wrapper around `SourceAccessor::resolveSymlinks()`. */ SourcePath resolveSymlinks( - SymlinkResolution mode = SymlinkResolution::Full) const; + SymlinkResolution mode = SymlinkResolution::Full) const + { + return {accessor, accessor->resolveSymlinks(path, mode)}; + } }; std::ostream & operator << (std::ostream & str, const SourcePath & path); diff --git a/src/nix/unix/run.cc b/src/nix/unix/run.cc index 02e809e5c..dfd8b643c 100644 --- a/src/nix/unix/run.cc +++ b/src/nix/unix/run.cc @@ -124,7 +124,8 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (true) pathAdditions.push_back(store->printStorePath(path) + "/bin"); - auto propPath = CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"; + auto propPath = accessor->resolveSymlinks( + CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"); if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix index dfe66ef93..5c9b7a4d9 100644 --- a/tests/functional/shell-hello.nix +++ b/tests/functional/shell-hello.nix @@ -1,6 +1,6 @@ with import ./config.nix; -{ +rec { hello = mkDerivation { name = "hello"; outputs = [ "out" "dev" ]; @@ -24,6 +24,14 @@ with import ./config.nix; ''; }; + hello-symlink = mkDerivation { + name = "hello-symlink"; + buildCommand = + '' + ln -s ${hello} $out + ''; + }; + salve-mundi = mkDerivation { name = "salve-mundi"; outputs = [ "out" ]; diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index 8bbeabedf..d89801929 100644 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -10,6 +10,8 @@ nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2' nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2' +# Test output paths that are a symlink. +#nix shell -f shell-hello.nix hello-symlink -c hello | grep 'Hello World' if isDaemonNewer "2.20.0pre20231220"; then # Test that command line attribute ordering is reflected in the PATH From 9d50f57fa360df96a4f92c73db25dcec2a6f39d4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Apr 2024 09:00:47 +0200 Subject: [PATCH 157/211] Doh --- tests/functional/shell.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index d89801929..abc091d92 100644 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -11,7 +11,7 @@ nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2' nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2' # Test output paths that are a symlink. -#nix shell -f shell-hello.nix hello-symlink -c hello | grep 'Hello World' +nix shell -f shell-hello.nix hello-symlink -c hello | grep 'Hello World' if isDaemonNewer "2.20.0pre20231220"; then # Test that command line attribute ordering is reflected in the PATH From 26a4688a868e848760908ee15434eff2774952c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Apr 2024 09:04:26 +0200 Subject: [PATCH 158/211] nix shell: Test that store paths cannot link outside of the store --- tests/functional/shell-hello.nix | 8 ++++++++ tests/functional/shell.sh | 3 +++ 2 files changed, 11 insertions(+) diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix index 5c9b7a4d9..c46fdec8a 100644 --- a/tests/functional/shell-hello.nix +++ b/tests/functional/shell-hello.nix @@ -32,6 +32,14 @@ rec { ''; }; + forbidden-symlink = mkDerivation { + name = "forbidden-symlink"; + buildCommand = + '' + ln -s /tmp/foo/bar $out + ''; + }; + salve-mundi = mkDerivation { name = "salve-mundi"; outputs = [ "out" ]; diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index abc091d92..8a3fef3e7 100644 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -13,6 +13,9 @@ nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2' # Test output paths that are a symlink. nix shell -f shell-hello.nix hello-symlink -c hello | grep 'Hello World' +# Test that symlinks outside of the store don't work. +expect 1 nix shell -f shell-hello.nix forbidden-symlink -c hello 2>&1 | grepQuiet "is not in the Nix store" + if isDaemonNewer "2.20.0pre20231220"; then # Test that command line attribute ordering is reflected in the PATH # https://github.com/NixOS/nix/issues/7905 From 1f73de262961510318328463f943175833994a50 Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Thu, 14 Mar 2024 16:51:44 +0100 Subject: [PATCH 159/211] git fetcher: relax absolute URL check of resolveSubmoduleUrl This matches up the behavior with the internals of libgit2 Fixes #9979 --- src/libfetchers/git-utils.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 5e560f5f3..5777240ad 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -313,7 +313,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::string res(buf.ptr); - if (!hasPrefix(res, "/") && res.find("://") == res.npos) + // Git has a default protocol of 'ssh' for URLs without a protocol: + // https://git-scm.com/book/en/v2/Git-on-the-Server-The-Protocols#_the_ssh_protocol + // This code matches what git_submodule_resolve_url does inside of itself to see if the URL is already absolute. + // Inside libgit2 it just checks whether there's any ':' in the URL to default to the ssh:// protocol. + if (!hasPrefix(res, "/") && res.find(":") == res.npos) res = parseURL(base + "/" + res).canonicalise().to_string(); return res; From 1a76ca416108c8aefbafbf20f58f66725166b9f9 Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Fri, 15 Mar 2024 09:30:58 +0100 Subject: [PATCH 160/211] Set the origin instead of hacking in the URL resolving --- src/libfetchers/git-utils.cc | 18 +++++++----------- src/libfetchers/git-utils.hh | 6 +++--- src/libfetchers/unix/git.cc | 5 ++++- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 5777240ad..5ecd825b7 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -198,6 +198,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return git_repository_is_shallow(*this); } + void setRemote(const std::string & name, const std::string & url) override + { + if (git_remote_set_url(*this, name.c_str(), url.c_str())) + throw Error("setting remote '%s' URL to '%s': %s", name, url, git_error_last()->message); + } + Hash resolveRef(std::string ref) override { Object object; @@ -302,9 +308,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::vector> getSubmodules(const Hash & rev, bool exportIgnore) override; - std::string resolveSubmoduleUrl( - const std::string & url, - const std::string & base) override + std::string resolveSubmoduleUrl(const std::string & url) override { git_buf buf = GIT_BUF_INIT; if (git_submodule_resolve_url(&buf, *this, url.c_str())) @@ -312,14 +316,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this Finally cleanup = [&]() { git_buf_dispose(&buf); }; std::string res(buf.ptr); - - // Git has a default protocol of 'ssh' for URLs without a protocol: - // https://git-scm.com/book/en/v2/Git-on-the-Server-The-Protocols#_the_ssh_protocol - // This code matches what git_submodule_resolve_url does inside of itself to see if the URL is already absolute. - // Inside libgit2 it just checks whether there's any ':' in the URL to default to the ssh:// protocol. - if (!hasPrefix(res, "/") && res.find(":") == res.npos) - res = parseURL(base + "/" + res).canonicalise().to_string(); - return res; } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index fbb2d947b..600a42da0 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -32,6 +32,8 @@ struct GitRepo /* Return the commit hash to which a ref points. */ virtual Hash resolveRef(std::string ref) = 0; + virtual void setRemote(const std::string & name, const std::string & url) = 0; + /** * Info about a submodule. */ @@ -69,9 +71,7 @@ struct GitRepo */ virtual std::vector> getSubmodules(const Hash & rev, bool exportIgnore) = 0; - virtual std::string resolveSubmoduleUrl( - const std::string & url, - const std::string & base) = 0; + virtual std::string resolveSubmoduleUrl(const std::string & url) = 0; virtual bool hasObject(const Hash & oid) = 0; diff --git a/src/libfetchers/unix/git.cc b/src/libfetchers/unix/git.cc index 0966c4710..45e62ebe1 100644 --- a/src/libfetchers/unix/git.cc +++ b/src/libfetchers/unix/git.cc @@ -526,6 +526,9 @@ struct GitInputScheme : InputScheme auto repo = GitRepo::openRepo(cacheDir, true, true); + // We need to set the origin so resolving submodule URLs works + repo->setRemote("origin", repoInfo.url); + Path localRefFile = ref.compare(0, 5, "refs/") == 0 ? cacheDir + "/" + ref @@ -629,7 +632,7 @@ struct GitInputScheme : InputScheme std::map> mounts; for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, exportIgnore)) { - auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url); + auto resolved = repo->resolveSubmoduleUrl(submodule.url); debug("Git submodule %s: %s %s %s -> %s", submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); fetchers::Attrs attrs; From cd06193d1387921caed8aa78b3f3617bea11fd00 Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Thu, 11 Apr 2024 14:58:42 +0200 Subject: [PATCH 161/211] Add nixos test --- tests/nixos/default.nix | 2 ++ tests/nixos/git-submodules.nix | 54 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 tests/nixos/git-submodules.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 627728424..4edf40c16 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -145,6 +145,8 @@ in githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix; + gitSubmodules = runNixOSTestFor "x86_64-linux" ./git-submodules.nix; + sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./sourcehut-flakes.nix; tarballFlakes = runNixOSTestFor "x86_64-linux" ./tarball-flakes.nix; diff --git a/tests/nixos/git-submodules.nix b/tests/nixos/git-submodules.nix new file mode 100644 index 000000000..9264369ef --- /dev/null +++ b/tests/nixos/git-submodules.nix @@ -0,0 +1,54 @@ +# Test Nix's remote build feature. + +{ lib, hostPkgs, ... }: + +{ + config = { + name = lib.mkDefault "git-submodules"; + + nodes = + { + remote = + { config, pkgs, ... }: + { + services.openssh.enable = true; + environment.systemPackages = [ pkgs.git ]; + }; + + client = + { config, lib, pkgs, ... }: + { + programs.ssh.extraConfig = "ConnectTimeout 30"; + environment.systemPackages = [ pkgs.git ]; + nix.extraOptions = "experimental-features = nix-command flakes"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builders. + client.wait_for_unit("network.target") + + remote.succeed("mkdir -p -m 700 /root/.ssh") + remote.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + remote.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {remote.name} 'echo hello world'") + + remote.succeed("git init bar && git -C bar config user.email foobar@example.com && git -C bar config user.name Foobar && echo test >> bar/content && git -C bar add content && git -C bar commit -m 'Initial commit'") + client.succeed(f"git init foo && git -C foo config user.email foobar@example.com && git -C foo config user.name Foobar && git -C foo submodule add root@{remote.name}:/tmp/bar sub && git -C foo add sub && git -C foo commit -m 'Add submodule'") + client.succeed("nix --flake-registry \"\" flake prefetch 'git+file:///tmp/foo?submodules=1&ref=master'") + ''; + }; +} From 1e4f902b28231497c6f2fe6db3f2d85352efc752 Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Thu, 11 Apr 2024 15:31:21 +0200 Subject: [PATCH 162/211] Add gitSubmodules test to github actions --- .github/workflows/ci.yml | 2 +- tests/nixos/git-submodules.nix | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b8eac49d..cfd1dec5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,4 +166,4 @@ jobs: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes + - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.gitSubmodules diff --git a/tests/nixos/git-submodules.nix b/tests/nixos/git-submodules.nix index 9264369ef..570b1822b 100644 --- a/tests/nixos/git-submodules.nix +++ b/tests/nixos/git-submodules.nix @@ -46,8 +46,24 @@ remote.wait_for_unit("sshd") client.succeed(f"ssh -o StrictHostKeyChecking=no {remote.name} 'echo hello world'") - remote.succeed("git init bar && git -C bar config user.email foobar@example.com && git -C bar config user.name Foobar && echo test >> bar/content && git -C bar add content && git -C bar commit -m 'Initial commit'") - client.succeed(f"git init foo && git -C foo config user.email foobar@example.com && git -C foo config user.name Foobar && git -C foo submodule add root@{remote.name}:/tmp/bar sub && git -C foo add sub && git -C foo commit -m 'Add submodule'") + remote.succeed(""" + git init bar + git -C bar config user.email foobar@example.com + git -C bar config user.name Foobar + echo test >> bar/content + git -C bar add content + git -C bar commit -m 'Initial commit' + """) + + client.succeed(f""" + git init foo + git -C foo config user.email foobar@example.com + git -C foo config user.name Foobar + git -C foo submodule add root@{remote.name}:/tmp/bar sub + git -C foo add sub + git -C foo commit -m 'Add submodule' + """) + client.succeed("nix --flake-registry \"\" flake prefetch 'git+file:///tmp/foo?submodules=1&ref=master'") ''; }; From ed13cf05a224a08e4f120e0c932289db2d20be84 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 7 Apr 2024 16:31:57 +0200 Subject: [PATCH 163/211] build-hook: Allow empty Like always declining; local builds only, as can be inferred from the docs. (Not worth spending too many words on this pretty obvious behavior, I think. Also, plans to remove it? https://github.com/NixOS/nix/issues/1221) --- src/libstore/build/derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 29bf2852f..4d4342996 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1138,7 +1138,7 @@ void DerivationGoal::resolvedFinished() HookReply DerivationGoal::tryBuildHook() { - if (!worker.tryBuildHook || !useDerivation) return rpDecline; + if (settings.buildHook.get().empty() || !worker.tryBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique(); From 94d9819bdc9dfa0a62c1e75fc90b2df0409be1ec Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 7 Apr 2024 16:36:05 +0200 Subject: [PATCH 164/211] tests/unit/libexpr/main: Fix realisation --- tests/unit/libexpr/main.cc | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/unit/libexpr/main.cc diff --git a/tests/unit/libexpr/main.cc b/tests/unit/libexpr/main.cc new file mode 100644 index 000000000..cf7fcf5a3 --- /dev/null +++ b/tests/unit/libexpr/main.cc @@ -0,0 +1,39 @@ +#include +#include +#include "globals.hh" +#include "logging.hh" + +using namespace nix; + +int main (int argc, char **argv) { + if (argc > 1 && std::string_view(argv[1]) == "__build-remote") { + printError("test-build-remote: not supported in libexpr unit tests"); + return 1; + } + + // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook. + settings.buildHook = {}; + + #if __linux__ // should match the conditional around sandboxBuildDir declaration. + + // When building and testing nix within the host's Nix sandbox, our store dir will be located in the host's sandboxBuildDir, e.g.: + // Host + // storeDir = /nix/store + // sandboxBuildDir = /build + // This process + // storeDir = /build/foo/bar/store + // sandboxBuildDir = /build + // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different sandboxBuildDir. + settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir"; + #endif + + #if __APPLE__ + // Avoid this error, when already running in a sandbox: + // sandbox-exec: sandbox_apply: Operation not permitted + settings.sandboxMode = smDisabled; + setEnv("_NIX_TEST_NO_SANDBOX", "1"); + #endif + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 48808a53205ceec94bad761266c4325fd28963d1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 7 Apr 2024 21:19:44 +0200 Subject: [PATCH 165/211] tests/unit/libexpr: Enable nix_store_realise test, and add docs --- src/libstore-c/nix_api_store.h | 4 +++- tests/unit/libexpr/nix_api_expr.cc | 11 ++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 1309f99b7..577b70151 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -111,7 +111,9 @@ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * /** * @brief Realise a Nix store path * - * Blocking, calls callback once for each realised output + * Blocking, calls callback once for each realised output. + * + * @note When working with expressions, consider using e.g. nix_string_realise to get the output. `.drvPath` may not be accurate or available in the future. See https://github.com/NixOS/nix/issues/6507 * * @param[out] context Optional, stores error information * @param[in] store Nix Store reference diff --git a/tests/unit/libexpr/nix_api_expr.cc b/tests/unit/libexpr/nix_api_expr.cc index 3808bf0eb..fad078d0c 100644 --- a/tests/unit/libexpr/nix_api_expr.cc +++ b/tests/unit/libexpr/nix_api_expr.cc @@ -74,6 +74,9 @@ TEST_F(nix_api_expr_test, nix_build_drv) std::string pEnd = "-myname.drv"; ASSERT_EQ(pEnd, p.substr(p.size() - pEnd.size())); + // NOTE: .drvPath should be usually be ignored. Output paths are more versatile. + // See https://github.com/NixOS/nix/issues/6507 + // Use e.g. nix_string_realise to realise the output. StorePath * drvStorePath = nix_store_parse_path(ctx, store, drvPath); ASSERT_EQ(true, nix_store_is_valid_path(ctx, store, drvStorePath)); @@ -88,11 +91,9 @@ TEST_F(nix_api_expr_test, nix_build_drv) StorePath * outStorePath = nix_store_parse_path(ctx, store, outPath); ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, outStorePath)); - // TODO figure out why fails. - // `make libexpr-tests_RUN` works, but `nix build .` enters an infinite loop - /* nix_store_realise(ctx, store, drvStorePath, nullptr, nullptr); */ - /* auto is_valid_path = nix_store_is_valid_path(ctx, store, outStorePath); */ - /* ASSERT_EQ(true, is_valid_path); */ + nix_store_realise(ctx, store, drvStorePath, nullptr, nullptr); + auto is_valid_path = nix_store_is_valid_path(ctx, store, outStorePath); + ASSERT_EQ(true, is_valid_path); // Clean up nix_store_path_free(drvStorePath); From 1233bcde37e25a41f2230a64aded9bd5813098cd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 8 Apr 2024 10:34:58 +0200 Subject: [PATCH 166/211] libstore-c: Add nix_store_path_clone --- src/libstore-c/nix_api_store.cc | 5 +++++ src/libstore-c/nix_api_store.h | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index d80ba332e..d044ba14f 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -132,3 +132,8 @@ void nix_store_path_free(StorePath * sp) { delete sp; } + +StorePath * nix_store_path_clone(const StorePath * p) +{ + return new StorePath{p->path}; +} diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 577b70151..ab86a81df 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -90,6 +90,14 @@ nix_err nix_store_get_uri(nix_c_context * context, Store * store, void * callbac */ StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path); +/** + * @brief Copy a StorePath + * + * @param[in] p the path to copy + * @return a new StorePath + */ +StorePath * nix_store_path_clone(const StorePath * p); + /** @brief Deallocate a StorePath * * Does not fail. From 876e70bc9afb15b5ee71d2f6b3090acca315f320 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 8 Apr 2024 13:07:36 +0200 Subject: [PATCH 167/211] tests/unit/libexpr/local.mk A proper build system would catch errors like this. --- tests/unit/libexpr/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk index 8df1a5207..c59191db4 100644 --- a/tests/unit/libexpr/local.mk +++ b/tests/unit/libexpr/local.mk @@ -34,7 +34,7 @@ libexpr-tests_EXTRA_INCLUDES = \ libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) libexpr-tests_LIBS = \ - libexpr-test-support libstore-test-support libutils-test-support \ + libexpr-test-support libstore-test-support libutil-test-support \ libexpr libexprc libfetchers libstore libstorec libutil libutilc libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock From a512f4eebcbfed5db8e8c48fd93d99201267df04 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 8 Apr 2024 13:13:02 +0200 Subject: [PATCH 168/211] test/libutil: Add OBSERVE_STRING macro Makes string callback easier to pass, without mistakes. --- tests/unit/libstore/nix_api_store.cc | 12 ++++------- .../libutil-support/tests/string_callback.cc | 9 +++++++++ .../libutil-support/tests/string_callback.hh | 12 +++++++++++ tests/unit/libutil/nix_api_util.cc | 20 ++++++++----------- 4 files changed, 33 insertions(+), 20 deletions(-) create mode 100644 tests/unit/libutil-support/tests/string_callback.cc create mode 100644 tests/unit/libutil-support/tests/string_callback.hh diff --git a/tests/unit/libstore/nix_api_store.cc b/tests/unit/libstore/nix_api_store.cc index a31d66a4c..7c6ec0780 100644 --- a/tests/unit/libstore/nix_api_store.cc +++ b/tests/unit/libstore/nix_api_store.cc @@ -4,14 +4,10 @@ #include "nix_api_store_internal.h" #include "tests/nix_api_store.hh" +#include "tests/string_callback.hh" namespace nixC { -void observe_string_cb(const char * start, unsigned int n, std::string * user_data) -{ - *user_data = std::string(start); -} - std::string PATH_SUFFIX = "/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-name"; TEST_F(nix_api_util_context, nix_libstore_init) @@ -23,7 +19,7 @@ TEST_F(nix_api_util_context, nix_libstore_init) TEST_F(nix_api_store_test, nix_store_get_uri) { std::string str; - auto ret = nix_store_get_uri(ctx, store, (void *) observe_string_cb, &str); + auto ret = nix_store_get_uri(ctx, store, OBSERVE_STRING(str)); ASSERT_EQ(NIX_OK, ret); ASSERT_STREQ("local", str.c_str()); } @@ -56,7 +52,7 @@ TEST_F(nix_api_store_test, DoesNotCrashWhenContextIsNull) TEST_F(nix_api_store_test, get_version) { std::string str; - auto ret = nix_store_get_version(ctx, store, (void *) observe_string_cb, &str); + auto ret = nix_store_get_version(ctx, store, OBSERVE_STRING(str)); ASSERT_EQ(NIX_OK, ret); ASSERT_STREQ(PACKAGE_VERSION, str.c_str()); } @@ -69,7 +65,7 @@ TEST_F(nix_api_util_context, nix_store_open_dummy) ASSERT_STREQ("dummy", store->ptr->getUri().c_str()); std::string str; - nix_store_get_version(ctx, store, (void *) observe_string_cb, &str); + nix_store_get_version(ctx, store, OBSERVE_STRING(str)); ASSERT_STREQ("", str.c_str()); nix_store_free(store); diff --git a/tests/unit/libutil-support/tests/string_callback.cc b/tests/unit/libutil-support/tests/string_callback.cc new file mode 100644 index 000000000..28ac8b10c --- /dev/null +++ b/tests/unit/libutil-support/tests/string_callback.cc @@ -0,0 +1,9 @@ +#include "string_callback.hh" + +namespace nix::testing { + +void observe_string_cb(const char * start, unsigned int n, std::string * user_data) { + *user_data = std::string(start); +} + +} diff --git a/tests/unit/libutil-support/tests/string_callback.hh b/tests/unit/libutil-support/tests/string_callback.hh new file mode 100644 index 000000000..808fb707b --- /dev/null +++ b/tests/unit/libutil-support/tests/string_callback.hh @@ -0,0 +1,12 @@ +#pragma once +#include + +namespace nix::testing { + +void observe_string_cb(const char * start, unsigned int n, std::string * user_data); +inline void * observe_string_cb_data(std::string & out) { + return (void *) &out; +}; +#define OBSERVE_STRING(str) (void *)nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) + +} diff --git a/tests/unit/libutil/nix_api_util.cc b/tests/unit/libutil/nix_api_util.cc index 09f3f3e05..d2999f55b 100644 --- a/tests/unit/libutil/nix_api_util.cc +++ b/tests/unit/libutil/nix_api_util.cc @@ -3,16 +3,12 @@ #include "nix_api_util.h" #include "nix_api_util_internal.h" #include "tests/nix_api_util.hh" +#include "tests/string_callback.hh" #include namespace nixC { -void observe_string_cb(const char * start, unsigned int n, std::string * user_data) -{ - *user_data = std::string(start); -} - TEST_F(nix_api_util_context, nix_context_error) { std::string err_msg_ref; @@ -62,10 +58,10 @@ TEST_F(nix_api_util_context, nix_setting_get) { ASSERT_EQ(ctx->last_err_code, NIX_OK); std::string setting_value; - nix_err result = nix_setting_get(ctx, "invalid-key", (void *) observe_string_cb, &setting_value); + nix_err result = nix_setting_get(ctx, "invalid-key", OBSERVE_STRING(setting_value)); ASSERT_EQ(result, NIX_ERR_KEY); - result = nix_setting_get(ctx, "setting-name", (void *) observe_string_cb, &setting_value); + result = nix_setting_get(ctx, "setting-name", OBSERVE_STRING(setting_value)); ASSERT_EQ(result, NIX_OK); ASSERT_STREQ("empty", setting_value.c_str()); } @@ -79,7 +75,7 @@ TEST_F(nix_api_util_context, nix_setting_set) ASSERT_EQ(result, NIX_OK); std::string setting_value; - result = nix_setting_get(ctx, "setting-name", (void *) observe_string_cb, &setting_value); + result = nix_setting_get(ctx, "setting-name", OBSERVE_STRING(setting_value)); ASSERT_EQ(result, NIX_OK); ASSERT_STREQ("new-value", setting_value.c_str()); } @@ -107,14 +103,14 @@ TEST_F(nix_api_util_context, nix_err_info_msg) std::string err_info; // no error - EXPECT_THROW(nix_err_info_msg(NULL, ctx, (void *) observe_string_cb, &err_info), nix::Error); + EXPECT_THROW(nix_err_info_msg(NULL, ctx, OBSERVE_STRING(err_info)), nix::Error); try { throw nix::Error("testing error"); } catch (...) { nix_context_error(ctx); } - nix_err_info_msg(nix_c_context_create(), ctx, (void *) observe_string_cb, &err_info); + nix_err_info_msg(nix_c_context_create(), ctx, OBSERVE_STRING(err_info)); ASSERT_STREQ("testing error", err_info.c_str()); } @@ -123,7 +119,7 @@ TEST_F(nix_api_util_context, nix_err_name) std::string err_name; // no error - EXPECT_THROW(nix_err_name(NULL, ctx, (void *) observe_string_cb, &err_name), nix::Error); + EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error); std::string err_msg_ref; try { @@ -131,7 +127,7 @@ TEST_F(nix_api_util_context, nix_err_name) } catch (...) { nix_context_error(ctx); } - nix_err_name(nix_c_context_create(), ctx, (void *) observe_string_cb, &err_name); + nix_err_name(nix_c_context_create(), ctx, OBSERVE_STRING(err_name)); ASSERT_EQ(std::string(err_name), "nix::Error"); } From f2522d4ecdcd693a2f94cd118eb7fa509983a54e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 8 Apr 2024 13:14:03 +0200 Subject: [PATCH 169/211] libexpr-c: Add nix_store_path_name --- src/libstore-c/nix_api_store.cc | 7 +++++++ src/libstore-c/nix_api_store.h | 9 +++++++++ tests/unit/libexpr/nix_api_expr.cc | 8 ++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index d044ba14f..511ba0fad 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -128,6 +128,13 @@ nix_err nix_store_realise( NIXC_CATCH_ERRS } +void nix_store_path_name(const StorePath *store_path, void * callback, void * user_data) +{ + std::string_view name = store_path->path.name(); + ((nix_get_string_callback) callback)(name.data(), name.size(), user_data); +} + + void nix_store_path_free(StorePath * sp) { delete sp; diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index ab86a81df..ca8996681 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -90,6 +90,15 @@ nix_err nix_store_get_uri(nix_c_context * context, Store * store, void * callbac */ StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path); +/** + * @brief Get the path name (e.g. "name" in /nix/store/...-name) + * + * @param[in] store_path the path to get the name from + * @param[in] callback called with the name + * @param[in] user_data arbitrary data, passed to the callback when it's called. + */ +void nix_store_path_name(const StorePath *store_path, void * callback, void * user_data); + /** * @brief Copy a StorePath * diff --git a/tests/unit/libexpr/nix_api_expr.cc b/tests/unit/libexpr/nix_api_expr.cc index fad078d0c..f5c66536d 100644 --- a/tests/unit/libexpr/nix_api_expr.cc +++ b/tests/unit/libexpr/nix_api_expr.cc @@ -6,6 +6,7 @@ #include "nix_api_value.h" #include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" #include "gmock/gmock.h" #include @@ -168,11 +169,14 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context) EXPECT_THAT(s, testing::HasSubstr("a derivation path by itself:")); EXPECT_THAT(s, testing::EndsWith("-not-actually-built-yet.drv\n")); - std::vector names; + std::vector names; size_t n = nix_realised_string_get_store_path_count(r); for (size_t i = 0; i < n; ++i) { const StorePath * p = nix_realised_string_get_store_path(r, i); - names.push_back(p->path.name()); + ASSERT_NE(nullptr, p); + std::string name; + nix_store_path_name(p, OBSERVE_STRING(name)); + names.push_back(name); } std::sort(names.begin(), names.end()); ASSERT_EQ(3, names.size()); From acbb1523c1dc28043d6dab729db696485938f969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 12 Apr 2024 15:57:53 +0200 Subject: [PATCH 170/211] Fix the access of symlinks to host files in the sandbox https://github.com/NixOS/nix/pull/10456 fixed the addition of symlink store paths to the sandbox, but also made it so that the hardcoded sandbox paths (like `/etc/hosts`) were now bind-mounted without following the possible symlinks. This made these files unreadable if there were symlinks (because the sandbox would now contain a symlink to an unreachable file rather than the underlying file). In particular, this broke FOD derivations on NixOS as `/etc/hosts` is a symlink there. Fix that by canonicalizing all these hardcoded sandbox paths before adding them to the sandbox. --- src/libstore/build/local-derivation-goal.cc | 13 +++-- tests/functional/linux-sandbox.sh | 14 +++++- tests/functional/symlink-derivation.nix | 55 +++++++++++++++------ 3 files changed, 62 insertions(+), 20 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index db12af810..c2af0b270 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1823,11 +1823,18 @@ void LocalDerivationGoal::runChild() if (pathExists(path)) ss.push_back(path); - if (settings.caFile != "") - pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); + if (settings.caFile != "" && pathExists(settings.caFile)) { + Path caFile = settings.caFile; + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); + } } - for (auto & i : ss) pathsInChroot.emplace(i, i); + for (auto & i : ss) { + // For backwards-compatibiliy, resolve all the symlinks in the + // chroot paths + auto canonicalPath = canonPath(i, true); + pathsInChroot.emplace(i, canonicalPath); + } /* Bind-mount all the directories from the "host" filesystem that we want in the chroot diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index 04209277b..880d56fca 100644 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -60,7 +60,11 @@ testCert () { nocert=$TEST_ROOT/no-cert-file.pem cert=$TEST_ROOT/some-cert-file.pem +symlinkcert=$TEST_ROOT/symlink-cert-file.pem +symlinkDir=$TEST_ROOT/symlink-dir echo -n "CERT_CONTENT" > $cert +ln -s $cert $symlinkcert +ln -s $TEST_ROOT $symlinkDir # No cert in sandbox when not a fixed-output derivation testCert missing normal "$cert" @@ -74,5 +78,13 @@ testCert missing fixed-output "$nocert" # Cert in sandbox when ssl-cert-file is set to an existing file testCert present fixed-output "$cert" +# Cert in sandbox when ssl-cert-file is set to a symlink to an existing file +testCert present fixed-output "$symlinkcert" + # Symlinks should be added in the sandbox directly and not followed -nix-sandbox-build symlink-derivation.nix +nix-sandbox-build symlink-derivation.nix -A depends_on_symlink +nix-sandbox-build symlink-derivation.nix -A test_sandbox_paths \ + --option extra-sandbox-paths "/file=$cert" \ + --option extra-sandbox-paths "/dir=$TEST_ROOT" \ + --option extra-sandbox-paths "/symlinkDir=$symlinkDir" \ + --option extra-sandbox-paths "/symlink=$symlinkcert" diff --git a/tests/functional/symlink-derivation.nix b/tests/functional/symlink-derivation.nix index 17ba37424..e9a74cdce 100644 --- a/tests/functional/symlink-derivation.nix +++ b/tests/functional/symlink-derivation.nix @@ -15,22 +15,45 @@ let ''; }; in -mkDerivation { - name = "depends-on-symlink"; - buildCommand = '' - ( - set -x +{ + depends_on_symlink = mkDerivation { + name = "depends-on-symlink"; + buildCommand = '' + ( + set -x - # `foo_symlink` should be a symlink pointing to `foo_in_store` - [[ -L ${foo_symlink} ]] - [[ $(readlink ${foo_symlink}) == ${foo_in_store} ]] + # `foo_symlink` should be a symlink pointing to `foo_in_store` + [[ -L ${foo_symlink} ]] + [[ $(readlink ${foo_symlink}) == ${foo_in_store} ]] - # `symlink_to_not_in_store` should be a symlink pointing to `./.`, which - # is not available in the sandbox - [[ -L ${symlink_to_not_in_store} ]] - [[ $(readlink ${symlink_to_not_in_store}) == ${builtins.toString ./.} ]] - (! ls ${symlink_to_not_in_store}/) - ) - echo "Success!" > $out - ''; + # `symlink_to_not_in_store` should be a symlink pointing to `./.`, which + # is not available in the sandbox + [[ -L ${symlink_to_not_in_store} ]] + [[ $(readlink ${symlink_to_not_in_store}) == ${builtins.toString ./.} ]] + (! ls ${symlink_to_not_in_store}/) + + # Native paths + ) + echo "Success!" > $out + ''; + }; + + test_sandbox_paths = mkDerivation { + # Depends on the caller to set a bunch of `--sandbox-path` arguments + name = "test-sandbox-paths"; + buildCommand = '' + ( + set -x + [[ -f /file ]] + [[ -d /dir ]] + + # /symlink and /symlinkDir should be available as raw symlinks + # (pointing to files outside of the sandbox) + [[ -L /symlink ]] && [[ ! -e $(readlink /symlink) ]] + [[ -L /symlinkDir ]] && [[ ! -e $(readlink /symlinkDir) ]] + ) + + touch $out + ''; + }; } From cef677ddbcad420220474935b660c147718a3a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 12 Apr 2024 16:10:22 +0200 Subject: [PATCH 171/211] Test the inclusion of transitive symlinks in the sandbox --- tests/functional/linux-sandbox.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index 880d56fca..e553791d9 100644 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -61,9 +61,11 @@ testCert () { nocert=$TEST_ROOT/no-cert-file.pem cert=$TEST_ROOT/some-cert-file.pem symlinkcert=$TEST_ROOT/symlink-cert-file.pem +transitivesymlinkcert=$TEST_ROOT/transitive-symlink-cert-file.pem symlinkDir=$TEST_ROOT/symlink-dir echo -n "CERT_CONTENT" > $cert ln -s $cert $symlinkcert +ln -s $symlinkcert $transitivesymlinkcert ln -s $TEST_ROOT $symlinkDir # No cert in sandbox when not a fixed-output derivation @@ -78,8 +80,9 @@ testCert missing fixed-output "$nocert" # Cert in sandbox when ssl-cert-file is set to an existing file testCert present fixed-output "$cert" -# Cert in sandbox when ssl-cert-file is set to a symlink to an existing file +# Cert in sandbox when ssl-cert-file is set to a (potentially transitive) symlink to an existing file testCert present fixed-output "$symlinkcert" +testCert present fixed-output "$transitivesymlinkcert" # Symlinks should be added in the sandbox directly and not followed nix-sandbox-build symlink-derivation.nix -A depends_on_symlink From 13c2005e7df7effe29449556edbc41314d0af915 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 12 Apr 2024 17:43:35 +0200 Subject: [PATCH 172/211] add intermediate variables and clarifying comments (#9274) * add intermediate variables and clarifying comments Co-authored-by: Alexander Groleau Co-authored-by: Robert Hensing --- src/libstore/build/local-derivation-goal.cc | 13 +++++++++++-- src/libstore/store-api.cc | 10 +++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index db12af810..297435f90 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2960,16 +2960,25 @@ bool LocalDerivationGoal::isReadDesc(int fd) StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/src/protocols/store-path.md for details + // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName); return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), + pathType, + // pass an all-zeroes hash Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); } StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/src/protocols/store-path.md for details + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), + pathType, + // pass an all-zeroes hash Hash(HashAlgorithm::SHA256), path.name()); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 77005870a..79beeebbd 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -131,12 +131,12 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed throw Error("fixed output derivation '%s' is not allowed to refer to other store paths.\nYou may need to use the 'unsafeDiscardReferences' derivation attribute, see the manual for more details.", name); } - return makeStorePath("output:out", - hashString(HashAlgorithm::SHA256, - "fixed:out:" + // make a unique digest based on the parameters for creating this store object + auto payload = "fixed:out:" + makeFileIngestionPrefix(info.method) - + info.hash.to_string(HashFormat::Base16, true) + ":"), - name); + + info.hash.to_string(HashFormat::Base16, true) + ":"; + auto digest = hashString(HashAlgorithm::SHA256, payload); + return makeStorePath("output:out", digest, name); } } From 95ae12b6079db7a30e2686138209e3a363eb85c9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Apr 2024 11:06:47 -0400 Subject: [PATCH 173/211] docs: Refer to the glossary with `@docroot@` instead of `..` These unweildy relative paths probably predate the `@docroot@` mechanism. --- doc/manual/src/architecture/architecture.md | 2 +- doc/manual/src/command-ref/nix-build.md | 2 +- doc/manual/src/command-ref/nix-copy-closure.md | 2 +- doc/manual/src/command-ref/nix-instantiate.md | 2 +- doc/manual/src/command-ref/nix-store/query.md | 8 ++++---- doc/manual/src/language/advanced-attributes.md | 2 +- doc/manual/src/language/operators.md | 4 ++-- doc/manual/src/language/string-interpolation.md | 6 +++--- doc/manual/src/language/values.md | 2 +- doc/manual/src/release-notes/rl-2.15.md | 2 +- src/libcmd/command.cc | 2 +- src/nix/nix.md | 2 +- src/nix/path-info.md | 2 +- src/nix/store-copy-log.md | 2 +- 14 files changed, 20 insertions(+), 20 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 2fec4ed20..867a9c992 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -69,7 +69,7 @@ It can also execute build plans to produce new data, which are made available to A build plan itself is a series of *build tasks*, together with their build inputs. > **Important** -> A build task in Nix is called [derivation](../glossary.md#gloss-derivation). +> A build task in Nix is called [derivation](@docroot@/glossary.md#gloss-derivation). Each build task has a special build input executed as *build instructions* in order to perform the build. The result of a build task can be input to another build task. diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index b548edf82..e4223b542 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -41,7 +41,7 @@ expression to a low-level [store derivation]) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store derivation). -[store derivation]: ../glossary.md#gloss-store-derivation +[store derivation]: @docroot@/glossary.md#gloss-store-derivation > **Warning** > diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index fbf6828da..eb1693e1e 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`. - `--include-outputs`\ Also copy the outputs of [store derivation]s included in the closure. - [store derivation]: ../glossary.md#gloss-store-derivation + [store derivation]: @docroot@/glossary.md#gloss-store-derivation - `--use-substitutes` / `-s`\ Attempt to download missing paths on the target machine using Nix’s diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index 479c9abcf..dffbb2d70 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -23,7 +23,7 @@ It evaluates the Nix expressions in each of *files* (which defaults to derivation, a list of derivations, or a set of derivations. The paths of the resulting store derivations are printed on standard output. -[store derivation]: ../glossary.md#gloss-store-derivation +[store derivation]: @docroot@/glossary.md#gloss-store-derivation If *files* is the character `-`, then a Nix expression will be read from standard input. diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..0bcacfe0c 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -40,12 +40,12 @@ symlink. derivations *paths*. These are the paths that will be produced when the derivation is built. - [output paths]: ../../glossary.md#gloss-output-path + [output paths]: @docroot@/glossary.md#gloss-output-path - `--requisites`; `-R`\ Prints out the [closure] of the store path *paths*. - [closure]: ../../glossary.md#gloss-closure + [closure]: @docroot@/glossary.md#gloss-closure This query has one option: @@ -66,7 +66,7 @@ symlink. *paths*, that is, their immediate dependencies. (For *all* dependencies, use `--requisites`.) - [references]: ../../glossary.md#gloss-reference + [references]: @docroot@/glossary.md#gloss-reference - `--referrers`\ Prints the set of *referrers* of the store paths *paths*, that is, @@ -90,7 +90,7 @@ symlink. example when *paths* were substituted from a binary cache. Use `--valid-derivers` instead to obtain valid paths only. - [deriver]: ../../glossary.md#gloss-deriver + [deriver]: @docroot@/glossary.md#gloss-deriver - `--valid-derivers`\ Prints a set of derivation files (`.drv`) which are supposed produce diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 16dcc6ba9..1fcc5a95b 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -303,7 +303,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/glossary.md#gloss-store-object). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..698fed47e 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -128,8 +128,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/glossary.md#gloss-store-path +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1f8fecca8 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -107,9 +107,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/glossary.md#gloss-store-object). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/glossary.md#gloss-store-path > **Example** > diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 74ffc7070..568542c0b 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -113,7 +113,7 @@ For example, assume you used a file path in an interpolated string during a `nix repl` session. Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - [store path]: ../glossary.md#gloss-store-path + [store path]: @docroot@/glossary.md#gloss-store-path Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..220a90cf6 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -148,7 +148,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/nix/nix.md b/src/nix/nix.md index 749456014..4464bef37 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -229,7 +229,7 @@ operate are determined as follows: Note that a [store derivation] (given by its `.drv` file store path) doesn't have any attributes like `meta`, and thus this case doesn't apply to it. - [store derivation]: ../../glossary.md#gloss-store-derivation + [store derivation]: @docroot@/glossary.md#gloss-store-derivation * Otherwise, Nix will use all outputs of the derivation. diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 4594854eb..789984559 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -70,7 +70,7 @@ R""( * Print the path of the [store derivation] produced by `nixpkgs#hello`: - [store derivation]: ../../glossary.md#gloss-store-derivation + [store derivation]: @docroot@/glossary.md#gloss-store-derivation ```console # nix path-info --derivation nixpkgs#hello diff --git a/src/nix/store-copy-log.md b/src/nix/store-copy-log.md index 0937250f2..61daa75c1 100644 --- a/src/nix/store-copy-log.md +++ b/src/nix/store-copy-log.md @@ -20,7 +20,7 @@ R""( * To copy the log for a specific [store derivation] via SSH: - [store derivation]: ../../glossary.md#gloss-store-derivation + [store derivation]: @docroot@/glossary.md#gloss-store-derivation ```console # nix store copy-log --to ssh-ng://machine /nix/store/ilgm50plpmcgjhcp33z6n4qbnpqfhxym-glibc-2.33-59.drv From 01bad63c720cb3d7280484f87f3ff9734b2b7117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= Date: Fri, 12 Apr 2024 21:41:15 +0200 Subject: [PATCH 174/211] C API: Safer function pointer casting See https://github.com/NixOS/nix/pull/8699#discussion_r1554312181 Casting a function pointer to `void*` is undefined behavior in the C spec, since there are platforms with different sizes for these two kinds of pointers. A safe alternative might be `void (*callback)()` --- src/libexpr-c/nix_api_value.cc | 15 ++++++------- src/libstore-c/nix_api_store.cc | 18 ++++++++++++---- src/libstore-c/nix_api_store.h | 20 ++++++++++++++---- src/libutil-c/nix_api_util.cc | 21 +++++++++++++++---- src/libutil-c/nix_api_util.h | 19 +++++++++++++---- src/libutil-c/nix_api_util_internal.h | 3 ++- .../libutil-support/tests/string_callback.cc | 3 ++- .../libutil-support/tests/string_callback.hh | 9 ++++++-- 8 files changed, 79 insertions(+), 29 deletions(-) diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index fd1bfc165..87e25f9d9 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -15,9 +15,9 @@ #include "value/context.hh" #ifdef HAVE_BOEHMGC -# include "gc/gc.h" -# define GC_INCLUDE_NEW 1 -# include "gc_cpp.h" +#include "gc/gc.h" +#define GC_INCLUDE_NEW 1 +#include "gc_cpp.h" #endif // Helper function to throw an exception if value is null @@ -537,7 +537,7 @@ nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * st if (context) context->last_err_code = NIX_OK; try { - auto &v = check_value_not_null(value); + auto & v = check_value_not_null(value); nix::NixStringContext stringContext; auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned(); nix::StorePathSet storePaths; @@ -547,14 +547,11 @@ nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * st // Convert to the C API StorePath type and convert to vector for index-based access std::vector vec; - for (auto &sp : storePaths) { + for (auto & sp : storePaths) { vec.push_back(StorePath{sp}); } - return new nix_realised_string { - .str = s, - .storePaths = vec - }; + return new nix_realised_string{.str = s, .storePaths = vec}; } NIXC_CATCH_ERRS_NULL } diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 511ba0fad..aa4ab521a 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -56,7 +56,11 @@ void nix_store_free(Store * store) delete store; } -nix_err nix_store_get_uri(nix_c_context * context, Store * store, void * callback, void * user_data) +nix_err nix_store_get_uri( + nix_c_context * context, + Store * store, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -67,7 +71,11 @@ nix_err nix_store_get_uri(nix_c_context * context, Store * store, void * callbac NIXC_CATCH_ERRS } -nix_err nix_store_get_version(nix_c_context * context, Store * store, void * callback, void * user_data) +nix_err nix_store_get_version( + nix_c_context * context, + Store * store, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -128,13 +136,15 @@ nix_err nix_store_realise( NIXC_CATCH_ERRS } -void nix_store_path_name(const StorePath *store_path, void * callback, void * user_data) +void nix_store_path_name( + const StorePath * store_path, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data) { std::string_view name = store_path->path.name(); ((nix_get_string_callback) callback)(name.data(), name.size(), user_data); } - void nix_store_path_free(StorePath * sp) { delete sp; diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index ca8996681..efeedbf7b 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -76,7 +76,11 @@ void nix_store_free(Store * store); * @see nix_get_string_callback * @return error code, NIX_OK on success. */ -nix_err nix_store_get_uri(nix_c_context * context, Store * store, void * callback, void * user_data); +nix_err nix_store_get_uri( + nix_c_context * context, + Store * store, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data); // returns: owned StorePath* /** @@ -97,7 +101,10 @@ StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const c * @param[in] callback called with the name * @param[in] user_data arbitrary data, passed to the callback when it's called. */ -void nix_store_path_name(const StorePath *store_path, void * callback, void * user_data); +void nix_store_path_name( + const StorePath * store_path, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data); /** * @brief Copy a StorePath @@ -130,7 +137,8 @@ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * * * Blocking, calls callback once for each realised output. * - * @note When working with expressions, consider using e.g. nix_string_realise to get the output. `.drvPath` may not be accurate or available in the future. See https://github.com/NixOS/nix/issues/6507 + * @note When working with expressions, consider using e.g. nix_string_realise to get the output. `.drvPath` may not be + * accurate or available in the future. See https://github.com/NixOS/nix/issues/6507 * * @param[out] context Optional, stores error information * @param[in] store Nix Store reference @@ -155,7 +163,11 @@ nix_err nix_store_realise( * @see nix_get_string_callback * @return error code, NIX_OK on success. */ -nix_err nix_store_get_version(nix_c_context * context, Store * store, void * callback, void * user_data); +nix_err nix_store_get_version( + nix_c_context * context, + Store * store, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data); // cffi end #ifdef __cplusplus diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 8d0f7ac38..4999e28e9 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -64,7 +64,11 @@ const char * nix_version_get() // Implementations -nix_err nix_setting_get(nix_c_context * context, const char * key, void * callback, void * user_data) +nix_err nix_setting_get( + nix_c_context * context, + const char * key, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -115,7 +119,11 @@ const char * nix_err_msg(nix_c_context * context, const nix_c_context * read_con return nullptr; } -nix_err nix_err_name(nix_c_context * context, const nix_c_context * read_context, void * callback, void * user_data) +nix_err nix_err_name( + nix_c_context * context, + const nix_c_context * read_context, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -125,7 +133,11 @@ nix_err nix_err_name(nix_c_context * context, const nix_c_context * read_context return call_nix_get_string_callback(read_context->name, callback, user_data); } -nix_err nix_err_info_msg(nix_c_context * context, const nix_c_context * read_context, void * callback, void * user_data) +nix_err nix_err_info_msg( + nix_c_context * context, + const nix_c_context * read_context, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -141,7 +153,8 @@ nix_err nix_err_code(const nix_c_context * read_context) } // internal -nix_err call_nix_get_string_callback(const std::string str, void * callback, void * user_data) +nix_err call_nix_get_string_callback( + const std::string str, void (*callback)(const char * start, unsigned int n, void * user_data), void * user_data) { ((nix_get_string_callback) callback)(str.c_str(), str.size(), user_data); return NIX_OK; diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index cb506ca90..36a3f76cb 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -175,7 +175,11 @@ nix_err nix_libutil_init(nix_c_context * context); * @return NIX_ERR_KEY if the setting is unknown, or NIX_OK if the setting was retrieved * successfully. */ -nix_err nix_setting_get(nix_c_context * context, const char * key, void * callback, void * user_data); +nix_err nix_setting_get( + nix_c_context * context, + const char * key, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data); /** * @brief Sets a setting in the nix global configuration. @@ -241,8 +245,11 @@ const char * nix_err_msg(nix_c_context * context, const nix_c_context * ctx, uns * @see nix_get_string_callback * @return NIX_OK if there were no errors, an error code otherwise. */ -nix_err -nix_err_info_msg(nix_c_context * context, const nix_c_context * read_context, void * callback, void * user_data); +nix_err nix_err_info_msg( + nix_c_context * context, + const nix_c_context * read_context, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data); /** * @brief Retrieves the error name from a context. @@ -260,7 +267,11 @@ nix_err_info_msg(nix_c_context * context, const nix_c_context * read_context, vo * @see nix_get_string_callback * @return NIX_OK if there were no errors, an error code otherwise. */ -nix_err nix_err_name(nix_c_context * context, const nix_c_context * read_context, void * callback, void * user_data); +nix_err nix_err_name( + nix_c_context * context, + const nix_c_context * read_context, + void (*callback)(const char * start, unsigned int n, void * user_data), + void * user_data); /** * @brief Retrieves the most recent error code from a nix_c_context diff --git a/src/libutil-c/nix_api_util_internal.h b/src/libutil-c/nix_api_util_internal.h index 6e8eac020..fe5d5db18 100644 --- a/src/libutil-c/nix_api_util_internal.h +++ b/src/libutil-c/nix_api_util_internal.h @@ -29,7 +29,8 @@ nix_err nix_context_error(nix_c_context * context); * @return NIX_OK if there were no errors. * @see nix_get_string_callback */ -nix_err call_nix_get_string_callback(const std::string str, void * callback, void * user_data); +nix_err call_nix_get_string_callback( + const std::string str, void (*callback)(const char * start, unsigned int n, void * user_data), void * user_data); #define NIXC_CATCH_ERRS \ catch (...) \ diff --git a/tests/unit/libutil-support/tests/string_callback.cc b/tests/unit/libutil-support/tests/string_callback.cc index 28ac8b10c..2d0e0dad0 100644 --- a/tests/unit/libutil-support/tests/string_callback.cc +++ b/tests/unit/libutil-support/tests/string_callback.cc @@ -2,7 +2,8 @@ namespace nix::testing { -void observe_string_cb(const char * start, unsigned int n, std::string * user_data) { +void observe_string_cb(const char * start, unsigned int n, std::string * user_data) +{ *user_data = std::string(start); } diff --git a/tests/unit/libutil-support/tests/string_callback.hh b/tests/unit/libutil-support/tests/string_callback.hh index 808fb707b..6420810b6 100644 --- a/tests/unit/libutil-support/tests/string_callback.hh +++ b/tests/unit/libutil-support/tests/string_callback.hh @@ -4,9 +4,14 @@ namespace nix::testing { void observe_string_cb(const char * start, unsigned int n, std::string * user_data); -inline void * observe_string_cb_data(std::string & out) { + +inline void * observe_string_cb_data(std::string & out) +{ return (void *) &out; }; -#define OBSERVE_STRING(str) (void *)nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) + +#define OBSERVE_STRING(str) \ + (void (*)(const char *, unsigned int, void *)) nix::testing::observe_string_cb, \ + nix::testing::observe_string_cb_data(str) } From 76444a395825cca5e00b3eecef78109740b24114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= Date: Sun, 14 Apr 2024 16:18:32 +0200 Subject: [PATCH 175/211] C API: proper `ifdef` `endif` indentation --- src/libexpr-c/nix_api_value.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 87e25f9d9..817464fa8 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -15,9 +15,9 @@ #include "value/context.hh" #ifdef HAVE_BOEHMGC -#include "gc/gc.h" -#define GC_INCLUDE_NEW 1 -#include "gc_cpp.h" +# include "gc/gc.h" +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" #endif // Helper function to throw an exception if value is null From bb939d37727f2a046c96dc6ca4a8b2ea8b85531f Mon Sep 17 00:00:00 2001 From: HaeNoe Date: Sun, 14 Apr 2024 22:35:51 +0200 Subject: [PATCH 176/211] change implementation of `optionalValueAt` --- src/libutil/json-utils.cc | 10 ++++------ src/libutil/json-utils.hh | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index 7a7264a9a..1b911bf75 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -30,14 +30,12 @@ const nlohmann::json & valueAt( return map.at(key); } -std::optional optionalValueAt(const nlohmann::json & value, const std::string & key) +std::optional optionalValueAt(const nlohmann::json::object_t & map, const std::string & key) { - try { - auto & v = valueAt(value, key); - return v.get(); - } catch (...) { + if (!map.contains(key)) return std::nullopt; - } + + return std::optional { map.at(key) }; } diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 2024624f4..08c98cc8c 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -23,7 +23,7 @@ const nlohmann::json & valueAt( const nlohmann::json::object_t & map, const std::string & key); -std::optional optionalValueAt(const nlohmann::json & value, const std::string & key); +std::optional optionalValueAt(const nlohmann::json::object_t & value, const std::string & key); /** * Downcast the json object, failing with a nice error if the conversion fails. From ff4c286e8006e3d13da67076e973badf78edf22b Mon Sep 17 00:00:00 2001 From: HaeNoe Date: Sun, 14 Apr 2024 22:36:03 +0200 Subject: [PATCH 177/211] add tests for `optionalValueAt` --- tests/unit/libutil/json-utils.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/libutil/json-utils.cc b/tests/unit/libutil/json-utils.cc index ffa667806..ec653fff5 100644 --- a/tests/unit/libutil/json-utils.cc +++ b/tests/unit/libutil/json-utils.cc @@ -160,4 +160,16 @@ TEST(getBoolean, wrongAssertions) { ASSERT_THROW(getBoolean(valueAt(json, "int")), Error); } +TEST(optionalValueAt, existing) { + auto json = R"({ "string": "ssh-rsa" })"_json; + + ASSERT_EQ(optionalValueAt(json, "string"), std::optional { "ssh-rsa" }); +} + +TEST(optionalValueAt, empty) { + auto json = R"({})"_json; + + ASSERT_EQ(optionalValueAt(json, "string2"), std::nullopt); +} + } /* namespace nix */ From e3fed2ebcf5d5b701f7c4a2d368006e20dfd4f8b Mon Sep 17 00:00:00 2001 From: HaeNoe Date: Sun, 14 Apr 2024 22:37:08 +0200 Subject: [PATCH 178/211] update `fetchers::PublicKey` json (de)serialization --- src/libfetchers/fetchers.cc | 18 ++++++++++++++++++ src/libfetchers/fetchers.hh | 4 +++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 483796f0b..a06d931db 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -3,6 +3,7 @@ #include "input-accessor.hh" #include "source-path.hh" #include "fetch-to-store.hh" +#include "json-utils.hh" #include @@ -412,3 +413,20 @@ std::string publicKeys_to_string(const std::vector& publicKeys) } } + +namespace nlohmann { + +using namespace nix; + +fetchers::PublicKey adl_serializer::from_json(const json & json) { + auto type = optionalValueAt(json, "type").value_or("ssh-ed25519"); + auto key = valueAt(json, "key"); + return fetchers::PublicKey { getString(type), getString(key) }; +} + +void adl_serializer::to_json(json & json, fetchers::PublicKey p) { + json["type"] = p.type; + json["key"] = p.key; +} + +} diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index cd11f9eae..bb21c68cc 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "hash.hh" #include "canon-path.hh" +#include "json-impls.hh" #include "attrs.hh" #include "url.hh" @@ -230,8 +231,9 @@ struct PublicKey std::string type = "ssh-ed25519"; std::string key; }; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key) std::string publicKeys_to_string(const std::vector&); } + +JSON_IMPL(fetchers::PublicKey) From 774e7213e85447e159b93437f5f269a2f14621ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= Date: Mon, 15 Apr 2024 12:05:57 +0200 Subject: [PATCH 179/211] C API: Use `nix_get_string_callback` typedef --- src/libstore-c/nix_api_store.cc | 20 +++++------------- src/libstore-c/nix_api_store.h | 18 ++++------------ src/libutil-c/nix_api_util.cc | 21 +++++-------------- src/libutil-c/nix_api_util.h | 16 +++----------- src/libutil-c/nix_api_util_internal.h | 3 +-- .../libutil-support/tests/string_callback.hh | 3 +-- 6 files changed, 19 insertions(+), 62 deletions(-) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index aa4ab521a..6ce4d01bb 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -56,11 +56,7 @@ void nix_store_free(Store * store) delete store; } -nix_err nix_store_get_uri( - nix_c_context * context, - Store * store, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data) +nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -71,11 +67,8 @@ nix_err nix_store_get_uri( NIXC_CATCH_ERRS } -nix_err nix_store_get_version( - nix_c_context * context, - Store * store, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data) +nix_err +nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -136,13 +129,10 @@ nix_err nix_store_realise( NIXC_CATCH_ERRS } -void nix_store_path_name( - const StorePath * store_path, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data) +void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data) { std::string_view name = store_path->path.name(); - ((nix_get_string_callback) callback)(name.data(), name.size(), user_data); + callback(name.data(), name.size(), user_data); } void nix_store_path_free(StorePath * sp) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index efeedbf7b..c83aca3f7 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -76,11 +76,7 @@ void nix_store_free(Store * store); * @see nix_get_string_callback * @return error code, NIX_OK on success. */ -nix_err nix_store_get_uri( - nix_c_context * context, - Store * store, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data); +nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); // returns: owned StorePath* /** @@ -101,10 +97,7 @@ StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const c * @param[in] callback called with the name * @param[in] user_data arbitrary data, passed to the callback when it's called. */ -void nix_store_path_name( - const StorePath * store_path, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data); +void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data); /** * @brief Copy a StorePath @@ -163,11 +156,8 @@ nix_err nix_store_realise( * @see nix_get_string_callback * @return error code, NIX_OK on success. */ -nix_err nix_store_get_version( - nix_c_context * context, - Store * store, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data); +nix_err +nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); // cffi end #ifdef __cplusplus diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 4999e28e9..0a9b49345 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -64,11 +64,7 @@ const char * nix_version_get() // Implementations -nix_err nix_setting_get( - nix_c_context * context, - const char * key, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data) +nix_err nix_setting_get(nix_c_context * context, const char * key, nix_get_string_callback callback, void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -120,10 +116,7 @@ const char * nix_err_msg(nix_c_context * context, const nix_c_context * read_con } nix_err nix_err_name( - nix_c_context * context, - const nix_c_context * read_context, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data) + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -134,10 +127,7 @@ nix_err nix_err_name( } nix_err nix_err_info_msg( - nix_c_context * context, - const nix_c_context * read_context, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data) + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -153,9 +143,8 @@ nix_err nix_err_code(const nix_c_context * read_context) } // internal -nix_err call_nix_get_string_callback( - const std::string str, void (*callback)(const char * start, unsigned int n, void * user_data), void * user_data) +nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data) { - ((nix_get_string_callback) callback)(str.c_str(), str.size(), user_data); + callback(str.c_str(), str.size(), user_data); return NIX_OK; } diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 36a3f76cb..e0ca04e69 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -175,11 +175,7 @@ nix_err nix_libutil_init(nix_c_context * context); * @return NIX_ERR_KEY if the setting is unknown, or NIX_OK if the setting was retrieved * successfully. */ -nix_err nix_setting_get( - nix_c_context * context, - const char * key, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data); +nix_err nix_setting_get(nix_c_context * context, const char * key, nix_get_string_callback callback, void * user_data); /** * @brief Sets a setting in the nix global configuration. @@ -246,10 +242,7 @@ const char * nix_err_msg(nix_c_context * context, const nix_c_context * ctx, uns * @return NIX_OK if there were no errors, an error code otherwise. */ nix_err nix_err_info_msg( - nix_c_context * context, - const nix_c_context * read_context, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data); + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data); /** * @brief Retrieves the error name from a context. @@ -268,10 +261,7 @@ nix_err nix_err_info_msg( * @return NIX_OK if there were no errors, an error code otherwise. */ nix_err nix_err_name( - nix_c_context * context, - const nix_c_context * read_context, - void (*callback)(const char * start, unsigned int n, void * user_data), - void * user_data); + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data); /** * @brief Retrieves the most recent error code from a nix_c_context diff --git a/src/libutil-c/nix_api_util_internal.h b/src/libutil-c/nix_api_util_internal.h index fe5d5db18..aa829feaf 100644 --- a/src/libutil-c/nix_api_util_internal.h +++ b/src/libutil-c/nix_api_util_internal.h @@ -29,8 +29,7 @@ nix_err nix_context_error(nix_c_context * context); * @return NIX_OK if there were no errors. * @see nix_get_string_callback */ -nix_err call_nix_get_string_callback( - const std::string str, void (*callback)(const char * start, unsigned int n, void * user_data), void * user_data); +nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data); #define NIXC_CATCH_ERRS \ catch (...) \ diff --git a/tests/unit/libutil-support/tests/string_callback.hh b/tests/unit/libutil-support/tests/string_callback.hh index 6420810b6..3a3e545e9 100644 --- a/tests/unit/libutil-support/tests/string_callback.hh +++ b/tests/unit/libutil-support/tests/string_callback.hh @@ -11,7 +11,6 @@ inline void * observe_string_cb_data(std::string & out) }; #define OBSERVE_STRING(str) \ - (void (*)(const char *, unsigned int, void *)) nix::testing::observe_string_cb, \ - nix::testing::observe_string_cb_data(str) + (nix_get_string_callback) nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) } From bcda38c27282124f37c2d9356539886291801f6e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 Apr 2024 10:17:58 -0400 Subject: [PATCH 180/211] Have `clang-format` indent conditional CPP This reflects the style I've been introducing in tandem with Windows support. See https://clang.llvm.org/docs/ClangFormatStyleOptions.html#indentppdirectives for this configuration option. --- .clang-format | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.clang-format b/.clang-format index 9c0c0946a..73eac7ef6 100644 --- a/.clang-format +++ b/.clang-format @@ -28,3 +28,5 @@ EmptyLineBeforeAccessModifier: Leave #PackConstructorInitializers: BinPack BreakBeforeBinaryOperators: NonAssignment AlwaysBreakBeforeMultilineStrings: true +IndentPPDirectives: AfterHash +PPIndentWidth: 2 From fa01db9626a2e7ba97d9fea013f2f80f14251987 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Apr 2024 17:38:29 +0200 Subject: [PATCH 181/211] StorePathAccessor: Fix path display Set the prefix to the store path to fix messages like "copying '/' to the store" to "copying '/nix/store/bla' to the store". --- src/libfetchers/fs-input-accessor.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index ee24c621a..d85363808 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -24,7 +24,10 @@ ref makeStorePathAccessor( const StorePath & storePath) { // FIXME: should use `store->getFSAccessor()` - return makeFSInputAccessor(std::filesystem::path { store->toRealPath(storePath) }); + auto root = std::filesystem::path { store->toRealPath(storePath) }; + auto accessor = makeFSInputAccessor(root); + accessor->setPathDisplay(root); + return accessor; } SourcePath getUnfilteredRootPath(CanonPath path) From 6df58a0891ecec7beb09b6459d432a005aafc7d4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Apr 2024 18:22:33 +0200 Subject: [PATCH 182/211] MercurialInputScheme: Improve path display --- src/libfetchers/unix/mercurial.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/unix/mercurial.cc b/src/libfetchers/unix/mercurial.cc index a2702338f..4e0b26274 100644 --- a/src/libfetchers/unix/mercurial.cc +++ b/src/libfetchers/unix/mercurial.cc @@ -352,7 +352,11 @@ struct MercurialInputScheme : InputScheme auto storePath = fetchToStore(store, input); - return {makeStorePathAccessor(store, storePath), input}; + auto accessor = makeStorePathAccessor(store, storePath); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + return {accessor, input}; } bool isLocked(const Input & input) const override From 25265a9365ee8ad5917919be61d2c948a5dc6ffb Mon Sep 17 00:00:00 2001 From: crayor1 <126188437+crayor1@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:36:41 +0200 Subject: [PATCH 183/211] Double word is superfluous --- doc/manual/src/protocols/json/store-object-info.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..d462f74f8 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -83,7 +83,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traverising the other fields across all the store objects in a [closure]. * `closureSize`: From 548a12c1fe9dc9b6065b575db0337033385e18f3 Mon Sep 17 00:00:00 2001 From: crayor1 <126188437+crayor1@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:54:45 +0200 Subject: [PATCH 184/211] Fix typo in hacking.md --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 2ff70f500..d56ac29a4 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -196,7 +196,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] From 79363b22730cb1a0e20e34b9c9f63f5568217926 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2024 15:38:44 +0200 Subject: [PATCH 185/211] MountedInputAccessor, FilteringInputAccessor: Respect the path display prefix/suffix This was causing Git paths not to be rendered correctly. --- src/libfetchers/filtering-input-accessor.cc | 2 +- src/libfetchers/filtering-input-accessor.hh | 4 +++- src/libfetchers/mounted-input-accessor.cc | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc index 32343abc4..e0cbfd905 100644 --- a/src/libfetchers/filtering-input-accessor.cc +++ b/src/libfetchers/filtering-input-accessor.cc @@ -38,7 +38,7 @@ std::string FilteringInputAccessor::readLink(const CanonPath & path) std::string FilteringInputAccessor::showPath(const CanonPath & path) { - return next->showPath(prefix / path); + return displayPrefix + next->showPath(prefix / path) + displaySuffix; } void FilteringInputAccessor::checkAccess(const CanonPath & path) diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh index 8111a72c5..133a6cee3 100644 --- a/src/libfetchers/filtering-input-accessor.hh +++ b/src/libfetchers/filtering-input-accessor.hh @@ -27,7 +27,9 @@ struct FilteringInputAccessor : InputAccessor : next(src.accessor) , prefix(src.path) , makeNotAllowedError(std::move(makeNotAllowedError)) - { } + { + displayPrefix.clear(); + } std::string readFile(const CanonPath & path) override; diff --git a/src/libfetchers/mounted-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc index 6f397eb17..b1eeaa97d 100644 --- a/src/libfetchers/mounted-input-accessor.cc +++ b/src/libfetchers/mounted-input-accessor.cc @@ -9,6 +9,8 @@ struct MountedInputAccessor : InputAccessor MountedInputAccessor(std::map> _mounts) : mounts(std::move(_mounts)) { + displayPrefix.clear(); + // Currently we require a root filesystem. This could be relaxed. assert(mounts.contains(CanonPath::root)); @@ -48,7 +50,7 @@ struct MountedInputAccessor : InputAccessor std::string showPath(const CanonPath & path) override { auto [accessor, subpath] = resolve(path); - return accessor->showPath(subpath); + return displayPrefix + accessor->showPath(subpath) + displaySuffix; } std::pair, CanonPath> resolve(CanonPath path) From 6892c9803c1559a7bfda57b7a5afdd6b62c4516e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2024 16:11:57 +0200 Subject: [PATCH 186/211] GitInputScheme: Fix path display for workdirs and submodules --- src/libfetchers/unix/git.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libfetchers/unix/git.cc b/src/libfetchers/unix/git.cc index 45e62ebe1..18915c0a7 100644 --- a/src/libfetchers/unix/git.cc +++ b/src/libfetchers/unix/git.cc @@ -645,6 +645,7 @@ struct GitInputScheme : InputScheme auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); mounts.insert_or_assign(submodule.path, submoduleAccessor); } @@ -681,6 +682,8 @@ struct GitInputScheme : InputScheme exportIgnore, makeNotAllowedError(repoInfo.url)); + accessor->setPathDisplay(repoInfo.url); + /* If the repo has submodules, return a mounted input accessor consisting of the accessor for the top-level repo and the accessors for the submodule workdirs. */ @@ -697,6 +700,7 @@ struct GitInputScheme : InputScheme auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); /* If the submodule is dirty, mark this repo dirty as well. */ From 28e0f0a04cefee9820492b89ca54a93f6790b3ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2024 17:43:17 +0200 Subject: [PATCH 187/211] Fix another typo --- doc/manual/src/protocols/json/store-object-info.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index d462f74f8..179cafbb4 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -83,7 +83,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: From c75b143b6caf1e671c7f3a70dd91eb20c3036ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= Date: Mon, 15 Apr 2024 21:05:52 +0200 Subject: [PATCH 188/211] C API: nix_get_string now accepts a callback to return the value --- doc/external-api/README.md | 37 +++++++++++++++++--------- src/libexpr-c/nix_api_value.cc | 12 ++++----- src/libexpr-c/nix_api_value.h | 8 +++--- tests/unit/libexpr/nix_api_expr.cc | 20 ++++++++------ tests/unit/libexpr/nix_api_external.cc | 7 ++++- tests/unit/libexpr/nix_api_value.cc | 16 +++++++---- 6 files changed, 65 insertions(+), 35 deletions(-) diff --git a/doc/external-api/README.md b/doc/external-api/README.md index 8a6f1c085..167c02199 100644 --- a/doc/external-api/README.md +++ b/doc/external-api/README.md @@ -40,24 +40,37 @@ Nix expression `builtins.nixVersion`. #include #include #include +#include +#include // NOTE: This example lacks all error handling. Production code must check for // errors, as some return values will be undefined. -int main() { - nix_libexpr_init(NULL); - Store* store = nix_store_open(NULL, "dummy://", NULL); - EvalState* state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) - Value *value = nix_alloc_value(NULL, state); +void my_get_string_cb(const char * start, unsigned int n, char ** user_data) +{ + *user_data = strdup(start); +} - nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); - nix_value_force(NULL, state, value); - printf("Nix version: %s\n", nix_get_string(NULL, value)); +int main() +{ + nix_libexpr_init(NULL); - nix_gc_decref(NULL, value); - nix_state_free(state); - nix_store_free(store); - return 0; + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; } ``` diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 817464fa8..6592591db 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -15,9 +15,9 @@ #include "value/context.hh" #ifdef HAVE_BOEHMGC -# include "gc/gc.h" -# define GC_INCLUDE_NEW 1 -# include "gc_cpp.h" +# include "gc/gc.h" +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" #endif // Helper function to throw an exception if value is null @@ -166,16 +166,16 @@ bool nix_get_bool(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(false); } -const char * nix_get_string(nix_c_context * context, const Value * value) +nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data) { if (context) context->last_err_code = NIX_OK; try { auto & v = check_value_not_null(value); assert(v.type() == nix::nString); - return v.c_str(); + call_nix_get_string_callback(v.c_str(), callback, user_data); } - NIXC_CATCH_ERRS_NULL + NIXC_CATCH_ERRS } const char * nix_get_path_string(nix_c_context * context, const Value * value) diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index c8e85d181..e6744e610 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -178,10 +178,13 @@ bool nix_get_bool(nix_c_context * context, const Value * value); * * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect + * @param[in] callback Called with the string value. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. * @return string - * @return NULL in case of error. + * @return error code, NIX_OK on success. */ -const char * nix_get_string(nix_c_context * context, const Value * value); +nix_err +nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data); /** @brief Get path as string * @param[out] context Optional, stores error information @@ -482,7 +485,6 @@ const StorePath * nix_realised_string_get_store_path(nix_realised_string * reali */ void nix_realised_string_free(nix_realised_string * realised_string); - // cffi end #ifdef __cplusplus } diff --git a/tests/unit/libexpr/nix_api_expr.cc b/tests/unit/libexpr/nix_api_expr.cc index f5c66536d..0818f1cab 100644 --- a/tests/unit/libexpr/nix_api_expr.cc +++ b/tests/unit/libexpr/nix_api_expr.cc @@ -17,9 +17,10 @@ TEST_F(nix_api_expr_test, nix_expr_eval_from_string) { nix_expr_eval_from_string(nullptr, state, "builtins.nixVersion", ".", value); nix_value_force(nullptr, state, value); - auto result = nix_get_string(nullptr, value); + std::string result; + nix_get_string(nullptr, value, OBSERVE_STRING(result)); - ASSERT_STREQ(PACKAGE_VERSION, result); + ASSERT_STREQ(PACKAGE_VERSION, result.c_str()); } TEST_F(nix_api_expr_test, nix_expr_eval_add_numbers) @@ -47,7 +48,8 @@ TEST_F(nix_api_expr_test, nix_expr_eval_drv) nix_value_call(ctx, stateResult, valueFn, value, valueResult); ASSERT_EQ(NIX_TYPE_STRING, nix_get_type(nullptr, valueResult)); - std::string p = nix_get_string(nullptr, valueResult); + std::string p; + nix_get_string(nullptr, valueResult, OBSERVE_STRING(p)); std::string pEnd = "-myname"; ASSERT_EQ(pEnd, p.substr(p.size() - pEnd.size())); @@ -69,7 +71,8 @@ TEST_F(nix_api_expr_test, nix_build_drv) nix_expr_eval_from_string(nullptr, state, expr, ".", value); Value * drvPathValue = nix_get_attr_byname(nullptr, value, state, "drvPath"); - const char * drvPath = nix_get_string(nullptr, drvPathValue); + std::string drvPath; + nix_get_string(nullptr, drvPathValue, OBSERVE_STRING(drvPath)); std::string p = drvPath; std::string pEnd = "-myname.drv"; @@ -78,18 +81,19 @@ TEST_F(nix_api_expr_test, nix_build_drv) // NOTE: .drvPath should be usually be ignored. Output paths are more versatile. // See https://github.com/NixOS/nix/issues/6507 // Use e.g. nix_string_realise to realise the output. - StorePath * drvStorePath = nix_store_parse_path(ctx, store, drvPath); + StorePath * drvStorePath = nix_store_parse_path(ctx, store, drvPath.c_str()); ASSERT_EQ(true, nix_store_is_valid_path(ctx, store, drvStorePath)); Value * outPathValue = nix_get_attr_byname(ctx, value, state, "outPath"); - const char * outPath = nix_get_string(ctx, outPathValue); + std::string outPath; + nix_get_string(ctx, outPathValue, OBSERVE_STRING(outPath)); p = outPath; pEnd = "-myname"; ASSERT_EQ(pEnd, p.substr(p.size() - pEnd.size())); ASSERT_EQ(true, drvStorePath->path.isDerivation()); - StorePath * outStorePath = nix_store_parse_path(ctx, store, outPath); + StorePath * outStorePath = nix_store_parse_path(ctx, store, outPath.c_str()); ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, outStorePath)); nix_store_realise(ctx, store, drvStorePath, nullptr, nullptr); @@ -142,7 +146,7 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context) }} a path: ${builtins.toFile "just-a-file" "ooh file good"} a derivation path by itself: ${ - builtins.unsafeDiscardOutputDependency + builtins.unsafeDiscardOutputDependency (derivation { name = "not-actually-built-yet"; system = builtins.currentSystem; diff --git a/tests/unit/libexpr/nix_api_external.cc b/tests/unit/libexpr/nix_api_external.cc index 7e2caed1b..2391f8317 100644 --- a/tests/unit/libexpr/nix_api_external.cc +++ b/tests/unit/libexpr/nix_api_external.cc @@ -6,7 +6,9 @@ #include "nix_api_expr_internal.h" #include "nix_api_value.h" #include "nix_api_external.h" + #include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" #include @@ -58,6 +60,9 @@ TEST_F(nix_api_expr_test, nix_expr_eval_external) nix_value_call(ctx, state, valueFn, value, valueResult); - ASSERT_STREQ("nix-external", nix_get_string(nullptr, valueResult)); + std::string string_value; + nix_get_string(nullptr, valueResult, OBSERVE_STRING(string_value)); + ASSERT_STREQ("nix-external", string_value.c_str()); } + } diff --git a/tests/unit/libexpr/nix_api_value.cc b/tests/unit/libexpr/nix_api_value.cc index 726960638..7fbb2bbdc 100644 --- a/tests/unit/libexpr/nix_api_value.cc +++ b/tests/unit/libexpr/nix_api_value.cc @@ -6,6 +6,7 @@ #include "nix_api_value.h" #include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" #include #include @@ -53,13 +54,15 @@ TEST_F(nix_api_expr_test, nix_value_set_get_bool) TEST_F(nix_api_expr_test, nix_value_set_get_string) { - ASSERT_EQ(nullptr, nix_get_string(ctx, nullptr)); - ASSERT_DEATH(nix_get_string(ctx, value), ""); + std::string string_value; + ASSERT_EQ(NIX_ERR_UNKNOWN, nix_get_string(ctx, nullptr, OBSERVE_STRING(string_value))); + ASSERT_DEATH(nix_get_string(ctx, value, OBSERVE_STRING(string_value)), ""); const char * myString = "some string"; nix_init_string(ctx, value, myString); - ASSERT_STREQ(myString, nix_get_string(ctx, value)); + nix_get_string(ctx, value, OBSERVE_STRING(string_value)); + ASSERT_STREQ(myString, string_value.c_str()); ASSERT_STREQ("a string", nix_get_typename(ctx, value)); ASSERT_EQ(NIX_TYPE_STRING, nix_get_type(ctx, value)); } @@ -162,11 +165,14 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) ASSERT_EQ(false, nix_has_attr_byname(ctx, value, state, "no-value")); out_value = nix_get_attr_byname(ctx, value, state, "b"); - ASSERT_STREQ("foo", nix_get_string(ctx, out_value)); + std::string string_value; + nix_get_string(ctx, out_value, OBSERVE_STRING(string_value)); + ASSERT_STREQ("foo", string_value.c_str()); nix_gc_decref(nullptr, out_value); out_value = nix_get_attr_byidx(ctx, value, state, 1, out_name); - ASSERT_STREQ("foo", nix_get_string(ctx, out_value)); + nix_get_string(ctx, out_value, OBSERVE_STRING(string_value)); + ASSERT_STREQ("foo", string_value.c_str()); ASSERT_STREQ("b", *out_name); nix_gc_decref(nullptr, out_value); From 94c861bebfd7d84607aa3d52617adea8c7eb5222 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Apr 2024 12:50:55 +0200 Subject: [PATCH 189/211] labeler.yml: Add contributor-experience Not exhaustive perhaps, but a good start. --- .github/labeler.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/labeler.yml b/.github/labeler.yml index b1b18c488..280265dbe 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,3 +1,10 @@ +"contributor-experience": + - changed-files: + - any-glob-to-any-file: "CONTRIBUTING.md" + - any-glob-to-any-file: ".github/ISSUE_TEMPLATE/*" + - any-glob-to-any-file: ".github/PULL_REQUEST_TEMPLATE.md" + - any-glob-to-any-file: "doc/manual/src/contributing/**" + "documentation": - changed-files: - any-glob-to-any-file: "doc/manual/*" From 45a1142a8e771aeccd19c83e3a52c5dade510c83 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Apr 2024 13:11:52 +0200 Subject: [PATCH 190/211] devShell: enable API docs Affects both the deps and the configure flags in configurePhase. --- package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.nix b/package.nix index c9e50c399..1e5b9e449 100644 --- a/package.nix +++ b/package.nix @@ -94,8 +94,8 @@ # Whether to build the internal/external API docs, can be done separately from # everything else. -, enableInternalAPIDocs ? false -, enableExternalAPIDocs ? false +, enableInternalAPIDocs ? forDevShell +, enableExternalAPIDocs ? forDevShell # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so From a863a75f0b205e5c4ed09346b6b92322a7925ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:34:53 +0200 Subject: [PATCH 191/211] Remove the giy-submodule test from Github actions It causes the CI to time out, so let's just run it on Hydra --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfd1dec5d..2b8eac49d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,4 +166,4 @@ jobs: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.gitSubmodules + - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes From 1f1cd97c715680369c36bb740e05636f0d29a1ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= Date: Wed, 17 Apr 2024 15:21:17 +0200 Subject: [PATCH 192/211] C API: Add section in Nix manual (#10519) --- doc/manual/src/contributing/documentation.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index 88b0bdaa9..e7f94ab8c 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -206,3 +206,22 @@ or inside `nix-shell` or `nix develop`: # make internal-api-html # xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html ``` + +## C API documentation (experimental) + +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +# nix build .#hydraJobs.external-api-docs +# xdg-open ./result/share/doc/nix/external-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +``` +# make external-api-html +# xdg-open ./outputs/doc/share/doc/nix/external-api/html/index.html +``` From 7a1135901dd161318d70a7fdc03124e20c175eb9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Apr 2024 15:34:40 +0200 Subject: [PATCH 193/211] local.mk: Solve warnings local.mk:5: warning: overriding recipe for target 'outputs/dev/include/nix/nix_api_expr.h' local.mk:5: warning: ignoring old recipe for target 'outputs/dev/include/nix/nix_api_expr.h' local.mk:5: warning: overriding recipe for target 'outputs/dev/include/nix/nix_api_external.h' local.mk:5: warning: ignoring old recipe for target 'outputs/dev/include/nix/nix_api_external.h' local.mk:5: warning: overriding recipe for target 'outputs/dev/include/nix/nix_api_value.h' local.mk:5: warning: ignoring old recipe for target 'outputs/dev/include/nix/nix_api_value.h' local.mk:5: warning: overriding recipe for target 'outputs/dev/include/nix/nix_api_store.h' local.mk:5: warning: ignoring old recipe for target 'outputs/dev/include/nix/nix_api_store.h' local.mk:5: warning: overriding recipe for target 'outputs/dev/include/nix/nix_api_util.h' local.mk:5: warning: ignoring old recipe for target 'outputs/dev/include/nix/nix_api_util.h' --- local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local.mk b/local.mk index 67ec35dcd..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -2,7 +2,7 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh) $(wildcard src/lib*/*.h $(filter-out %_internal.h, $(wildcard src/lib*c/*.h))), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) ifdef HOST_UNIX From d42a2643b0de67c97449f05607f696c86015f0ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= Date: Wed, 17 Apr 2024 16:11:03 +0200 Subject: [PATCH 194/211] Auto label C API PRs --- .github/labeler.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/labeler.yml b/.github/labeler.yml index 280265dbe..e036eb3c8 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,3 +1,9 @@ +"c api": + - changed-files: + - any-glob-to-any-file: "src/lib*-c/**/*" + - any-glob-to-any-file: "test/unit/**/nix_api_*" + - any-glob-to-any-file: "doc/external-api/**/*" + "contributor-experience": - changed-files: - any-glob-to-any-file: "CONTRIBUTING.md" From 05b9dac7547009821f4a698b139a49d5d1e4bc9e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 17 Apr 2024 11:47:07 -0400 Subject: [PATCH 195/211] Fix friend `struct`/`class` mismatch warning --- src/libexpr/attr-set.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index c90bb0633..ba798196d 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -189,7 +189,7 @@ public: bindings = newBindings; } - friend class ExprAttrs; + friend struct ExprAttrs; }; } From 2248a3f5451adf6c098fc871fb61fae6cc2b5979 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 17 Apr 2024 11:34:09 -0400 Subject: [PATCH 196/211] Create no-op Window pathlocks implementation This keeps the call sites simple, eventually this should be filled in. --- src/build-remote/build-remote.cc | 1 + src/libstore/gc.cc | 1 + src/libstore/local-store.cc | 2 + src/libstore/local.mk | 3 + src/libstore/lock.cc | 2 + src/libstore/pathlocks.cc | 152 ----------------------- src/libstore/pathlocks.hh | 32 +---- src/libstore/unix/pathlocks-impl.hh | 38 ++++++ src/libstore/unix/pathlocks.cc | 165 +++++++++++++++++++++++++ src/libstore/windows/pathlocks-impl.hh | 2 + src/libstore/windows/pathlocks.cc | 16 +++ 11 files changed, 232 insertions(+), 182 deletions(-) create mode 100644 src/libstore/unix/pathlocks-impl.hh create mode 100644 src/libstore/unix/pathlocks.cc create mode 100644 src/libstore/windows/pathlocks-impl.hh create mode 100644 src/libstore/windows/pathlocks.cc diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 18eee830b..2a4723643 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -22,6 +22,7 @@ #include "experimental-features.hh" using namespace nix; +using namespace nix::unix; using std::cin; static void handleAlarm(int sig) { diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 88e943263..9b2e6d525 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -29,6 +29,7 @@ namespace nix { +using namespace nix::unix; static std::string gcSocketPath = "/gc-socket/socket"; static std::string gcRootsDir = "gcroots"; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a32b349a1..1593affd6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -52,6 +52,8 @@ namespace nix { +using namespace nix::unix; + std::string LocalStoreConfig::doc() { return diff --git a/src/libstore/local.mk b/src/libstore/local.mk index ccb7aeee2..0a91fce4b 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -36,6 +36,9 @@ INCLUDE_libstore := -I $(d) -I $(d)/build ifdef HOST_UNIX INCLUDE_libstore += -I $(d)/unix endif +ifdef HOST_WINDOWS + INCLUDE_libstore += -I $(d)/windows +endif libstore_CXXFLAGS += \ $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libstore) \ diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 023c74e34..fd7af171f 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -9,6 +9,8 @@ namespace nix { +using namespace nix::unix; + #if __linux__ static std::vector get_group_list(const char *username, gid_t group_id) diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 2b5b8dfe7..37793db5b 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -6,69 +6,9 @@ #include #include -#include -#include -#include -#include - namespace nix { - -AutoCloseFD openLockFile(const Path & path, bool create) -{ - AutoCloseFD fd; - - fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); - if (!fd && (create || errno != ENOENT)) - throw SysError("opening lock file '%1%'", path); - - return fd; -} - - -void deleteLockFile(const Path & path, int fd) -{ - /* Get rid of the lock file. Have to be careful not to introduce - races. Write a (meaningless) token to the file to indicate to - other processes waiting on this lock that the lock is stale - (deleted). */ - unlink(path.c_str()); - writeFull(fd, "d"); - /* Note that the result of unlink() is ignored; removing the lock - file is an optimisation, not a necessity. */ -} - - -bool lockFile(int fd, LockType lockType, bool wait) -{ - int type; - if (lockType == ltRead) type = LOCK_SH; - else if (lockType == ltWrite) type = LOCK_EX; - else if (lockType == ltNone) type = LOCK_UN; - else abort(); - - if (wait) { - while (flock(fd, type) != 0) { - checkInterrupt(); - if (errno != EINTR) - throw SysError("acquiring/releasing lock"); - else - return false; - } - } else { - while (flock(fd, type | LOCK_NB) != 0) { - checkInterrupt(); - if (errno == EWOULDBLOCK) return false; - if (errno != EINTR) - throw SysError("acquiring/releasing lock"); - } - } - - return true; -} - - PathLocks::PathLocks() : deletePaths(false) { @@ -82,68 +22,6 @@ PathLocks::PathLocks(const PathSet & paths, const std::string & waitMsg) } -bool PathLocks::lockPaths(const PathSet & paths, - const std::string & waitMsg, bool wait) -{ - assert(fds.empty()); - - /* Note that `fds' is built incrementally so that the destructor - will only release those locks that we have already acquired. */ - - /* Acquire the lock for each path in sorted order. This ensures - that locks are always acquired in the same order, thus - preventing deadlocks. */ - for (auto & path : paths) { - checkInterrupt(); - Path lockPath = path + ".lock"; - - debug("locking path '%1%'", path); - - AutoCloseFD fd; - - while (1) { - - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); - - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - return false; - } - } - - debug("lock acquired on '%1%'", lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("statting lock file '%1%'", lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug("open lock file '%1%' has become stale", lockPath); - else - break; - } - - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); - } - - return true; -} - - PathLocks::~PathLocks() { try { @@ -154,40 +32,10 @@ PathLocks::~PathLocks() } -void PathLocks::unlock() -{ - for (auto & i : fds) { - if (deletePaths) deleteLockFile(i.second, i.first); - - if (close(i.first) == -1) - printError( - "error (ignored): cannot close lock file on '%1%'", - i.second); - - debug("lock released on '%1%'", i.second); - } - - fds.clear(); -} - - void PathLocks::setDeletion(bool deletePaths) { this->deletePaths = deletePaths; } -FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg) - : fd(fd) -{ - if (wait) { - if (!lockFile(fd, lockType, false)) { - printInfo("%s", waitMsg); - acquired = lockFile(fd, lockType, true); - } - } else - acquired = lockFile(fd, lockType, false); -} - - } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 7fcfa2e40..b97fbecb9 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -5,22 +5,6 @@ namespace nix { -/** - * Open (possibly create) a lock file and return the file descriptor. - * -1 is returned if create is false and the lock could not be opened - * because it doesn't exist. Any other error throws an exception. - */ -AutoCloseFD openLockFile(const Path & path, bool create); - -/** - * Delete an open lock file. - */ -void deleteLockFile(const Path & path, int fd); - -enum LockType { ltRead, ltWrite, ltNone }; - -bool lockFile(int fd, LockType lockType, bool wait); - class PathLocks { private: @@ -40,18 +24,6 @@ public: void setDeletion(bool deletePaths); }; -struct FdLock -{ - int fd; - bool acquired = false; - - FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg); - - ~FdLock() - { - if (acquired) - lockFile(fd, ltNone, false); - } -}; - } + +#include "pathlocks-impl.hh" diff --git a/src/libstore/unix/pathlocks-impl.hh b/src/libstore/unix/pathlocks-impl.hh new file mode 100644 index 000000000..31fe968bb --- /dev/null +++ b/src/libstore/unix/pathlocks-impl.hh @@ -0,0 +1,38 @@ +#pragma once +///@file + +#include "file-descriptor.hh" + +namespace nix::unix { + +/** + * Open (possibly create) a lock file and return the file descriptor. + * -1 is returned if create is false and the lock could not be opened + * because it doesn't exist. Any other error throws an exception. + */ +AutoCloseFD openLockFile(const Path & path, bool create); + +/** + * Delete an open lock file. + */ +void deleteLockFile(const Path & path, int fd); + +enum LockType { ltRead, ltWrite, ltNone }; + +bool lockFile(int fd, LockType lockType, bool wait); + +struct FdLock +{ + int fd; + bool acquired = false; + + FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg); + + ~FdLock() + { + if (acquired) + lockFile(fd, ltNone, false); + } +}; + +} diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc new file mode 100644 index 000000000..32c1b9ff4 --- /dev/null +++ b/src/libstore/unix/pathlocks.cc @@ -0,0 +1,165 @@ +#include "pathlocks.hh" +#include "util.hh" +#include "sync.hh" +#include "signals.hh" + +#include +#include + +#include +#include +#include +#include + + +namespace nix { + +using namespace nix::unix; + +AutoCloseFD unix::openLockFile(const Path & path, bool create) +{ + AutoCloseFD fd; + + fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); + if (!fd && (create || errno != ENOENT)) + throw SysError("opening lock file '%1%'", path); + + return fd; +} + + +void unix::deleteLockFile(const Path & path, int fd) +{ + /* Get rid of the lock file. Have to be careful not to introduce + races. Write a (meaningless) token to the file to indicate to + other processes waiting on this lock that the lock is stale + (deleted). */ + unlink(path.c_str()); + writeFull(fd, "d"); + /* Note that the result of unlink() is ignored; removing the lock + file is an optimisation, not a necessity. */ +} + + +bool unix::lockFile(int fd, LockType lockType, bool wait) +{ + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; + else abort(); + + if (wait) { + while (flock(fd, type) != 0) { + checkInterrupt(); + if (errno != EINTR) + throw SysError("acquiring/releasing lock"); + else + return false; + } + } else { + while (flock(fd, type | LOCK_NB) != 0) { + checkInterrupt(); + if (errno == EWOULDBLOCK) return false; + if (errno != EINTR) + throw SysError("acquiring/releasing lock"); + } + } + + return true; +} + + +bool PathLocks::lockPaths(const PathSet & paths, + const std::string & waitMsg, bool wait) +{ + assert(fds.empty()); + + /* Note that `fds' is built incrementally so that the destructor + will only release those locks that we have already acquired. */ + + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ + for (auto & path : paths) { + checkInterrupt(); + Path lockPath = path + ".lock"; + + debug("locking path '%1%'", path); + + AutoCloseFD fd; + + while (1) { + + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); + + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; + } + } + + debug("lock acquired on '%1%'", lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting lock file '%1%'", lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug("open lock file '%1%' has become stale", lockPath); + else + break; + } + + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); + } + + return true; +} + + +void PathLocks::unlock() +{ + for (auto & i : fds) { + if (deletePaths) deleteLockFile(i.second, i.first); + + if (close(i.first) == -1) + printError( + "error (ignored): cannot close lock file on '%1%'", + i.second); + + debug("lock released on '%1%'", i.second); + } + + fds.clear(); +} + + +FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg) + : fd(fd) +{ + if (wait) { + if (!lockFile(fd, lockType, false)) { + printInfo("%s", waitMsg); + acquired = lockFile(fd, lockType, true); + } + } else + acquired = lockFile(fd, lockType, false); +} + + +} diff --git a/src/libstore/windows/pathlocks-impl.hh b/src/libstore/windows/pathlocks-impl.hh new file mode 100644 index 000000000..ba3ad28d9 --- /dev/null +++ b/src/libstore/windows/pathlocks-impl.hh @@ -0,0 +1,2 @@ +#pragma once +///@file Needed because Unix-specific counterpart diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc new file mode 100644 index 000000000..ab4294c2a --- /dev/null +++ b/src/libstore/windows/pathlocks.cc @@ -0,0 +1,16 @@ +#include "logging.hh" +#include "pathlocks.hh" + +namespace nix { + +bool PathLocks::lockPaths(const PathSet & _paths, const std::string & waitMsg, bool wait) +{ + return true; +} + +void PathLocks::unlock() +{ + warn("PathLocks::unlock: not yet implemented"); +} + +} From 8433027e353b03bcbf36fae180394aaf8a21a19b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 Sep 2023 17:35:16 -0400 Subject: [PATCH 197/211] Build a minimized Nix with MinGW At this point many features are stripped out, but this works: - Can run libnix{util,store,expr} unit tests - Can run some Nix commands Co-Authored-By volth Co-Authored-By Brian McKenna --- Makefile | 12 +- m4/gcc_bug_80431.m4 | 8 +- precompiled-headers.h | 21 +-- src/libcmd/common-eval-args.cc | 2 +- src/libcmd/markdown.cc | 4 +- src/libcmd/repl-interacter.cc | 4 + src/libexpr/eval.cc | 10 +- src/libexpr/eval.hh | 2 + src/libexpr/primops.cc | 11 +- src/libexpr/search-path.hh | 3 + src/libfetchers/fs-input-accessor.cc | 2 +- src/libfetchers/git-utils.cc | 16 +- src/libmain/shared.cc | 15 ++ src/libmain/shared.hh | 7 +- src/libmain/{ => unix}/stack.cc | 0 src/libstore/builtins/buildenv.cc | 6 +- src/libstore/daemon.cc | 7 +- src/libstore/filetransfer.cc | 12 ++ src/libstore/globals.cc | 17 +- src/libstore/globals.hh | 2 + src/libstore/local.mk | 17 +- src/libstore/misc.cc | 1 - src/libstore/profiles.hh | 1 + src/libstore/remote-fs-accessor.cc | 8 +- src/libstore/ssh.cc | 16 +- src/libstore/ssh.hh | 10 +- src/libstore/store-api.cc | 21 ++- src/libstore/{ => unix}/build/child.cc | 0 src/libstore/{ => unix}/build/child.hh | 0 .../{ => unix}/build/derivation-goal.cc | 0 .../{ => unix}/build/derivation-goal.hh | 0 .../build/drv-output-substitution-goal.cc | 0 .../build/drv-output-substitution-goal.hh | 0 src/libstore/{ => unix}/build/entry-points.cc | 0 src/libstore/{ => unix}/build/goal.cc | 0 src/libstore/{ => unix}/build/goal.hh | 0 .../{ => unix}/build/hook-instance.cc | 0 .../{ => unix}/build/hook-instance.hh | 0 .../{ => unix}/build/local-derivation-goal.cc | 0 .../{ => unix}/build/local-derivation-goal.hh | 0 src/libstore/{ => unix}/build/personality.cc | 0 src/libstore/{ => unix}/build/personality.hh | 0 .../{ => unix}/build/sandbox-defaults.sb | 0 .../{ => unix}/build/sandbox-minimal.sb | 0 .../{ => unix}/build/sandbox-network.sb | 0 .../{ => unix}/build/substitution-goal.cc | 0 .../{ => unix}/build/substitution-goal.hh | 0 src/libstore/{ => unix}/build/worker.cc | 0 src/libstore/{ => unix}/build/worker.hh | 0 src/libstore/{ => unix}/builtins/fetchurl.cc | 0 .../{ => unix}/builtins/unpack-channel.cc | 0 .../{ => unix}/ca-specific-schema.sql | 0 src/libstore/{ => unix}/gc.cc | 0 .../{ => unix}/local-overlay-store.cc | 0 .../{ => unix}/local-overlay-store.hh | 0 .../{ => unix}/local-overlay-store.md | 0 src/libstore/{ => unix}/local-store.cc | 0 src/libstore/{ => unix}/local-store.hh | 0 src/libstore/{ => unix}/local-store.md | 0 src/libstore/{ => unix}/lock.cc | 0 src/libstore/{ => unix}/lock.hh | 0 src/libstore/{ => unix}/optimise-store.cc | 0 .../{ => unix}/posix-fs-canonicalise.cc | 0 .../{ => unix}/posix-fs-canonicalise.hh | 0 src/libstore/{ => unix}/schema.sql | 0 src/libstore/{ => unix}/uds-remote-store.cc | 0 src/libstore/{ => unix}/uds-remote-store.hh | 0 src/libstore/{ => unix}/uds-remote-store.md | 0 src/libstore/windows/build.cc | 37 +++++ src/libutil/args.cc | 6 +- src/libutil/current-process.cc | 10 +- src/libutil/current-process.hh | 9 +- src/libutil/environment-variables.hh | 7 + src/libutil/error.hh | 17 ++ src/libutil/file-descriptor.cc | 27 +++- src/libutil/file-descriptor.hh | 60 ++++++- src/libutil/file-path.hh | 52 ++++++ src/libutil/file-system.cc | 109 ++++++++++--- src/libutil/file-system.hh | 14 ++ src/libutil/fs-sink.cc | 31 +++- src/libutil/local.mk | 6 + src/libutil/logging.cc | 19 ++- src/libutil/posix-source-accessor.cc | 28 ++-- src/libutil/{unix => }/processes.hh | 12 +- src/libutil/serialise.cc | 14 ++ src/libutil/terminal.cc | 9 +- src/libutil/terminal.hh | 4 + src/libutil/thread-pool.cc | 2 + src/libutil/unix/file-path.cc | 31 ++++ src/libutil/users.hh | 8 +- src/libutil/windows/environment-variables.cc | 17 ++ src/libutil/windows/file-descriptor.cc | 148 ++++++++++++++++++ src/libutil/windows/file-path.cc | 52 ++++++ src/libutil/windows/processes.cc | 48 ++++++ src/libutil/windows/signals-impl.hh | 41 +++++ src/libutil/windows/users.cc | 50 ++++++ src/libutil/windows/windows-error.cc | 31 ++++ src/libutil/windows/windows-error.hh | 51 ++++++ src/nix-env/nix-env.cc | 10 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-store/nix-store.cc | 29 +++- src/nix/cat.cc | 3 +- src/nix/develop.cc | 9 +- src/nix/dump-path.cc | 4 +- src/nix/eval.cc | 8 +- src/nix/local.mk | 12 +- src/nix/log.cc | 2 +- src/nix/main.cc | 30 +++- src/nix/prefetch.cc | 2 +- src/nix/sigs.cc | 4 +- tests/unit/libutil/tests.cc | 2 + 111 files changed, 1162 insertions(+), 140 deletions(-) rename src/libmain/{ => unix}/stack.cc (100%) rename src/libstore/{ => unix}/build/child.cc (100%) rename src/libstore/{ => unix}/build/child.hh (100%) rename src/libstore/{ => unix}/build/derivation-goal.cc (100%) rename src/libstore/{ => unix}/build/derivation-goal.hh (100%) rename src/libstore/{ => unix}/build/drv-output-substitution-goal.cc (100%) rename src/libstore/{ => unix}/build/drv-output-substitution-goal.hh (100%) rename src/libstore/{ => unix}/build/entry-points.cc (100%) rename src/libstore/{ => unix}/build/goal.cc (100%) rename src/libstore/{ => unix}/build/goal.hh (100%) rename src/libstore/{ => unix}/build/hook-instance.cc (100%) rename src/libstore/{ => unix}/build/hook-instance.hh (100%) rename src/libstore/{ => unix}/build/local-derivation-goal.cc (100%) rename src/libstore/{ => unix}/build/local-derivation-goal.hh (100%) rename src/libstore/{ => unix}/build/personality.cc (100%) rename src/libstore/{ => unix}/build/personality.hh (100%) rename src/libstore/{ => unix}/build/sandbox-defaults.sb (100%) rename src/libstore/{ => unix}/build/sandbox-minimal.sb (100%) rename src/libstore/{ => unix}/build/sandbox-network.sb (100%) rename src/libstore/{ => unix}/build/substitution-goal.cc (100%) rename src/libstore/{ => unix}/build/substitution-goal.hh (100%) rename src/libstore/{ => unix}/build/worker.cc (100%) rename src/libstore/{ => unix}/build/worker.hh (100%) rename src/libstore/{ => unix}/builtins/fetchurl.cc (100%) rename src/libstore/{ => unix}/builtins/unpack-channel.cc (100%) rename src/libstore/{ => unix}/ca-specific-schema.sql (100%) rename src/libstore/{ => unix}/gc.cc (100%) rename src/libstore/{ => unix}/local-overlay-store.cc (100%) rename src/libstore/{ => unix}/local-overlay-store.hh (100%) rename src/libstore/{ => unix}/local-overlay-store.md (100%) rename src/libstore/{ => unix}/local-store.cc (100%) rename src/libstore/{ => unix}/local-store.hh (100%) rename src/libstore/{ => unix}/local-store.md (100%) rename src/libstore/{ => unix}/lock.cc (100%) rename src/libstore/{ => unix}/lock.hh (100%) rename src/libstore/{ => unix}/optimise-store.cc (100%) rename src/libstore/{ => unix}/posix-fs-canonicalise.cc (100%) rename src/libstore/{ => unix}/posix-fs-canonicalise.hh (100%) rename src/libstore/{ => unix}/schema.sql (100%) rename src/libstore/{ => unix}/uds-remote-store.cc (100%) rename src/libstore/{ => unix}/uds-remote-store.hh (100%) rename src/libstore/{ => unix}/uds-remote-store.md (100%) create mode 100644 src/libstore/windows/build.cc create mode 100644 src/libutil/file-path.hh rename src/libutil/{unix => }/processes.hh (95%) create mode 100644 src/libutil/unix/file-path.cc create mode 100644 src/libutil/windows/environment-variables.cc create mode 100644 src/libutil/windows/file-descriptor.cc create mode 100644 src/libutil/windows/file-path.cc create mode 100644 src/libutil/windows/processes.cc create mode 100644 src/libutil/windows/signals-impl.hh create mode 100644 src/libutil/windows/users.cc create mode 100644 src/libutil/windows/windows-error.cc create mode 100644 src/libutil/windows/windows-error.hh diff --git a/Makefile b/Makefile index 0028c957a..ba5a6cd92 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ clean-files += $(buildprefix)Makefile.config # List makefiles +include mk/platform.mk + ifeq ($(ENABLE_BUILD), yes) makefiles = \ mk/precompiled-headers.mk \ @@ -20,7 +22,10 @@ makefiles = \ src/nix/local.mk \ src/libutil-c/local.mk \ src/libstore-c/local.mk \ - src/libexpr-c/local.mk \ + src/libexpr-c/local.mk + +ifdef HOST_UNIX +makefiles += \ scripts/local.mk \ misc/bash/local.mk \ misc/fish/local.mk \ @@ -29,6 +34,7 @@ makefiles = \ misc/launchd/local.mk \ misc/upstart/local.mk endif +endif ifeq ($(ENABLE_UNIT_TESTS), yes) makefiles += \ @@ -42,6 +48,7 @@ makefiles += \ endif ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) +ifdef HOST_UNIX makefiles += \ tests/functional/local.mk \ tests/functional/ca/local.mk \ @@ -51,6 +58,7 @@ makefiles += \ tests/functional/test-libstoreconsumer/local.mk \ tests/functional/plugins/local.mk endif +endif # Some makefiles require access to built programs and must be included late. makefiles-late = @@ -79,8 +87,6 @@ else unexport NIX_HARDENING_ENABLE endif -include mk/platform.mk - ifdef HOST_WINDOWS # Windows DLLs are stricter about symbol visibility than Unix shared # objects --- see https://gcc.gnu.org/wiki/Visibility for details. diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 index e42f01956..cdc4ddb40 100644 --- a/m4/gcc_bug_80431.m4 +++ b/m4/gcc_bug_80431.m4 @@ -46,11 +46,13 @@ AC_DEFUN([ENSURE_NO_GCC_BUG_80431], ]])], [status_80431=0], [status_80431=$?], - [ - # Assume we're bug-free when cross-compiling - ]) + [status_80431='']) AC_LANG_POP(C++) AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], [0],[ AC_MSG_RESULT(yes) ], diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index b87bbbc27..c6ee0d0b2 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -181,7 +181,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) v->mkString(arg.s); }, [&](const AutoArgFile & arg) { - v->mkString(readFile(arg.path)); + v->mkString(readFile(arg.path.string())); }, [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO)); diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index d62ff0d96..88c3f640b 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -3,9 +3,9 @@ #include "finally.hh" #include "terminal.hh" -#include #if HAVE_LOWDOWN -#include +# include +# include #endif namespace nix { diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 3e34ecdb6..eb4361e25 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -137,6 +137,7 @@ static constexpr const char * promptForType(ReplPromptType promptType) bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) { +#ifndef _WIN32 // TODO use more signals.hh for this struct sigaction act, old; sigset_t savedSignalMask, set; @@ -161,9 +162,12 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT }; setupSignals(); +#endif char * s = readline(promptForType(promptType)); Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this restoreSignals(); +#endif if (g_signal_received) { g_signal_received = 0; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a21e1d75e..72da1c465 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -33,15 +33,17 @@ #include #include #include -#include #include #include #include -#include #include #include +#ifndef _WIN32 // TODO use portable implementation +# include +#endif + #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -2627,9 +2629,11 @@ void EvalState::maybePrintStats() void EvalState::printStatistics() { +#ifndef _WIN32 // TODO use portable implementation struct rusage buf; getrusage(RUSAGE_SELF, &buf); float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); +#endif uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *); uint64_t bLists = nrListElems * sizeof(Value *); @@ -2646,7 +2650,9 @@ void EvalState::printStatistics() if (outPath != "-") fs.open(outPath, std::fstream::out); json topObj = json::object(); +#ifndef _WIN32 // TODO implement topObj["cpuTime"] = cpuTime; +#endif topObj["envs"] = { {"number", nrEnvs}, {"elements", nrValuesInEnvs}, diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index df388d93e..af65fdcba 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -161,6 +161,8 @@ struct DebugTrace { bool isError; }; +// Don't want Windows function +#undef SearchPath class EvalState : public std::enable_shared_from_this { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7e7952735..f03acc2da 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -28,7 +28,10 @@ #include #include #include -#include + +#ifndef _WIN32 +# include +#endif #include @@ -331,6 +334,8 @@ static RegisterPrimOp primop_import({ } }); +#ifndef _WIN32 // TODO implement via DLL loading on Windows + /* Want reasonable symbol names, so extern C */ /* !!! Should we pass the Pos or the file name too? */ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); @@ -403,6 +408,8 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) } } +#endif + /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { @@ -4593,6 +4600,7 @@ void EvalState::createBaseEnv() )", }); +#ifndef _WIN32 // TODO implement on Windows // Miscellaneous if (evalSettings.enableNativeCode) { addPrimOp({ @@ -4606,6 +4614,7 @@ void EvalState::createBaseEnv() .fun = prim_exec, }); } +#endif addPrimOp({ .name = "__traceVerbose", diff --git a/src/libexpr/search-path.hh b/src/libexpr/search-path.hh index ce78135b5..231752ea6 100644 --- a/src/libexpr/search-path.hh +++ b/src/libexpr/search-path.hh @@ -8,6 +8,9 @@ namespace nix { +// Do not want the windows macro (alias to `SearchPathA`) +#undef SearchPath + /** * A "search path" is a list of ways look for something, used with * `builtins.findFile` and `< >` lookup expressions. diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index d85363808..2bbe53e11 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -26,7 +26,7 @@ ref makeStorePathAccessor( // FIXME: should use `store->getFSAccessor()` auto root = std::filesystem::path { store->toRealPath(storePath) }; auto accessor = makeFSInputAccessor(root); - accessor->setPathDisplay(root); + accessor->setPathDisplay(root.string()); return accessor; } diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 5ecd825b7..a4a00374c 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -151,11 +151,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { initLibGit2(); - if (pathExists(path.native())) { - if (git_repository_open(Setter(repo), path.c_str())) + if (pathExists(path.string())) { + if (git_repository_open(Setter(repo), path.string().c_str())) throw Error("opening Git repository '%s': %s", path, git_error_last()->message); } else { - if (git_repository_init(Setter(repo), path.c_str(), bare)) + if (git_repository_init(Setter(repo), path.string().c_str(), bare)) throw Error("creating Git repository '%s': %s", path, git_error_last()->message); } } @@ -216,7 +216,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::vector parseSubmodules(const std::filesystem::path & configFile) { GitConfig config; - if (git_config_open_ondisk(Setter(config), configFile.c_str())) + if (git_config_open_ondisk(Setter(config), configFile.string().c_str())) throw Error("parsing .gitmodules file: %s", git_error_last()->message); ConfigIterator it; @@ -288,7 +288,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this /* Get submodule info. */ auto modulesFile = path / ".gitmodules"; - if (pathExists(modulesFile)) + if (pathExists(modulesFile.string())) info.submodules = parseSubmodules(modulesFile); return info; @@ -377,10 +377,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this auto dir = this->path; Strings gitArgs; if (shallow) { - gitArgs = { "-C", dir, "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec }; + gitArgs = { "-C", dir.string(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec }; } else { - gitArgs = { "-C", dir, "fetch", "--quiet", "--force", "--", url, refspec }; + gitArgs = { "-C", dir.string(), "fetch", "--quiet", "--force", "--", url, refspec }; } runProgram(RunOptions { @@ -426,7 +426,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this .args = { "-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, - "-C", path, + "-C", path.string(), "verify-commit", rev.gitRev() }, diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4c9051d3b..a43a00f16 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -108,7 +108,9 @@ std::string getArg(const std::string & opt, return *i; } +#ifndef _WIN32 static void sigHandler(int signo) { } +#endif void initNix() @@ -121,6 +123,7 @@ void initNix() initLibStore(); +#ifndef _WIN32 unix::startSignalHandlerThread(); /* Reset SIGCHLD to its default. */ @@ -135,6 +138,7 @@ void initNix() /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ act.sa_handler = sigHandler; if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); +#endif #if __APPLE__ /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. @@ -156,21 +160,26 @@ void initNix() if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); #endif +#ifndef _WIN32 /* Register a SIGSEGV handler to detect stack overflows. Why not initLibExpr()? initGC() is essentially that, but detectStackOverflow is not an instance of the init function concept, as it may have to be invoked more than once per process. */ detectStackOverflow(); +#endif /* There is no privacy in the Nix system ;-) At least not for now. In particular, store objects should be readable by everybody. */ umask(0022); +#ifndef _WIN32 /* Initialise the PRNG. */ struct timeval tv; gettimeofday(&tv, 0); srandom(tv.tv_usec); +#endif + } @@ -365,6 +374,9 @@ RunPager::RunPager() Pipe toPager; toPager.create(); +#ifdef _WIN32 // TODO re-enable on Windows, once we can start processes. + throw Error("Commit signature verification not implemented on Windows yet"); +#else pid = startProcess([&]() { if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); @@ -383,17 +395,20 @@ RunPager::RunPager() std_out = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping standard output"); +#endif } RunPager::~RunPager() { try { +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. if (pid != -1) { std::cout.flush(); dup2(std_out, STDOUT_FILENO); pid.wait(); } +#endif } catch (...) { ignoreException(); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 99c3dffab..3c657d2b7 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "file-descriptor.hh" #include "processes.hh" #include "args.hh" #include "args/root.hh" @@ -89,8 +90,10 @@ public: ~RunPager(); private: +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. Pid pid; - int std_out; +#endif + Descriptor std_out; }; extern volatile ::sig_atomic_t blockInt; @@ -112,6 +115,7 @@ struct PrintFreed }; +#ifndef _WIN32 /** * Install a SIGSEGV handler to detect stack overflows. */ @@ -141,5 +145,6 @@ extern std::function stackOverflowHandler; * logger. Exits the process immediately after. */ void defaultStackOverflowHandler(siginfo_t * info, void * ctx); +#endif } diff --git a/src/libmain/stack.cc b/src/libmain/unix/stack.cc similarity index 100% rename from src/libmain/stack.cc rename to src/libmain/unix/stack.cc diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 31a6b32f1..e009f5b9d 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -76,7 +76,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); if (unlink(dstFile.c_str()) == -1) throw SysError("unlinking '%1%'", dstFile); - if (mkdir(dstFile.c_str(), 0755) == -1) + if (mkdir(dstFile.c_str() + #ifndef _WIN32 // TODO abstract mkdir perms for Windows + , 0755 + #endif + ) == -1) throw SysError("creating directory '%1%'", dstFile); createLinks(state, target, dstFile, state.priorities[dstFile]); createLinks(state, srcFile, dstFile, priority); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index def2c80b2..47d6d5541 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,5 +1,4 @@ #include "daemon.hh" -#include "monitor-fd.hh" #include "signals.hh" #include "worker-protocol.hh" #include "worker-protocol-impl.hh" @@ -16,6 +15,10 @@ #include "args.hh" #include "git.hh" +#ifndef _WIN32 // TODO need graceful async exit support on Windows? +# include "monitor-fd.hh" +#endif + namespace nix::daemon { Sink & operator << (Sink & sink, const Logger::Fields & fields) @@ -1018,7 +1021,9 @@ void processConnection( TrustedFlag trusted, RecursiveFlag recursive) { +#ifndef _WIN32 // TODO need graceful async exit support on Windows? auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; +#endif /* Exchange the greeting. */ unsigned int magic = readInt(from); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index df89b5bd1..219b60c44 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -516,10 +516,12 @@ struct curlFileTransfer : public FileTransfer Sync state_; + #ifndef _WIN32 // TODO need graceful async exit support on Windows? /* We can't use a std::condition_variable to wake up the curl thread, because it only monitors file descriptors. So use a pipe instead. */ Pipe wakeupPipe; + #endif std::thread workerThread; @@ -539,8 +541,10 @@ struct curlFileTransfer : public FileTransfer fileTransferSettings.httpConnections.get()); #endif + #ifndef _WIN32 // TODO need graceful async exit support on Windows? wakeupPipe.create(); fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); + #endif workerThread = std::thread([&]() { workerThreadEntry(); }); } @@ -561,15 +565,19 @@ struct curlFileTransfer : public FileTransfer auto state(state_.lock()); state->quit = true; } + #ifndef _WIN32 // TODO need graceful async exit support on Windows? writeFull(wakeupPipe.writeSide.get(), " ", false); + #endif } void workerThreadMain() { /* Cause this thread to be notified on SIGINT. */ + #ifndef _WIN32 // TODO need graceful async exit support on Windows? auto callback = createInterruptCallback([&]() { stopWorkerThread(); }); + #endif #if __linux__ unshareFilesystem(); @@ -607,9 +615,11 @@ struct curlFileTransfer : public FileTransfer /* Wait for activity, including wakeup events. */ int numfds = 0; struct curl_waitfd extraFDs[1]; + #ifndef _WIN32 // TODO need graceful async exit support on Windows? extraFDs[0].fd = wakeupPipe.readSide.get(); extraFDs[0].events = CURL_WAIT_POLLIN; extraFDs[0].revents = 0; + #endif long maxSleepTimeMs = items.empty() ? 10000 : 100; auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() @@ -693,7 +703,9 @@ struct curlFileTransfer : public FileTransfer throw nix::Error("cannot enqueue download request because the download thread is shutting down"); state->incoming.push(item); } + #ifndef _WIN32 // TODO need graceful async exit support on Windows? writeFull(wakeupPipe.writeSide.get(), " "); + #endif } #if ENABLE_S3 diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4229fb4df..ae268e54e 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -9,11 +9,14 @@ #include #include #include -#include -#include #include +#ifndef _WIN32 +# include +# include +#endif + #ifdef __GLIBC__ # include # include @@ -56,7 +59,9 @@ Settings::Settings() , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { +#ifndef _WIN32 buildUsersGroup = isRootUser() ? "nixbld" : ""; +#endif allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); @@ -239,11 +244,15 @@ StringSet Settings::getDefaultExtraPlatforms() bool Settings::isWSL1() { +#if __linux__ struct utsname utsbuf; uname(&utsbuf); // WSL1 uses -Microsoft suffix // WSL2 uses -microsoft-standard suffix return hasSuffix(utsbuf.release, "-Microsoft"); +#else + return false; +#endif } Path Settings::getDefaultSSLCertFile() @@ -341,6 +350,7 @@ void initPlugins() for (const auto & file : pluginFiles) { /* handle is purposefully leaked as there may be state in the DSO needed by the action of the plugin. */ +#ifndef _WIN32 // TODO implement via DLL loading on Windows void *handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) @@ -351,6 +361,9 @@ void initPlugins() void (*nix_plugin_entry)() = (void (*)())dlsym(handle, "nix_plugin_entry"); if (nix_plugin_entry) nix_plugin_entry(); +#else + throw Error("could not dynamically open plugin file '%s'", file); +#endif } } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4bdbe3333..852dba764 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -666,6 +666,7 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; +#ifndef _WIN32 Setting requireDropSupplementaryGroups{this, isRootUser(), "require-drop-supplementary-groups", R"( Following the principle of least privilege, @@ -683,6 +684,7 @@ public: (since `root` usually has permissions to call setgroups) and `false` otherwise. )"}; +#endif #if __linux__ Setting sandboxShmSize{ diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 0a91fce4b..b74a047bf 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -4,9 +4,12 @@ libstore_NAME = libnixstore libstore_DIR := $(d) -libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) ifdef HOST_UNIX - libstore_SOURCES += $(wildcard $(d)/unix/*.cc) + libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/builtins/*.cc $(d)/unix/build/*.cc) +endif +ifdef HOST_WINDOWS + libstore_SOURCES += $(wildcard $(d)/windows/*.cc) endif libstore_LIBS = libutil @@ -55,9 +58,9 @@ libstore_CXXFLAGS += \ ifeq ($(embedded_sandbox_shell),yes) libstore_CXXFLAGS += -DSANDBOX_SHELL=\"__embedded_sandbox_shell__\" -$(d)/build/local-derivation-goal.cc: $(d)/embedded-sandbox-shell.gen.hh +$(d)/unix/build/local-derivation-goal.cc: $(d)/unix/embedded-sandbox-shell.gen.hh -$(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell) +$(d)/unix/embedded-sandbox-shell.gen.hh: $(sandbox_shell) $(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp @mv $@.tmp $@ else @@ -66,11 +69,11 @@ else endif endif -$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh +$(d)/unix/local-store.cc: $(d)/unix/schema.sql.gen.hh $(d)/unix/ca-specific-schema.sql.gen.hh -$(d)/build.cc: +$(d)/unix/build.cc: -clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh +clean-files += $(d)/unix/schema.sql.gen.hh $(d)/unix/ca-specific-schema.sql.gen.hh $(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index cc8ad3d02..cc3f4884f 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,7 +1,6 @@ #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" -#include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" #include "realisation.hh" diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 193c0bf21..b10a72330 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -8,6 +8,7 @@ #include "types.hh" #include "pathlocks.hh" +#include #include diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index b44edfe89..20f1d826c 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -71,11 +71,15 @@ std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPat auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) { - AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC); + AutoCloseFD fd = toDescriptor(open(cacheFile.c_str(), O_RDONLY + #ifndef _WIN32 + | O_CLOEXEC + #endif + )); if (!fd) throw SysError("opening NAR cache file '%s'", cacheFile); - if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset) + if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset) throw SysError("seeking in '%s'", cacheFile); std::string buf(length, 0); diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 30fe73adb..04f458279 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -55,6 +55,9 @@ bool SSHMaster::isMasterRunning() { std::unique_ptr SSHMaster::startCommand( Strings && command, Strings && extraSshArgs) { +#ifdef _WIN32 // TODO re-enable on Windows, once we can start processes. + throw UnimplementedError("cannot yet SSH on windows because spawning processes is not yet implemented"); +#else Path socketPath = startMaster(); Pipe in, out; @@ -105,8 +108,8 @@ std::unique_ptr SSHMaster::startCommand( }, options); - in.readSide = -1; - out.writeSide = -1; + in.readSide = INVALID_DESCRIPTOR; + out.writeSide = INVALID_DESCRIPTOR; // Wait for the SSH connection to be established, // So that we don't overwrite the password prompt with our progress bar. @@ -126,15 +129,18 @@ std::unique_ptr SSHMaster::startCommand( conn->in = std::move(in.writeSide); return conn; +#endif } +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. + Path SSHMaster::startMaster() { if (!useMaster) return ""; auto state(state_.lock()); - if (state->sshMaster != -1) return state->socketPath; + if (state->sshMaster != INVALID_DESCRIPTOR) return state->socketPath; state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; @@ -167,7 +173,7 @@ Path SSHMaster::startMaster() throw SysError("unable to execute '%s'", args.front()); }, options); - out.writeSide = -1; + out.writeSide = INVALID_DESCRIPTOR; std::string reply; try { @@ -182,4 +188,6 @@ Path SSHMaster::startMaster() return state->socketPath; } +#endif + } diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 08bb43dfa..3b1a0827a 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -21,7 +21,9 @@ private: struct State { +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. Pid sshMaster; +#endif std::unique_ptr tmpDir; Path socketPath; }; @@ -31,13 +33,19 @@ private: void addCommonSSHOpts(Strings & args); bool isMasterRunning(); +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. + Path startMaster(); +#endif + public: SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1); struct Connection { +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. Pid sshPid; +#endif AutoCloseFD out, in; }; @@ -51,8 +59,6 @@ public: std::unique_ptr startCommand( Strings && command, Strings && extraSshArgs = {}); - - Path startMaster(); }; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 79beeebbd..118e5de9f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -13,7 +13,6 @@ #include "archive.hh" #include "callback.hh" #include "git.hh" -#include "remote-store.hh" #include "posix-source-accessor.hh" // FIXME this should not be here, see TODO below on // `addMultipleToStore`. @@ -21,6 +20,10 @@ #include "signals.hh" #include "users.hh" +#ifndef _WIN32 +# include "remote-store.hh" +#endif + #include #include @@ -1266,9 +1269,10 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath) } - -#include "local-store.hh" -#include "uds-remote-store.hh" +#ifndef _WIN32 +# include "local-store.hh" +# include "uds-remote-store.hh" +#endif namespace nix { @@ -1286,6 +1290,9 @@ std::pair splitUriAndParams(const std::string & uri_ return {uri, params}; } +#ifdef _WIN32 // Unused on Windows because the next `#ifndef` +[[maybe_unused]] +#endif static bool isNonUriPath(const std::string & spec) { return @@ -1298,6 +1305,9 @@ static bool isNonUriPath(const std::string & spec) std::shared_ptr openFromNonUri(const std::string & uri, const Store::Params & params) { + // TODO reenable on Windows once we have `LocalStore` and + // `UDSRemoteStore`. + #ifndef _WIN32 if (uri == "" || uri == "auto") { auto stateDir = getOr(params, "state", settings.nixStateDir); if (access(stateDir.c_str(), R_OK | W_OK) == 0) @@ -1342,6 +1352,9 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para } else { return nullptr; } + #else + return nullptr; + #endif } // The `parseURL` function supports both IPv6 URIs as defined in diff --git a/src/libstore/build/child.cc b/src/libstore/unix/build/child.cc similarity index 100% rename from src/libstore/build/child.cc rename to src/libstore/unix/build/child.cc diff --git a/src/libstore/build/child.hh b/src/libstore/unix/build/child.hh similarity index 100% rename from src/libstore/build/child.hh rename to src/libstore/unix/build/child.hh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/unix/build/derivation-goal.cc similarity index 100% rename from src/libstore/build/derivation-goal.cc rename to src/libstore/unix/build/derivation-goal.cc diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/unix/build/derivation-goal.hh similarity index 100% rename from src/libstore/build/derivation-goal.hh rename to src/libstore/unix/build/derivation-goal.hh diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/unix/build/drv-output-substitution-goal.cc similarity index 100% rename from src/libstore/build/drv-output-substitution-goal.cc rename to src/libstore/unix/build/drv-output-substitution-goal.cc diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/unix/build/drv-output-substitution-goal.hh similarity index 100% rename from src/libstore/build/drv-output-substitution-goal.hh rename to src/libstore/unix/build/drv-output-substitution-goal.hh diff --git a/src/libstore/build/entry-points.cc b/src/libstore/unix/build/entry-points.cc similarity index 100% rename from src/libstore/build/entry-points.cc rename to src/libstore/unix/build/entry-points.cc diff --git a/src/libstore/build/goal.cc b/src/libstore/unix/build/goal.cc similarity index 100% rename from src/libstore/build/goal.cc rename to src/libstore/unix/build/goal.cc diff --git a/src/libstore/build/goal.hh b/src/libstore/unix/build/goal.hh similarity index 100% rename from src/libstore/build/goal.hh rename to src/libstore/unix/build/goal.hh diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc similarity index 100% rename from src/libstore/build/hook-instance.cc rename to src/libstore/unix/build/hook-instance.cc diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/unix/build/hook-instance.hh similarity index 100% rename from src/libstore/build/hook-instance.hh rename to src/libstore/unix/build/hook-instance.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc similarity index 100% rename from src/libstore/build/local-derivation-goal.cc rename to src/libstore/unix/build/local-derivation-goal.cc diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/unix/build/local-derivation-goal.hh similarity index 100% rename from src/libstore/build/local-derivation-goal.hh rename to src/libstore/unix/build/local-derivation-goal.hh diff --git a/src/libstore/build/personality.cc b/src/libstore/unix/build/personality.cc similarity index 100% rename from src/libstore/build/personality.cc rename to src/libstore/unix/build/personality.cc diff --git a/src/libstore/build/personality.hh b/src/libstore/unix/build/personality.hh similarity index 100% rename from src/libstore/build/personality.hh rename to src/libstore/unix/build/personality.hh diff --git a/src/libstore/build/sandbox-defaults.sb b/src/libstore/unix/build/sandbox-defaults.sb similarity index 100% rename from src/libstore/build/sandbox-defaults.sb rename to src/libstore/unix/build/sandbox-defaults.sb diff --git a/src/libstore/build/sandbox-minimal.sb b/src/libstore/unix/build/sandbox-minimal.sb similarity index 100% rename from src/libstore/build/sandbox-minimal.sb rename to src/libstore/unix/build/sandbox-minimal.sb diff --git a/src/libstore/build/sandbox-network.sb b/src/libstore/unix/build/sandbox-network.sb similarity index 100% rename from src/libstore/build/sandbox-network.sb rename to src/libstore/unix/build/sandbox-network.sb diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/unix/build/substitution-goal.cc similarity index 100% rename from src/libstore/build/substitution-goal.cc rename to src/libstore/unix/build/substitution-goal.cc diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/unix/build/substitution-goal.hh similarity index 100% rename from src/libstore/build/substitution-goal.hh rename to src/libstore/unix/build/substitution-goal.hh diff --git a/src/libstore/build/worker.cc b/src/libstore/unix/build/worker.cc similarity index 100% rename from src/libstore/build/worker.cc rename to src/libstore/unix/build/worker.cc diff --git a/src/libstore/build/worker.hh b/src/libstore/unix/build/worker.hh similarity index 100% rename from src/libstore/build/worker.hh rename to src/libstore/unix/build/worker.hh diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/unix/builtins/fetchurl.cc similarity index 100% rename from src/libstore/builtins/fetchurl.cc rename to src/libstore/unix/builtins/fetchurl.cc diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/unix/builtins/unpack-channel.cc similarity index 100% rename from src/libstore/builtins/unpack-channel.cc rename to src/libstore/unix/builtins/unpack-channel.cc diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/unix/ca-specific-schema.sql similarity index 100% rename from src/libstore/ca-specific-schema.sql rename to src/libstore/unix/ca-specific-schema.sql diff --git a/src/libstore/gc.cc b/src/libstore/unix/gc.cc similarity index 100% rename from src/libstore/gc.cc rename to src/libstore/unix/gc.cc diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/unix/local-overlay-store.cc similarity index 100% rename from src/libstore/local-overlay-store.cc rename to src/libstore/unix/local-overlay-store.cc diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/unix/local-overlay-store.hh similarity index 100% rename from src/libstore/local-overlay-store.hh rename to src/libstore/unix/local-overlay-store.hh diff --git a/src/libstore/local-overlay-store.md b/src/libstore/unix/local-overlay-store.md similarity index 100% rename from src/libstore/local-overlay-store.md rename to src/libstore/unix/local-overlay-store.md diff --git a/src/libstore/local-store.cc b/src/libstore/unix/local-store.cc similarity index 100% rename from src/libstore/local-store.cc rename to src/libstore/unix/local-store.cc diff --git a/src/libstore/local-store.hh b/src/libstore/unix/local-store.hh similarity index 100% rename from src/libstore/local-store.hh rename to src/libstore/unix/local-store.hh diff --git a/src/libstore/local-store.md b/src/libstore/unix/local-store.md similarity index 100% rename from src/libstore/local-store.md rename to src/libstore/unix/local-store.md diff --git a/src/libstore/lock.cc b/src/libstore/unix/lock.cc similarity index 100% rename from src/libstore/lock.cc rename to src/libstore/unix/lock.cc diff --git a/src/libstore/lock.hh b/src/libstore/unix/lock.hh similarity index 100% rename from src/libstore/lock.hh rename to src/libstore/unix/lock.hh diff --git a/src/libstore/optimise-store.cc b/src/libstore/unix/optimise-store.cc similarity index 100% rename from src/libstore/optimise-store.cc rename to src/libstore/unix/optimise-store.cc diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/unix/posix-fs-canonicalise.cc similarity index 100% rename from src/libstore/posix-fs-canonicalise.cc rename to src/libstore/unix/posix-fs-canonicalise.cc diff --git a/src/libstore/posix-fs-canonicalise.hh b/src/libstore/unix/posix-fs-canonicalise.hh similarity index 100% rename from src/libstore/posix-fs-canonicalise.hh rename to src/libstore/unix/posix-fs-canonicalise.hh diff --git a/src/libstore/schema.sql b/src/libstore/unix/schema.sql similarity index 100% rename from src/libstore/schema.sql rename to src/libstore/unix/schema.sql diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/unix/uds-remote-store.cc similarity index 100% rename from src/libstore/uds-remote-store.cc rename to src/libstore/unix/uds-remote-store.cc diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/unix/uds-remote-store.hh similarity index 100% rename from src/libstore/uds-remote-store.hh rename to src/libstore/unix/uds-remote-store.hh diff --git a/src/libstore/uds-remote-store.md b/src/libstore/unix/uds-remote-store.md similarity index 100% rename from src/libstore/uds-remote-store.md rename to src/libstore/unix/uds-remote-store.md diff --git a/src/libstore/windows/build.cc b/src/libstore/windows/build.cc new file mode 100644 index 000000000..3eadc5bda --- /dev/null +++ b/src/libstore/windows/build.cc @@ -0,0 +1,37 @@ +#include "store-api.hh" +#include "build-result.hh" + +namespace nix { + +void Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) +{ + unsupported("buildPaths"); +} + +std::vector Store::buildPathsWithResults( + const std::vector & reqs, + BuildMode buildMode, + std::shared_ptr evalStore) +{ + unsupported("buildPathsWithResults"); +} + +BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + unsupported("buildDerivation"); +} + + +void Store::ensurePath(const StorePath & path) +{ + unsupported("ensurePath"); +} + + +void Store::repairPath(const StorePath & path) +{ + unsupported("repairPath"); +} + +} diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 834fc7314..243e3a5a6 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -9,7 +9,9 @@ #include #include #include -#include +#ifndef _WIN32 +# include +#endif namespace nix { @@ -547,6 +549,7 @@ nlohmann::json Args::toJSON() static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs) { completions.setType(Completions::Type::Filenames); + #ifndef _WIN32 // TODO implement globbing completions on Windows glob_t globbuf; int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR @@ -564,6 +567,7 @@ static void _completePath(AddCompletions & completions, std::string_view prefix, } } globfree(&globbuf); + #endif } void Args::completePath(AddCompletions & completions, size_t, std::string_view prefix) diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index d33f7163a..c88013b3c 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -19,7 +19,9 @@ # include "namespaces.hh" #endif -#include +#ifndef _WIN32 +# include +#endif namespace nix { @@ -57,6 +59,7 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// +#ifndef _WIN32 rlim_t savedStackSize = 0; void setStackSize(rlim_t stackSize) @@ -79,16 +82,20 @@ void setStackSize(rlim_t stackSize) } } } +#endif void restoreProcessContext(bool restoreMounts) { + #ifndef _WIN32 unix::restoreSignals(); + #endif if (restoreMounts) { #if __linux__ restoreMountNamespace(); #endif } + #ifndef _WIN32 if (savedStackSize) { struct rlimit limit; if (getrlimit(RLIMIT_STACK, &limit) == 0) { @@ -96,6 +103,7 @@ void restoreProcessContext(bool restoreMounts) setrlimit(RLIMIT_STACK, &limit); } } + #endif } diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh index 444c717d1..a5adb70cf 100644 --- a/src/libutil/current-process.hh +++ b/src/libutil/current-process.hh @@ -2,7 +2,10 @@ ///@file #include -#include + +#ifndef _WIN32 +# include +#endif #include "types.hh" @@ -14,16 +17,18 @@ namespace nix { */ unsigned int getMaxCPU(); +#ifndef _WIN32 // TODO implement on Windows, if needed. /** * Change the stack size. */ void setStackSize(rlim_t stackSize); +#endif /** * Restore the original inherited Unix process context (such as signal * masks, stack size). - * See startSignalHandlerThread(), saveSignalMask(). + * See unix::startSignalHandlerThread(), unix::saveSignalMask(). */ void restoreProcessContext(bool restoreMounts = true); diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh index 21c2356a4..e0649adac 100644 --- a/src/libutil/environment-variables.hh +++ b/src/libutil/environment-variables.hh @@ -28,6 +28,13 @@ std::optional getEnvNonEmpty(const std::string & key); */ std::map getEnv(); +#ifdef _WIN32 +/** + * Implementation of missing POSIX function. + */ +int unsetenv(const char * name); +#endif + /** * Like POSIX `setenv`, but always overrides. * diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 445b1e19c..0419f36d6 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -247,6 +247,23 @@ public: } }; +#ifdef _WIN32 +class WinError; +#endif + +/** + * Convenience alias for when we use a `errno`-based error handling + * function on Unix, and `GetLastError()`-based error handling on on + * Windows. + */ +using NativeSysError = +#ifdef _WIN32 + WinError +#else + SysError +#endif + ; + /** * Throw an exception for the purpose of checking that exception * handling works; see 'initLibUtil()'. diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 95cbb8537..3bbfc50ee 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -5,6 +5,11 @@ #include #include +#ifdef _WIN32 +# include +# include +# include "windows-error.hh" +#endif namespace nix { @@ -20,7 +25,13 @@ std::string drainFD(Descriptor fd, bool block, const size_t reserveSize) // the parser needs two extra bytes to append terminating characters, other users will // not care very much about the extra memory. StringSink sink(reserveSize + 2); +#ifdef _WIN32 + // non-blocking is not supported this way on Windows + assert(block); + drainFD(fd, sink); +#else drainFD(fd, sink, block); +#endif return std::move(sink.s); } @@ -68,9 +79,15 @@ Descriptor AutoCloseFD::get() const void AutoCloseFD::close() { if (fd != INVALID_DESCRIPTOR) { - if(::close(fd) == -1) + if( +#ifdef _WIN32 + ::CloseHandle(fd) +#else + ::close(fd) +#endif + == -1) /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); + throw NativeSysError("closing file descriptor %1%", fd); fd = INVALID_DESCRIPTOR; } } @@ -80,14 +97,16 @@ void AutoCloseFD::fsync() if (fd != INVALID_DESCRIPTOR) { int result; result = -#if __APPLE__ +#ifdef _WIN32 + ::FlushFileBuffers(fd) +#elif __APPLE__ ::fcntl(fd, F_FULLFSYNC) #else ::fsync(fd) #endif ; if (result == -1) - throw SysError("fsync file descriptor %1%", fd); + throw NativeSysError("fsync file descriptor %1%", fd); } } diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index 719e1e444..50201d846 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -4,6 +4,11 @@ #include "types.hh" #include "error.hh" +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# include +#endif + namespace nix { struct Sink; @@ -12,9 +17,21 @@ struct Source; /** * Operating System capability */ -typedef int Descriptor; +typedef +#if _WIN32 + HANDLE +#else + int +#endif + Descriptor; -const Descriptor INVALID_DESCRIPTOR = -1; +const Descriptor INVALID_DESCRIPTOR = +#if _WIN32 + INVALID_HANDLE_VALUE +#else + -1 +#endif + ; /** * Convert a native `Descriptor` to a POSIX file descriptor @@ -23,17 +40,26 @@ const Descriptor INVALID_DESCRIPTOR = -1; */ static inline Descriptor toDescriptor(int fd) { +#ifdef _WIN32 + return (HANDLE) _get_osfhandle(fd); +#else return fd; +#endif } /** - * Convert a POSIX file descriptor to a native `Descriptor` + * Convert a POSIX file descriptor to a native `Descriptor` in read-only + * mode. * * This is a no-op except on Windows. */ -static inline int fromDescriptor(Descriptor fd, int flags) +static inline int fromDescriptorReadOnly(Descriptor fd) { +#ifdef _WIN32 + return _open_osfhandle((intptr_t) fd, _O_RDONLY); +#else return fd; +#endif } /** @@ -64,11 +90,24 @@ void writeLine(Descriptor fd, std::string s); */ std::string drainFD(Descriptor fd, bool block = true, const size_t reserveSize=0); -void drainFD(Descriptor fd, Sink & sink, bool block = true); +/** + * The Windows version is always blocking. + */ +void drainFD( + Descriptor fd + , Sink & sink +#ifndef _WIN32 + , bool block = true +#endif + ); [[gnu::always_inline]] inline Descriptor getStandardOut() { +#ifndef _WIN32 return STDOUT_FILENO; +#else + return GetStdHandle(STD_OUTPUT_HANDLE); +#endif } /** @@ -100,6 +139,8 @@ public: void close(); }; +#ifndef _WIN32 // Not needed on Windows, where we don't fork + /** * Close all file descriptors except those listed in the given set. * Good practice in child processes. @@ -111,6 +152,15 @@ void closeMostFDs(const std::set & exceptions); */ void closeOnExec(Descriptor fd); +#endif + +#ifdef _WIN32 +# if _WIN32_WINNT >= 0x0600 +Path handleToPath(Descriptor handle); +std::wstring handleToFileName(Descriptor handle); +# endif +#endif + MakeError(EndOfFile, Error); } diff --git a/src/libutil/file-path.hh b/src/libutil/file-path.hh new file mode 100644 index 000000000..6fb100125 --- /dev/null +++ b/src/libutil/file-path.hh @@ -0,0 +1,52 @@ +#pragma once +///@file + +#include +#include + +#include "types.hh" + +namespace nix { + +/** + * Paths are just `std::filesystem::path`s. + * + * @todo drop `NG` suffix and replace the ones in `types.hh`. + */ +typedef std::filesystem::path PathNG; +typedef std::list PathsNG; +typedef std::set PathSetNG; + +/** + * Stop gap until `std::filesystem::path_view` from P1030R6 exists in a + * future C++ standard. + * + * @todo drop `NG` suffix and replace the one in `types.hh`. + */ +struct PathViewNG : std::basic_string_view +{ + using string_view = std::basic_string_view; + + using string_view::string_view; + + PathViewNG(const PathNG & path) + : std::basic_string_view(path.native()) + { } + + PathViewNG(const PathNG::string_type & path) + : std::basic_string_view(path) + { } + + const string_view & native() const { return *this; } + string_view & native() { return *this; } +}; + +std::string os_string_to_string(PathViewNG::string_view path); + +PathNG::string_type string_to_os_string(std::string_view s); + +std::optional maybePathNG(PathView path); + +PathNG pathNG(PathView path); + +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 89d309731..b03bb767b 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -1,5 +1,6 @@ #include "environment-variables.hh" #include "file-system.hh" +#include "file-path.hh" #include "file-path-impl.hh" #include "signals.hh" #include "finally.hh" @@ -18,6 +19,10 @@ #include #include +#ifdef _WIN32 +# include +#endif + namespace fs = std::filesystem; namespace nix { @@ -128,10 +133,10 @@ std::string_view baseNameOf(std::string_view path) return ""; auto last = path.size() - 1; - while (last > 0 && path[last] == '/') + while (last > 0 && NativePathTrait::isPathSep(path[last])) last -= 1; - auto pos = path.rfind('/', last); + auto pos = NativePathTrait::rfindPathSep(path, last); if (pos == path.npos) pos = 0; else @@ -164,11 +169,16 @@ struct stat stat(const Path & path) return st; } +#ifdef _WIN32 +# define STAT stat +#else +# define STAT lstat +#endif struct stat lstat(const Path & path) { struct stat st; - if (lstat(path.c_str(), &st)) + if (STAT(path.c_str(), &st)) throw SysError("getting status of '%1%'", path); return st; } @@ -177,7 +187,7 @@ struct stat lstat(const Path & path) std::optional maybeLstat(const Path & path) { std::optional st{std::in_place}; - if (lstat(path.c_str(), &*st)) + if (STAT(path.c_str(), &*st)) { if (errno == ENOENT || errno == ENOTDIR) st.reset(); @@ -207,6 +217,7 @@ bool pathAccessible(const Path & path) Path readLink(const Path & path) { +#ifndef _WIN32 checkInterrupt(); std::vector buf; for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { @@ -220,13 +231,16 @@ Path readLink(const Path & path) else if (rlSize < bufSize) return std::string(buf.data(), rlSize); } +#else + // TODO modern Windows does in fact support symlinks + throw UnimplementedError("reading symbolic link '%1%'", path); +#endif } bool isLink(const Path & path) { - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); + return getFileType(path) == DT_LNK; } @@ -274,7 +288,12 @@ unsigned char getFileType(const Path & path) std::string readFile(const Path & path) { - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY +// TODO +#ifndef _WIN32 + | O_CLOEXEC +#endif + )); if (!fd) throw SysError("opening file '%1%'", path); return readFile(fd.get()); @@ -283,7 +302,12 @@ std::string readFile(const Path & path) void readFile(const Path & path, Sink & sink) { - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY +// TODO +#ifndef _WIN32 + | O_CLOEXEC +#endif + )); if (!fd) throw SysError("opening file '%s'", path); drainFD(fd.get(), sink); @@ -292,7 +316,12 @@ void readFile(const Path & path, Sink & sink) void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) { - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT +// TODO +#ifndef _WIN32 + | O_CLOEXEC +#endif + , mode)); if (!fd) throw SysError("opening file '%1%'", path); try { @@ -312,7 +341,12 @@ void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) void writeFile(const Path & path, Source & source, mode_t mode, bool sync) { - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT +// TODO +#ifndef _WIN32 + | O_CLOEXEC +#endif + , mode)); if (!fd) throw SysError("opening file '%1%'", path); @@ -339,21 +373,23 @@ void writeFile(const Path & path, Source & source, mode_t mode, bool sync) void syncParent(const Path & path) { - AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); + AutoCloseFD fd = toDescriptor(open(dirOf(path).c_str(), O_RDONLY, 0)); if (!fd) throw SysError("opening file '%1%'", path); fd.fsync(); } -static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) +static void _deletePath(Descriptor parentfd, const Path & path, uint64_t & bytesFreed) { +#ifndef _WIN32 checkInterrupt(); std::string name(baseNameOf(path)); struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (fstatat(parentfd, name.c_str(), &st, + AT_SYMLINK_NOFOLLOW) == -1) { if (errno == ENOENT) return; throw SysError("getting status of '%1%'", path); } @@ -405,6 +441,10 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) if (errno == ENOENT) return; throw SysError("cannot unlink '%1%'", path); } +#else + // TODO implement + throw UnimplementedError("_deletePath"); +#endif } static void _deletePath(const Path & path, uint64_t & bytesFreed) @@ -413,7 +453,7 @@ static void _deletePath(const Path & path, uint64_t & bytesFreed) if (dir == "") dir = "/"; - AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; + AutoCloseFD dirfd = toDescriptor(open(dir.c_str(), O_RDONLY)); if (!dirfd) { if (errno == ENOENT) return; throw SysError("opening directory '%1%'", path); @@ -436,11 +476,15 @@ Paths createDirs(const Path & path) if (path == "/") return created; struct stat st; - if (lstat(path.c_str(), &st) == -1) { + if (STAT(path.c_str(), &st) == -1) { created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + if (mkdir(path.c_str() +#ifndef _WIN32 // TODO abstract mkdir perms for Windows + , 0777 +#endif + ) == -1 && errno != EEXIST) throw SysError("creating directory '%1%'", path); - st = lstat(path); + st = STAT(path); created.push_back(path); } @@ -526,7 +570,11 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, while (1) { checkInterrupt(); Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { + if (mkdir(tmpDir.c_str() +#ifndef _WIN32 // TODO abstract mkdir perms for Windows + , mode +#endif + ) == 0) { #if __FreeBSD__ /* Explicitly set the group of the directory. This is to work around around problems caused by BSD's group @@ -552,17 +600,24 @@ std::pair createTempFile(const Path & prefix) Path tmpl(defaultTempDir() + "/" + prefix + ".XXXXXX"); // Strictly speaking, this is UB, but who cares... // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + AutoCloseFD fd = toDescriptor(mkstemp((char *) tmpl.c_str())); if (!fd) throw SysError("creating temporary file '%s'", tmpl); +#ifndef _WIN32 closeOnExec(fd.get()); +#endif return {std::move(fd), tmpl}; } void createSymlink(const Path & target, const Path & link) { +#ifndef _WIN32 if (symlink(target.c_str(), link.c_str())) throw SysError("creating symlink from '%1%' to '%2%'", link, target); +#else + // TODO modern Windows does in fact support symlinks + throw UnimplementedError("createSymlink"); +#endif } void replaceSymlink(const Path & target, const Path & link) @@ -583,7 +638,8 @@ void replaceSymlink(const Path & target, const Path & link) } } -void setWriteTime(const fs::path & p, const struct stat & st) +#ifndef _WIN32 +static void setWriteTime(const fs::path & p, const struct stat & st) { struct timeval times[2]; times[0] = { @@ -597,11 +653,14 @@ void setWriteTime(const fs::path & p, const struct stat & st) if (lutimes(p.c_str(), times) != 0) throw SysError("changing modification time of '%s'", p); } +#endif void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) { +#ifndef _WIN32 // TODO: Rewrite the `is_*` to use `symlink_status()` auto statOfFrom = lstat(from.path().c_str()); +#endif auto fromStatus = from.symlink_status(); // Mark the directory as writable so that we can delete its children @@ -621,7 +680,9 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) throw Error("file '%s' has an unsupported type", from.path()); } +#ifndef _WIN32 setWriteTime(to, statOfFrom); +#endif if (andDelete) { if (!fs::is_symlink(fromStatus)) fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); @@ -648,14 +709,18 @@ void moveFile(const Path & oldName, const Path & newName) auto newPath = fs::path(newName); // For the move to be as atomic as possible, copy to a temporary // directory - fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + fs::path temp = createTempDir( + os_string_to_string(PathViewNG { newPath.parent_path() }), + "rename-tmp"); Finally removeTemp = [&]() { fs::remove(temp); }; auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { fs::remove(newPath); warn("Can’t rename %s as %s, copying instead", oldName, newName); copy(fs::directory_entry(oldPath), tempCopyTarget, true); - renameFile(tempCopyTarget, newPath); + renameFile( + os_string_to_string(PathViewNG { tempCopyTarget }), + os_string_to_string(PathViewNG { newPath })); } } } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 06a993829..0c4e7cfdd 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -14,6 +14,9 @@ #include #include #include +#ifdef _WIN32 +# include +#endif #include #include @@ -31,6 +34,17 @@ #define DT_DIR 3 #endif +/** + * Polyfill for MinGW + * + * Windows does in fact support symlinks, but the C runtime interfaces predate this. + * + * @todo get rid of this, and stop using `stat` when we want `lstat` too. + */ +#ifndef S_ISLNK +# define S_ISLNK(m) false +#endif + namespace nix { struct Sink; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 35ce0ac36..91070ea89 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -1,8 +1,15 @@ #include +#include "error.hh" #include "config.hh" #include "fs-sink.hh" +#if _WIN32 +# include +# include "file-path.hh" +# include "windows-error.hh" +#endif + namespace nix { void copyRecursive( @@ -65,8 +72,14 @@ static GlobalConfig::Register r1(&restoreSinkSettings); void RestoreSink::createDirectory(const Path & path) { Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); + if ( +#ifndef _WIN32 // TODO abstract mkdir perms for Windows + mkdir(p.c_str(), 0777) == -1 +#else + !CreateDirectoryW(pathNG(p).c_str(), NULL) +#endif + ) + throw NativeSysError("creating directory '%1%'", p); }; struct RestoreRegularFile : CreateRegularFileSink { @@ -81,18 +94,28 @@ void RestoreSink::createRegularFile(const Path & path, std::function nextId{0}; +static uint64_t getPid() +{ +#ifndef _WIN32 + return getpid(); +#else + return GetCurrentProcessId(); +#endif +} + Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s, const Logger::Fields & fields, ActivityId parent) - : logger(logger), id(nextId++ + (((uint64_t) getpid()) << 32)) + : logger(logger), id(nextId++ + (((uint64_t) getPid()) << 32)) { logger.startActivity(id, lvl, type, s, fields, parent); } diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 8039d4b80..a589bfd3d 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -10,7 +10,7 @@ PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && root) : root(std::move(root)) { assert(root.empty() || root.is_absolute()); - displayPrefix = root; + displayPrefix = root.string(); } PosixSourceAccessor::PosixSourceAccessor() @@ -19,10 +19,10 @@ PosixSourceAccessor::PosixSourceAccessor() std::pair PosixSourceAccessor::createAtRoot(const std::filesystem::path & path) { - std::filesystem::path path2 = absPath(path.native()); + std::filesystem::path path2 = absPath(path.string()); return { PosixSourceAccessor { path2.root_path() }, - CanonPath { static_cast(path2.relative_path()) }, + CanonPath { path2.relative_path().string() }, }; } @@ -47,12 +47,16 @@ void PosixSourceAccessor::readFile( auto ap = makeAbsPath(path); - AutoCloseFD fd = open(ap.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + AutoCloseFD fd = toDescriptor(open(ap.string().c_str(), O_RDONLY + #ifndef _WIN32 + | O_NOFOLLOW | O_CLOEXEC + #endif + )); if (!fd) - throw SysError("opening file '%1%'", ap.native()); + throw SysError("opening file '%1%'", ap.string()); struct stat st; - if (fstat(fd.get(), &st) == -1) + if (fstat(fromDescriptorReadOnly(fd.get()), &st) == -1) throw SysError("statting file"); sizeCallback(st.st_size); @@ -62,7 +66,7 @@ void PosixSourceAccessor::readFile( std::array buf; while (left) { checkInterrupt(); - ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + ssize_t rd = read(fromDescriptorReadOnly(fd.get()), buf.data(), (size_t) std::min(left, (off_t) buf.size())); if (rd == -1) { if (errno != EINTR) throw SysError("reading from file '%s'", showPath(path)); @@ -80,7 +84,7 @@ void PosixSourceAccessor::readFile( bool PosixSourceAccessor::pathExists(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); - return nix::pathExists(makeAbsPath(path)); + return nix::pathExists(makeAbsPath(path).string()); } std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) @@ -89,7 +93,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa // Note: we convert std::filesystem::path to Path because the // former is not hashable on libc++. - Path absPath = makeAbsPath(path); + Path absPath = makeAbsPath(path).string(); { auto cache(_cache.lock()); @@ -127,11 +131,13 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & { assertNoSymlinks(path); DirEntries res; - for (auto & entry : nix::readDirectory(makeAbsPath(path))) { + for (auto & entry : nix::readDirectory(makeAbsPath(path).string())) { std::optional type; switch (entry.type) { case DT_REG: type = Type::tRegular; break; + #ifndef _WIN32 case DT_LNK: type = Type::tSymlink; break; + #endif case DT_DIR: type = Type::tDirectory; break; } res.emplace(entry.name, type); @@ -142,7 +148,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & std::string PosixSourceAccessor::readLink(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); - return nix::readLink(makeAbsPath(path)); + return nix::readLink(makeAbsPath(path).string()); } std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) diff --git a/src/libutil/unix/processes.hh b/src/libutil/processes.hh similarity index 95% rename from src/libutil/unix/processes.hh rename to src/libutil/processes.hh index 978c37105..a7e85b5be 100644 --- a/src/libutil/unix/processes.hh +++ b/src/libutil/processes.hh @@ -25,6 +25,7 @@ namespace nix { struct Sink; struct Source; +#ifndef _WIN32 class Pid { pid_t pid = -1; @@ -43,13 +44,16 @@ public: void setKillSignal(int signal); pid_t release(); }; +#endif +#ifndef _WIN32 /** * Kill all processes running under the specified uid by sending them * a SIGKILL. */ void killUser(uid_t uid); +#endif /** @@ -68,8 +72,9 @@ struct ProcessOptions int cloneFlags = 0; }; +#ifndef _WIN32 pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); - +#endif /** * Run a program and return its stdout in a string (i.e., like the @@ -84,8 +89,10 @@ struct RunOptions Path program; bool searchPath = true; Strings args; +#ifndef _WIN32 std::optional uid; std::optional gid; +#endif std::optional chdir; std::optional> environment; std::optional input; @@ -111,6 +118,7 @@ public: { } }; +#ifndef _WIN32 /** * Convert the exit status of a child as returned by wait() into an @@ -120,4 +128,6 @@ std::string statusToString(int status); bool statusOk(int status); +#endif + } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 70c16ff0d..5ea27ccbe 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -7,6 +7,11 @@ #include +#ifdef _WIN32 +# include +# include "windows-error.hh" +#endif + namespace nix { @@ -126,6 +131,14 @@ bool BufferedSource::hasData() size_t FdSource::readUnbuffered(char * data, size_t len) { +#ifdef _WIN32 + DWORD n; + checkInterrupt(); + if (!::ReadFile(fd, data, len, &n, NULL)) { + _good = false; + throw WinError("ReadFile when FdSource::readUnbuffered"); + } +#else ssize_t n; do { checkInterrupt(); @@ -133,6 +146,7 @@ size_t FdSource::readUnbuffered(char * data, size_t len) } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile(std::string(*endOfFileError)); } +#endif read += n; return n; } diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 096252f03..4dc280f8c 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -2,7 +2,12 @@ #include "environment-variables.hh" #include "sync.hh" -#include +#if _WIN32 +# include +# define isatty _isatty +#else +# include +#endif #include namespace nix { @@ -92,6 +97,7 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w static Sync> windowSize{{0, 0}}; +#ifndef _WIN32 void updateWindowSize() { struct winsize ws; @@ -101,6 +107,7 @@ void updateWindowSize() windowSize_->second = ws.ws_col; } } +#endif std::pair getWindowSize() diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh index 9d8d0c743..628833283 100644 --- a/src/libutil/terminal.hh +++ b/src/libutil/terminal.hh @@ -21,12 +21,16 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll = false, unsigned int width = std::numeric_limits::max()); +#ifndef _WIN32 + /** * Recalculate the window size, updating a global variable. Used in the * `SIGWINCH` signal handler. */ void updateWindowSize(); +#endif + /** * @return the number of rows and columns of the terminal. * diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 805f31d80..0f6349642 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -81,8 +81,10 @@ void ThreadPool::doWork(bool mainThread) { ReceiveInterrupts receiveInterrupts; +#ifndef _WIN32 // Does Windows need anything similar for async exit handling? if (!mainThread) unix::interruptCheck = [&]() { return (bool) quit; }; +#endif bool didWork = false; std::exception_ptr exc; diff --git a/src/libutil/unix/file-path.cc b/src/libutil/unix/file-path.cc new file mode 100644 index 000000000..54a1cc278 --- /dev/null +++ b/src/libutil/unix/file-path.cc @@ -0,0 +1,31 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + return std::string { path }; +} + +PathNG::string_type string_to_os_string(std::string_view s) +{ + return std::string { s }; +} + +std::optional maybePathNG(PathView path) +{ + return { path }; +} + +PathNG pathNG(PathView path) +{ + return path; +} + +} diff --git a/src/libutil/users.hh b/src/libutil/users.hh index 449e5bbe9..153cc73fd 100644 --- a/src/libutil/users.hh +++ b/src/libutil/users.hh @@ -3,16 +3,20 @@ #include "types.hh" -#include +#ifndef _WIN32 +# include +#endif namespace nix { std::string getUserName(); +#ifndef _WIN32 /** * @return the given user's home directory from /etc/passwd. */ Path getHomeOf(uid_t userId); +#endif /** * @return $HOME or the user's home directory from /etc/passwd. @@ -58,6 +62,8 @@ std::string expandTilde(std::string_view path); /** * Is the current user UID 0 on Unix? + * + * Currently always false on Windows, but that may change. */ bool isRootUser(); diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc new file mode 100644 index 000000000..25ab9d63a --- /dev/null +++ b/src/libutil/windows/environment-variables.cc @@ -0,0 +1,17 @@ +#include "environment-variables.hh" + +#include "processenv.h" + +namespace nix { + +int unsetenv(const char *name) +{ + return -SetEnvironmentVariableA(name, nullptr); +} + +int setEnv(const char * name, const char * value) +{ + return -SetEnvironmentVariableA(name, value); +} + +} diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc new file mode 100644 index 000000000..26f769b66 --- /dev/null +++ b/src/libutil/windows/file-descriptor.cc @@ -0,0 +1,148 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" +#include "windows-error.hh" +#include "file-path.hh" + +#include +#include +#include +#include +#define WIN32_LEAN_AND_MEAN +#include + +namespace nix { + +std::string readFile(HANDLE handle) +{ + LARGE_INTEGER li; + if (!GetFileSizeEx(handle, &li)) + throw WinError("%s:%d statting file", __FILE__, __LINE__); + + return drainFD(handle, true, li.QuadPart); +} + + +void readFull(HANDLE handle, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + DWORD res; + if (!ReadFile(handle, (char *) buf, count, &res, NULL)) + throw WinError("%s:%d reading from file", __FILE__, __LINE__); + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(HANDLE handle, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + DWORD res; +#if _WIN32_WINNT >= 0x0600 + auto path = handleToPath(handle); // debug; do it before becuase handleToPath changes lasterror + if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) { + throw WinError("writing to file %1%:%2%", handle, path); + } +#else + if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) { + throw WinError("writing to file %1%", handle); + } +#endif + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string readLine(HANDLE handle) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + DWORD rd; + if (!ReadFile(handle, &ch, 1, &rd, NULL)) { + throw WinError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void drainFD(HANDLE handle, Sink & sink/*, bool block*/) +{ + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + DWORD rd; + if (!ReadFile(handle, buf.data(), buf.size(), &rd, NULL)) { + WinError winError("%s:%d reading from handle %p", __FILE__, __LINE__, handle); + if (winError.lastError == ERROR_BROKEN_PIPE) + break; + throw winError; + } + else if (rd == 0) break; + sink({(char *) buf.data(), (size_t) rd}); + } +} + + +////////////////////////////////////////////////////////////////////// + + +void Pipe::create() +{ + SECURITY_ATTRIBUTES saAttr = {0}; + saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + saAttr.lpSecurityDescriptor = NULL; + saAttr.bInheritHandle = TRUE; + + HANDLE hReadPipe, hWritePipe; + if (!CreatePipe(&hReadPipe, &hWritePipe, &saAttr, 0)) + throw WinError("CreatePipe"); + + readSide = hReadPipe; + writeSide = hWritePipe; +} + + +////////////////////////////////////////////////////////////////////// + +#if _WIN32_WINNT >= 0x0600 + +std::wstring handleToFileName(HANDLE handle) { + std::vector buf(0x100); + DWORD dw = GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED); + if (dw == 0) { + if (handle == GetStdHandle(STD_INPUT_HANDLE )) return L""; + if (handle == GetStdHandle(STD_OUTPUT_HANDLE)) return L""; + if (handle == GetStdHandle(STD_ERROR_HANDLE )) return L""; + return (boost::wformat(L"") % handle).str(); + } + if (dw > buf.size()) { + buf.resize(dw); + if (GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED) != dw-1) + throw WinError("GetFinalPathNameByHandleW"); + dw -= 1; + } + return std::wstring(buf.data(), dw); +} + + +Path handleToPath(HANDLE handle) { + return os_string_to_string(handleToFileName(handle)); +} + +#endif + +} diff --git a/src/libutil/windows/file-path.cc b/src/libutil/windows/file-path.cc new file mode 100644 index 000000000..d2f385f50 --- /dev/null +++ b/src/libutil/windows/file-path.cc @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "file-path-impl.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + std::wstring_convert> converter; + return converter.to_bytes(PathNG::string_type { path }); +} + +PathNG::string_type string_to_os_string(std::string_view s) +{ + std::wstring_convert> converter; + return converter.from_bytes(std::string { s }); +} + +std::optional maybePathNG(PathView path) +{ + if (path.length() >= 3 && (('A' <= path[0] && path[0] <= 'Z') || ('a' <= path[0] && path[0] <= 'z')) && path[1] == ':' && WindowsPathTrait::isPathSep(path[2])) { + PathNG::string_type sw = string_to_os_string( + std::string { "\\\\?\\" } + path); + std::replace(sw.begin(), sw.end(), '/', '\\'); + return sw; + } + if (path.length() >= 7 && path[0] == '\\' && path[1] == '\\' && (path[2] == '.' || path[2] == '?') && path[3] == '\\' && + ('A' <= path[4] && path[4] <= 'Z') && path[5] == ':' && WindowsPathTrait::isPathSep(path[6])) { + PathNG::string_type sw = string_to_os_string(path); + std::replace(sw.begin(), sw.end(), '/', '\\'); + return sw; + } + return std::optional(); +} + +PathNG pathNG(PathView path) +{ + std::optional sw = maybePathNG(path); + if (!sw) { + // FIXME why are we not using the regular error handling? + std::cerr << "invalid path for WinAPI call ["< +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef __APPLE__ +# include +#endif + +#ifdef __linux__ +# include +# include +#endif + + +namespace nix { + +std::string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional & input, bool isInteractive) +{ + throw UnimplementedError("Cannot shell out to git on Windows yet"); +} + +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + throw UnimplementedError("Cannot shell out to git on Windows yet"); +} + +void runProgram2(const RunOptions & options) +{ + throw UnimplementedError("Cannot shell out to git on Windows yet"); +} + +} diff --git a/src/libutil/windows/signals-impl.hh b/src/libutil/windows/signals-impl.hh new file mode 100644 index 000000000..26d2600bf --- /dev/null +++ b/src/libutil/windows/signals-impl.hh @@ -0,0 +1,41 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/* User interruption. */ + +static inline void setInterrupted(bool isInterrupted) +{ + /* Do nothing for now */ +} + +static inline bool getInterrupted() +{ + return false; +} + +inline void setInterruptThrown() +{ + /* Do nothing for now */ +} + +void inline checkInterrupt() +{ + /* Do nothing for now */ +} + +/** + * Does nothing, unlike Unix counterpart, but allows avoiding C++ + */ +struct ReceiveInterrupts +{ + /** + * Explicit destructor avoids dead code warnings. + */ + ~ReceiveInterrupts() {} +}; + +} diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc new file mode 100644 index 000000000..1792ff1a1 --- /dev/null +++ b/src/libutil/windows/users.cc @@ -0,0 +1,50 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" +#include "windows-error.hh" + +#define WIN32_LEAN_AND_MEAN +#include + +namespace nix { + +std::string getUserName() +{ + // Get the required buffer size + DWORD size = 0; + if (!GetUserNameA(nullptr, &size)) { + auto lastError = GetLastError(); + if (lastError != ERROR_INSUFFICIENT_BUFFER) + throw WinError(lastError, "cannot figure out size of user name"); + } + + std::string name; + // Allocate a buffer of sufficient size + // + // - 1 because no need for null byte + name.resize(size - 1); + + // Retrieve the username + if (!GetUserNameA(&name[0], &size)) + throw WinError("cannot figure out user name"); + + return name; +} + +Path getHome() +{ + static Path homeDir = []() + { + Path homeDir = getEnv("USERPROFILE").value_or("C:\\Users\\Default"); + assert(!homeDir.empty()); + return canonPath(homeDir); + }(); + return homeDir; +} + +bool isRootUser() { + return false; +} + +} diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc new file mode 100644 index 000000000..26faaae6d --- /dev/null +++ b/src/libutil/windows/windows-error.cc @@ -0,0 +1,31 @@ +#include "windows-error.hh" + +#include +#define WIN32_LEAN_AND_MEAN +#include + +namespace nix { + +std::string WinError::renderError(DWORD lastError) +{ + LPSTR errorText = NULL; + + FormatMessageA( FORMAT_MESSAGE_FROM_SYSTEM // use system message tables to retrieve error text + |FORMAT_MESSAGE_ALLOCATE_BUFFER // allocate buffer on local heap for error text + |FORMAT_MESSAGE_IGNORE_INSERTS, // Important! will fail otherwise, since we're not (and CANNOT) pass insertion parameters + NULL, // unused with FORMAT_MESSAGE_FROM_SYSTEM + lastError, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPTSTR)&errorText, // output + 0, // minimum size for output buffer + NULL); // arguments - see note + + if (NULL != errorText ) { + std::string s2 { errorText }; + LocalFree(errorText); + return s2; + } + return fmt("CODE=%d", lastError); +} + +} diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh new file mode 100644 index 000000000..fdfd0f52c --- /dev/null +++ b/src/libutil/windows/windows-error.hh @@ -0,0 +1,51 @@ +#pragma once +///@file + +#include + +#include "error.hh" + +namespace nix { + +/** + * Windows Error type. + * + * Unless you need to catch a specific error number, don't catch this in + * portable code. Catch `SystemError` instead. + */ +class WinError : public SystemError +{ +public: + DWORD lastError; + + /** + * Construct using the explicitly-provided error number. + * `FormatMessageA` will be used to try to add additional + * information to the message. + */ + template + WinError(DWORD lastError, const Args & ... args) + : SystemError(""), lastError(lastError) + { + auto hf = HintFmt(args...); + err.msg = HintFmt("%1%: %2%", Uncolored(hf.str()), renderError(lastError)); + } + + /** + * Construct using `GetLastError()` and the ambient "last error". + * + * Be sure to not perform another last-error-modifying operation + * before calling this constructor! + */ + template + WinError(const Args & ... args) + : WinError(GetLastError(), args ...) + { + } + +private: + + std::string renderError(DWORD lastError); +}; + +} diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index cf0569c9f..eeca01833 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1342,8 +1342,16 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs RunPager pager; for (auto & i : gens) { +#ifdef _WIN32 // TODO portable wrapper in libutil + tm * tp = localtime(&i.creationTime); + if (!tp) + throw Error("cannot convert time"); + auto & t = *tp; +#else tm t; - if (!localtime_r(&i.creationTime, &t)) throw Error("cannot convert time"); + if (!localtime_r(&i.creationTime, &t)) + throw Error("cannot convert time"); +#endif logger->cout("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||", i.number, t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 86e6f008d..1e1728225 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -168,7 +168,7 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { auto p = state->findFile(i); if (auto fn = p.getPhysicalPath()) - std::cout << fn->native() << std::endl; + std::cout << fn->string() << std::endl; else throw Error("'%s' has no physical path", p); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7c8905da6..719675cba 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -4,10 +4,8 @@ #include "globals.hh" #include "build-result.hh" #include "store-cast.hh" -#include "gc-store.hh" +#include "local-fs-store.hh" #include "log-store.hh" -#include "local-store.hh" -#include "monitor-fd.hh" #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "shared.hh" @@ -15,7 +13,12 @@ #include "legacy.hh" #include "posix-source-accessor.hh" #include "path-with-outputs.hh" -#include "posix-fs-canonicalise.hh" + +#ifndef _WIN32 // TODO implement on Windows or provide allowed-to-noop interface +# include "local-store.hh" +# include "monitor-fd.hh" +# include "posix-fs-canonicalise.hh" +#endif #include #include @@ -43,12 +46,14 @@ static bool noOutput = false; static std::shared_ptr store; +#ifndef _WIN32 // TODO reenable on Windows once we have `LocalStore` there ref ensureLocalStore() { auto store2 = std::dynamic_pointer_cast(store); if (!store2) throw Error("you don't have sufficient rights to use this command"); return ref(store2); } +#endif static StorePath useDeriver(const StorePath & path) @@ -550,7 +555,11 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (!store->isValidPath(info->path) || reregister) { /* !!! races */ if (canonicalise) +#ifdef _WIN32 // TODO implement on Windows + throw UnimplementedError("file attribute canonicalisation Is not implemented on Windows"); +#else canonicalisePathMetaData(store->printStorePath(info->path), {}); +#endif if (!hashGiven) { HashResult hash = hashPath( *store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }, @@ -563,7 +572,9 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) } } +#ifndef _WIN32 // TODO reenable on Windows once we have `LocalStore` there ensureLocalStore()->registerValidPaths(infos); +#endif } @@ -684,7 +695,7 @@ static void opDump(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); if (opArgs.size() != 1) throw UsageError("only one argument allowed"); - FdSink sink(STDOUT_FILENO); + FdSink sink(getStandardOut()); std::string path = *opArgs.begin(); dumpPath(path, sink); sink.flush(); @@ -712,7 +723,7 @@ static void opExport(Strings opFlags, Strings opArgs) for (auto & i : opArgs) paths.insert(store->followLinksToStorePath(i)); - FdSink sink(STDOUT_FILENO); + FdSink sink(getStandardOut()); store->exportPaths(paths, sink); sink.flush(); } @@ -825,7 +836,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource in(STDIN_FILENO); - FdSink out(STDOUT_FILENO); + FdSink out(getStandardOut()); /* Exchange the greeting. */ ServeProto::Version clientVersion = @@ -946,7 +957,9 @@ static void opServe(Strings opFlags, Strings opArgs) getBuildSettings(); try { +#ifndef _WIN32 // TODO figure out if Windows needs something similar MonitorFdHup monitor(in.fd); +#endif store->buildPaths(toDerivedPaths(paths)); out << 0; } catch (Error & e) { @@ -966,7 +979,9 @@ static void opServe(Strings opFlags, Strings opArgs) getBuildSettings(); +#ifndef _WIN32 // TODO figure out if Windows needs something similar MonitorFdHup monitor(in.fd); +#endif auto status = store->buildDerivation(drvPath, drv); ServeProto::write(*store, wconn, status); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 4df086d4f..ee904b0c5 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -15,7 +15,8 @@ struct MixCat : virtual Args if (st.type != SourceAccessor::Type::tRegular) throw Error("path '%1%' is not a regular file", path); stopProgressBar(); - writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path))); + + writeFull(getStandardOut(), accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index b654dc52f..35d3da912 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -7,7 +7,10 @@ #include "outputs-spec.hh" #include "derivations.hh" #include "progress-bar.hh" -#include "run.hh" + +#ifndef _WIN32 // TODO re-enable on Windows +# include "run.hh" +#endif #include #include @@ -662,6 +665,9 @@ struct CmdDevelop : Common, MixEnvironment // This is to make sure the system shell doesn't leak into the build environment. setEnv("SHELL", shell.c_str()); +#ifdef _WIN32 // TODO re-enable on Windows + throw UnimplementedError("Cannot yet spawn processes on Windows"); +#else // If running a phase or single command, don't want an interactive shell running after // Ctrl-C, so don't pass --rcfile auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} @@ -682,6 +688,7 @@ struct CmdDevelop : Common, MixEnvironment } runProgramInStore(store, UseSearchPath::Use, shell, args, buildEnvironment.getSystem()); +#endif } }; diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 0850d4c1c..953d77d31 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -20,7 +20,7 @@ struct CmdDumpPath : StorePathCommand void run(ref store, const StorePath & storePath) override { - FdSink sink(STDOUT_FILENO); + FdSink sink(getStandardOut()); store->narFromPath(storePath, sink); sink.flush(); } @@ -55,7 +55,7 @@ struct CmdDumpPath2 : Command void run() override { - FdSink sink(STDOUT_FILENO); + FdSink sink(getStandardOut()); dumpPath(path, sink); sink.flush(); } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 7b9bf7cb1..494735516 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -87,7 +87,11 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption // FIXME: disallow strings with contexts? writeFile(path, v.string_view()); else if (v.type() == nAttrs) { - if (mkdir(path.c_str(), 0777) == -1) + if (mkdir(path.c_str() +#ifndef _WIN32 // TODO abstract mkdir perms for Windows + , 0777 +#endif + ) == -1) throw SysError("creating directory '%s'", path); for (auto & attr : *v.attrs()) { std::string_view name = state->symbols[attr.name]; @@ -112,7 +116,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption else if (raw) { stopProgressBar(); - writeFull(STDOUT_FILENO, *state->coerceToString(noPos, *v, context, "while generating the eval command output")); + writeFull(getStandardOut(), *state->coerceToString(noPos, *v, context, "while generating the eval command output")); } else if (json) { diff --git a/src/nix/local.mk b/src/nix/local.mk index 9f6f31b3a..305b0e9df 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -4,19 +4,19 @@ nix_DIR := $(d) nix_SOURCES := \ $(wildcard $(d)/*.cc) \ - $(wildcard src/build-remote/*.cc) \ $(wildcard src/nix-build/*.cc) \ - $(wildcard src/nix-channel/*.cc) \ - $(wildcard src/nix-collect-garbage/*.cc) \ - $(wildcard src/nix-copy-closure/*.cc) \ - $(wildcard src/nix-daemon/*.cc) \ $(wildcard src/nix-env/*.cc) \ $(wildcard src/nix-instantiate/*.cc) \ $(wildcard src/nix-store/*.cc) ifdef HOST_UNIX nix_SOURCES += \ - $(wildcard $(d)/unix/*.cc) + $(wildcard $(d)/unix/*.cc) \ + $(wildcard src/build-remote/*.cc) \ + $(wildcard src/nix-channel/*.cc) \ + $(wildcard src/nix-collect-garbage/*.cc) \ + $(wildcard src/nix-copy-closure/*.cc) \ + $(wildcard src/nix-daemon/*.cc) endif INCLUDE_nix := -I $(d) diff --git a/src/nix/log.cc b/src/nix/log.cc index 9a9bd30f9..7f590c708 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -57,7 +57,7 @@ struct CmdLog : InstallableCommand if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); - writeFull(STDOUT_FILENO, *log); + writeFull(getStandardOut(), *log); return; } diff --git a/src/nix/main.cc b/src/nix/main.cc index af6498218..7b0478a9f 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -16,26 +16,34 @@ #include "markdown.hh" #include "memory-input-accessor.hh" #include "terminal.hh" +#include "users.hh" #include -#include -#include -#include -#include #include - #include +#ifndef _WIN32 +# include +# include +# include +# include +#endif + #if __linux__ # include "namespaces.hh" #endif +#ifndef _WIN32 extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); +#endif namespace nix { +#ifdef _WIN32 +[[maybe_unused]] +#endif static bool haveProxyEnvironmentVariables() { static const std::vector proxyVariables = { @@ -57,6 +65,7 @@ static bool haveProxyEnvironmentVariables() /* Check if we have a non-loopback/link-local network interface. */ static bool haveInternet() { +#ifndef _WIN32 struct ifaddrs * addrs; if (getifaddrs(&addrs)) @@ -80,6 +89,10 @@ static bool haveInternet() if (haveProxyEnvironmentVariables()) return true; return false; +#else + // TODO implement on Windows + return true; +#endif } std::string programPath; @@ -342,10 +355,12 @@ void mainWrapped(int argc, char * * argv) /* The chroot helper needs to be run before any threads have been started. */ +#ifndef _WIN32 if (argc > 0 && argv[0] == chrootHelperName) { chrootHelper(argc, argv); return; } +#endif initNix(); initGC(); @@ -364,6 +379,9 @@ void mainWrapped(int argc, char * * argv) programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); + auto extensionPos = programName.find_last_of("."); + if (extensionPos != std::string::npos) + programName.erase(extensionPos); if (argc > 1 && std::string_view(argv[1]) == "__build-remote") { programName = "build-remote"; @@ -524,9 +542,11 @@ void mainWrapped(int argc, char * * argv) int main(int argc, char * * argv) { +#ifndef _WIN32 // TODO implement on Windows // Increase the default stack size for the evaluator and for // libstdc++'s std::regex. nix::setStackSize(64 * 1024 * 1024); +#endif return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index d30e9d397..8e6a2e805 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -95,7 +95,7 @@ std::tuple prefetchFile( if (executable) mode = 0700; - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + AutoCloseFD fd = toDescriptor(open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode)); if (!fd) throw SysError("creating temporary file '%s'", tmpFile); FdSink sink(fd.get()); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index dfef44869..1e277cbbe 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -177,7 +177,7 @@ struct CmdKeyGenerateSecret : Command throw UsageError("required argument '--key-name' is missing"); stopProgressBar(); - writeFull(STDOUT_FILENO, SecretKey::generate(*keyName).to_string()); + writeFull(getStandardOut(), SecretKey::generate(*keyName).to_string()); } }; @@ -199,7 +199,7 @@ struct CmdKeyConvertSecretToPublic : Command { SecretKey secretKey(drainFD(STDIN_FILENO)); stopProgressBar(); - writeFull(STDOUT_FILENO, secretKey.toPublicKey().to_string()); + writeFull(getStandardOut(), secretKey.toPublicKey().to_string()); } }; diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index d7e9edf0a..b66872a6e 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -421,6 +421,7 @@ namespace nix { ASSERT_EQ(string2Int("-100"), -100); } +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes /* ---------------------------------------------------------------------------- * statusOk * --------------------------------------------------------------------------*/ @@ -429,6 +430,7 @@ namespace nix { ASSERT_EQ(statusOk(0), true); ASSERT_EQ(statusOk(1), false); } +#endif /* ---------------------------------------------------------------------------- From 98691b46e39fcaa0688e6f880e539bc88d86d63a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Jan 2024 20:39:46 -0500 Subject: [PATCH 198/211] Get rid of `shellCrossSystems` We don't need it now that our (minimized) Windows build of Nix succeeds! --- flake.nix | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index be4e68783..8ee46101b 100644 --- a/flake.nix +++ b/flake.nix @@ -32,12 +32,6 @@ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" "x86_64-unknown-netbsd" - ]; - - # Nix doesn't yet build on this platform, so we put it in a - # separate list. We just use this for `devShells` and - # `nixpkgsFor`, which this depends on. - shellCrossSystems = crossSystems ++ [ "x86_64-w64-mingw32" ]; @@ -83,7 +77,7 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); installScriptFor = tarballs: @@ -426,8 +420,8 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) - (makeShells "static" nixpkgsFor.${system}.static)) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // { default = self.devShells.${system}.native-stdenvPackages; } From b529d91902a46e1fe0954d44fbc3622edcbe99ca Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 13 Jan 2024 15:47:08 -0500 Subject: [PATCH 199/211] Prefix `-DNIX_` paths to be windows-complient for windows This is a hacky solution, but it will do for now. --- src/libstore/local.mk | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index b74a047bf..6be748581 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -43,17 +43,28 @@ ifdef HOST_WINDOWS INCLUDE_libstore += -I $(d)/windows endif +ifdef HOST_WINDOWS +NIX_ROOT = N:\\\\ +else +NIX_ROOT = +endif + +# Prefix all but `NIX_STORE_DIR`, since we aren't doing a local store +# yet so a "logical" store dir that is the same as unix is prefered. +# +# Also, it keeps the unit tests working. + libstore_CXXFLAGS += \ $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libstore) \ - -DNIX_PREFIX=\"$(prefix)\" \ + -DNIX_PREFIX=\"$(NIX_ROOT)$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ - -DNIX_DATA_DIR=\"$(datadir)\" \ - -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ - -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ - -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ - -DNIX_BIN_DIR=\"$(bindir)\" \ - -DNIX_MAN_DIR=\"$(mandir)\" \ - -DLSOF=\"$(lsof)\" + -DNIX_DATA_DIR=\"$(NIX_ROOT)$(datadir)\" \ + -DNIX_STATE_DIR=\"$(NIX_ROOT)$(localstatedir)/nix\" \ + -DNIX_LOG_DIR=\"$(NIX_ROOT)$(localstatedir)/log/nix\" \ + -DNIX_CONF_DIR=\"$(NIX_ROOT)$(sysconfdir)/nix\" \ + -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" \ + -DNIX_MAN_DIR=\"$(NIX_ROOT)$(mandir)\" \ + -DLSOF=\"$(NIX_ROOT)$(lsof)\" ifeq ($(embedded_sandbox_shell),yes) libstore_CXXFLAGS += -DSANDBOX_SHELL=\"__embedded_sandbox_shell__\" From 1063aa502a3b931f96c1ff8bb6147ab73e63d582 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 14 Jan 2024 13:36:53 -0500 Subject: [PATCH 200/211] Relax store path canonicalization --- src/libstore/globals.cc | 8 +++++++- src/libstore/path.cc | 13 ++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ae268e54e..83e54e008 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -49,7 +49,13 @@ static GlobalConfig::Register rSettings(&settings); Settings::Settings() : nixPrefix(NIX_PREFIX) - , nixStore(canonPath(getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR)))) + , nixStore( +#ifndef _WIN32 + // On Windows `/nix/store` is not a canonical path, but we dont' + // want to deal with that yet. + canonPath +#endif + (getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR)))) , nixDataDir(canonPath(getEnvNonEmpty("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) , nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 5db4b974c..4b806e408 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -63,7 +63,18 @@ StorePath StorePath::random(std::string_view name) StorePath StoreDirConfig::parseStorePath(std::string_view path) const { - auto p = canonPath(std::string(path)); + // On Windows, `/nix/store` is not a canonical path. More broadly it + // is unclear whether this function should be using the native + // notion of a canonical path at all. For example, it makes to + // support remote stores whose store dir is a non-native path (e.g. + // Windows <-> Unix ssh-ing). + auto p = +#ifdef _WIN32 + path +#else + canonPath(std::string(path)) +#endif + ; if (dirOf(p) != storeDir) throw BadStorePath("path '%s' is not in the Nix store", p); return StorePath(baseNameOf(p)); From 5fd28eeba49c7bc23b539b677f63313925acb484 Mon Sep 17 00:00:00 2001 From: a-kenji Date: Wed, 17 Apr 2024 19:25:11 +0200 Subject: [PATCH 201/211] docs: fix wiki link Link to the official wiki. --- src/nix/flake-init.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md index ea274bf29..e14dfad7e 100644 --- a/src/nix/flake-init.md +++ b/src/nix/flake-init.md @@ -56,7 +56,7 @@ outputs = { self }: { ## More info - [Rust language](https://www.rust-lang.org/) - - [Rust on the NixOS Wiki](https://nixos.wiki/wiki/Rust) + - [Rust on the NixOS Wiki](https://wiki.nixos.org/wiki/Rust) - ... ''; }; From 21d9412ddc07341df0194a0301d88e41ab9cf84c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 14 Apr 2024 12:28:37 -0400 Subject: [PATCH 202/211] Improve `local-overlay` docs in a few ways In response to https://discourse.nixos.org/t/super-colliding-nix-stores/28462/24 --- src/libstore/unix/local-overlay-store.md | 30 +++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/libstore/unix/local-overlay-store.md b/src/libstore/unix/local-overlay-store.md index cc310bc7f..1e1a3d26c 100644 --- a/src/libstore/unix/local-overlay-store.md +++ b/src/libstore/unix/local-overlay-store.md @@ -16,6 +16,8 @@ The parts of a local overlay store are as follows: - **Lower store**: + > Specified with the [`lower-store`](#store-experimental-local-overlay-store-lower-store) setting. + This is any store implementation that includes a store directory as part of the native operating system filesystem. For example, this could be a [local store], [local daemon store], or even another local overlay store. @@ -28,13 +30,11 @@ The parts of a local overlay store are as follows: The lower store must not change while it is mounted as part of an overlay store. To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`). - Specified with the [`lower-store`](#store-experimental-local-overlay-store-lower-store) setting. - - **Lower store directory**: - This is the directory used/exposed by the lower store. + > Specified with `lower-store.real` setting. - Specified with `lower-store.real` setting. + This is the directory used/exposed by the lower store. As specified above, Nix requires the local store can only grow not change in other ways. Linux's OverlayFS in addition imposes the further requirement that this directory cannot change at all. @@ -42,6 +42,9 @@ The parts of a local overlay store are as follows: - **Lower metadata source**: + > Not directly specified. + > A consequence of the `lower-store` setting, depending on the type of lower store chosen. + This is abstract, just some way to read the metadata of lower store [store objects][store object]. For example it could be a SQLite database (for the [local store]), or a socket connection (for the [local daemon store]). @@ -51,36 +54,41 @@ The parts of a local overlay store are as follows: - **Upper almost-store**: + > Not directly specified. + > Instead the constituent parts are independently specified as described below. + This is almost but not quite just a [local store]. That is because taken in isolation, not as part of a local overlay store, by itself, it would appear corrupted. But combined with everything else as part of an overlay local store, it is valid. - **Upper layer directory**: + > Specified with [`upper-layer`](#store-experimental-local-overlay-store-upper-layer) setting. + This contains additional [store objects][store object] (or, strictly speaking, their [file system objects][file system object] that the local overlay store will extend the lower store with). - Specified with [`upper-layer`](#store-experimental-local-overlay-store-upper-layer) setting. - - **Upper store directory**: + > Specified with the [`real`](#store-experimental-local-overlay-store-real) setting. + > This the same as the base local store setting, and can also be indirectly specified with the [`root`](#store-experimental-local-overlay-store-root) setting. + This contains all the store objects from each of the two directories. The lower store directory and upper layer directory are combined via OverlayFS to create this directory. Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first. Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings. - Specified with the [`real`](#store-experimental-local-overlay-store-real) setting. - - **Upper SQLite database**: + > Not directly specified. + > The location of the database instead depends on the [`state`](#store-experimental-local-overlay-store-state) setting. + > It is is always `${state}/db`. + This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta. The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer. (This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way.) - The location of the database is directly specified, but depends on the [`state`](#store-experimental-local-overlay-store-state) setting. - It is is always `${state}/db`. - [file system object]: @docroot@/store/file-system-object.md [store object]: @docroot@/store/store-object.md From 0774e8ba33c060f56bad3ff696796028249e915a Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Wed, 17 Apr 2024 21:51:59 +0200 Subject: [PATCH 203/211] Fix exportReferencesGraph when given store subpath With Nix 2.3, it was possible to pass a subpath of a store path to exportReferencesGraph: with import {}; let hello = writeShellScriptBin "hello" '' echo ${toString builtins.currentTime} ''; in writeClosure [ "${hello}/bin/hello" ] This regressed with Nix 2.4, with a very confusing error message, that presumably indicates it was unintentional: error: path '/nix/store/3gl7kgjr4pwf03f0x70dgx9ln3bhl7zc-hello/bin/hello' is not in the Nix store --- src/libstore/parsed-derivations.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 72f45143d..a29281953 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -186,7 +186,7 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s for (auto i = e->begin(); i != e->end(); ++i) { StorePathSet storePaths; for (auto & p : *i) - storePaths.insert(store.parseStorePath(p.get())); + storePaths.insert(store.toStorePath(p.get()).first); json[i.key()] = pathInfoToJSON(store, store.exportReferences(storePaths, inputPaths)); } From 6fa3656a3286d3e73a54277965a1b1ae8d34300d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 17 Apr 2024 16:20:56 -0400 Subject: [PATCH 204/211] Make a few commands that were Unix-only no longer Also clean up some more linux-specific (`setPersonality`) code in alignment with recent best practices. --- src/libstore/{unix/build => linux}/personality.cc | 6 +----- src/libstore/{unix/build => linux}/personality.hh | 2 +- src/libstore/local.mk | 6 ++++++ src/libstore/unix/build/local-derivation-goal.cc | 6 ++++-- src/nix/{unix => }/fmt.cc | 0 src/nix/{unix => }/fmt.md | 0 src/nix/{unix => }/run.cc | 14 +++++++++----- src/nix/{unix => }/run.hh | 0 src/nix/{unix => }/run.md | 0 src/nix/{unix => }/upgrade-nix.cc | 0 src/nix/{unix => }/upgrade-nix.md | 0 11 files changed, 21 insertions(+), 13 deletions(-) rename src/libstore/{unix/build => linux}/personality.cc (95%) rename src/libstore/{unix/build => linux}/personality.hh (80%) rename src/nix/{unix => }/fmt.cc (100%) rename src/nix/{unix => }/fmt.md (100%) rename src/nix/{unix => }/run.cc (97%) rename src/nix/{unix => }/run.hh (100%) rename src/nix/{unix => }/run.md (100%) rename src/nix/{unix => }/upgrade-nix.cc (100%) rename src/nix/{unix => }/upgrade-nix.md (100%) diff --git a/src/libstore/unix/build/personality.cc b/src/libstore/linux/personality.cc similarity index 95% rename from src/libstore/unix/build/personality.cc rename to src/libstore/linux/personality.cc index 1a6201758..255d174a6 100644 --- a/src/libstore/unix/build/personality.cc +++ b/src/libstore/linux/personality.cc @@ -1,18 +1,15 @@ #include "personality.hh" #include "globals.hh" -#if __linux__ #include #include -#endif #include -namespace nix { +namespace nix::linux { void setPersonality(std::string_view system) { -#if __linux__ /* Change the personality to 32-bit if we're doing an i686-linux build on an x86_64-linux machine. */ struct utsname utsbuf; @@ -39,7 +36,6 @@ void setPersonality(std::string_view system) determinism. */ int cur = personality(0xffffffff); if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif } } diff --git a/src/libstore/unix/build/personality.hh b/src/libstore/linux/personality.hh similarity index 80% rename from src/libstore/unix/build/personality.hh rename to src/libstore/linux/personality.hh index 91b730fab..6a6376f8f 100644 --- a/src/libstore/unix/build/personality.hh +++ b/src/libstore/linux/personality.hh @@ -3,7 +3,7 @@ #include -namespace nix { +namespace nix::linux { void setPersonality(std::string_view system); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 6be748581..2e118f6cb 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,6 +8,9 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) ifdef HOST_UNIX libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/builtins/*.cc $(d)/unix/build/*.cc) endif +ifdef HOST_LINUX + libstore_SOURCES += $(wildcard $(d)/linux/*.cc) +endif ifdef HOST_WINDOWS libstore_SOURCES += $(wildcard $(d)/windows/*.cc) endif @@ -39,6 +42,9 @@ INCLUDE_libstore := -I $(d) -I $(d)/build ifdef HOST_UNIX INCLUDE_libstore += -I $(d)/unix endif +ifdef HOST_LINUX + INCLUDE_libstore += -I $(d)/linux +endif ifdef HOST_WINDOWS INCLUDE_libstore += -I $(d)/windows endif diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 99a91a7f1..33208a622 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -14,7 +14,6 @@ #include "topo-sort.hh" #include "callback.hh" #include "json-utils.hh" -#include "personality.hh" #include "current-process.hh" #include "child.hh" #include "unix-domain-socket.hh" @@ -52,6 +51,7 @@ # endif # define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) # include "cgroup.hh" +# include "personality.hh" #endif #if __APPLE__ @@ -1957,7 +1957,9 @@ void LocalDerivationGoal::runChild() /* Close all other file descriptors. */ closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); - setPersonality(drv->platform); +#if __linux__ + linux::setPersonality(drv->platform); +#endif /* Disable core dumps by default. */ struct rlimit limit = { 0, RLIM_INFINITY }; diff --git a/src/nix/unix/fmt.cc b/src/nix/fmt.cc similarity index 100% rename from src/nix/unix/fmt.cc rename to src/nix/fmt.cc diff --git a/src/nix/unix/fmt.md b/src/nix/fmt.md similarity index 100% rename from src/nix/unix/fmt.md rename to src/nix/fmt.md diff --git a/src/nix/unix/run.cc b/src/nix/run.cc similarity index 97% rename from src/nix/unix/run.cc rename to src/nix/run.cc index dfd8b643c..c45683302 100644 --- a/src/nix/unix/run.cc +++ b/src/nix/run.cc @@ -5,15 +5,15 @@ #include "shared.hh" #include "store-api.hh" #include "derivations.hh" -#include "local-store.hh" +#include "local-fs-store.hh" #include "finally.hh" #include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" -#include "build/personality.hh" #if __linux__ -#include +# include +# include "personality.hh" #endif #include @@ -56,8 +56,10 @@ void runProgramInStore(ref store, throw SysError("could not execute chroot helper"); } +#if __linux__ if (system) - setPersonality(*system); + linux::setPersonality(*system); +#endif if (useSearchPath == UseSearchPath::Use) execvp(program.c_str(), stringsToCharPtrs(args).data()); @@ -277,8 +279,10 @@ void chrootHelper(int argc, char * * argv) writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); +#if __linux__ if (system != "") - setPersonality(system); + linux::setPersonality(system); +#endif execvp(cmd.c_str(), stringsToCharPtrs(args).data()); diff --git a/src/nix/unix/run.hh b/src/nix/run.hh similarity index 100% rename from src/nix/unix/run.hh rename to src/nix/run.hh diff --git a/src/nix/unix/run.md b/src/nix/run.md similarity index 100% rename from src/nix/unix/run.md rename to src/nix/run.md diff --git a/src/nix/unix/upgrade-nix.cc b/src/nix/upgrade-nix.cc similarity index 100% rename from src/nix/unix/upgrade-nix.cc rename to src/nix/upgrade-nix.cc diff --git a/src/nix/unix/upgrade-nix.md b/src/nix/upgrade-nix.md similarity index 100% rename from src/nix/unix/upgrade-nix.md rename to src/nix/upgrade-nix.md From 9c815db36681ac01ea7422725a0ac2e183e39b16 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 17 Apr 2024 19:55:40 -0400 Subject: [PATCH 205/211] `file-descriptor.hh`: Avoid some Cism for better C++isms - `reinterpret_cast` not C-style cast - `using` not `typedef` --- src/libutil/file-descriptor.hh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index 50201d846..84786e95a 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -17,13 +17,13 @@ struct Source; /** * Operating System capability */ -typedef +using Descriptor = #if _WIN32 HANDLE #else int #endif - Descriptor; + ; const Descriptor INVALID_DESCRIPTOR = #if _WIN32 @@ -41,7 +41,7 @@ const Descriptor INVALID_DESCRIPTOR = static inline Descriptor toDescriptor(int fd) { #ifdef _WIN32 - return (HANDLE) _get_osfhandle(fd); + return reinterpret_cast(_get_osfhandle(fd)); #else return fd; #endif @@ -56,7 +56,7 @@ static inline Descriptor toDescriptor(int fd) static inline int fromDescriptorReadOnly(Descriptor fd) { #ifdef _WIN32 - return _open_osfhandle((intptr_t) fd, _O_RDONLY); + return _open_osfhandle(reinterpret_cast(fd), _O_RDONLY); #else return fd; #endif From ba6804518772e6afb403dd55478365d4b863c854 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 14 Apr 2024 14:10:23 +0200 Subject: [PATCH 206/211] libstore/local-derivation-goal: prohibit creating setuid/setgid binaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With Linux kernel >=6.6 & glibc 2.39 a `fchmodat2(2)` is available that isn't filtered away by the libseccomp sandbox. Being able to use this to bypass that restriction has surprising results for some builds such as lxc[1]: > With kernel ≥6.6 and glibc 2.39, lxc's install phase uses fchmodat2, > which slips through https://github.com/NixOS/nix/blob/9b88e5284608116b7db0dbd3d5dd7a33b90d52d7/src/libstore/build/local-derivation-goal.cc#L1650-L1663. > The fixupPhase then uses fchmodat, which fails. > With older kernel or glibc, setting the suid bit fails in the > install phase, which is not treated as fatal, and then the > fixup phase does not try to set it again. Please note that there are still ways to bypass this sandbox[2] and this is mostly a fix for the breaking builds. This change works by creating a syscall filter for the `fchmodat2` syscall (number 452 on most systems). The problem is that glibc 2.39 and seccomp 2.5.5 are needed to have the correct syscall number available via `__NR_fchmodat2` / `__SNR_fchmodat2`, but this flake is still on nixpkgs 23.11. To have this change everywhere and not dependent on the glibc this package is built against, I added a header "fchmodat2-compat.hh" that sets the syscall number based on the architecture. On most platforms its 452 according to glibc with a few exceptions: $ rg --pcre2 'define __NR_fchmodat2 (?!452)' sysdeps/unix/sysv/linux/x86_64/x32/arch-syscall.h 58:#define __NR_fchmodat2 1073742276 sysdeps/unix/sysv/linux/mips/mips64/n32/arch-syscall.h 67:#define __NR_fchmodat2 6452 sysdeps/unix/sysv/linux/mips/mips64/n64/arch-syscall.h 62:#define __NR_fchmodat2 5452 sysdeps/unix/sysv/linux/mips/mips32/arch-syscall.h 70:#define __NR_fchmodat2 4452 sysdeps/unix/sysv/linux/alpha/arch-syscall.h 59:#define __NR_fchmodat2 562 I tested the change by adding the diff below as patch to `pkgs/tools/package-management/nix/common.nix` & then built a VM from the following config using my dirty nixpkgs master: { vm = { pkgs, ... }: { virtualisation.writableStore = true; virtualisation.memorySize = 8192; virtualisation.diskSize = 12 * 1024; nix.package = pkgs.nixVersions.nix_2_21; }; } The original issue can be triggered via nix build -L github:nixos/nixpkgs/d6dc19adbda4fd92fe9a332327a8113eaa843894#lxc \ --extra-experimental-features 'nix-command flakes' however the problem disappears with this patch applied. Closes #10424 [1] https://github.com/NixOS/nixpkgs/issues/300635#issuecomment-2031073804 [2] https://github.com/NixOS/nixpkgs/issues/300635#issuecomment-2030844251 --- src/libstore/linux/fchmodat2-compat.hh | 34 +++++++++++++++++++ .../unix/build/local-derivation-goal.cc | 5 +++ 2 files changed, 39 insertions(+) create mode 100644 src/libstore/linux/fchmodat2-compat.hh diff --git a/src/libstore/linux/fchmodat2-compat.hh b/src/libstore/linux/fchmodat2-compat.hh new file mode 100644 index 000000000..fd03b9ed5 --- /dev/null +++ b/src/libstore/linux/fchmodat2-compat.hh @@ -0,0 +1,34 @@ +/* + * Determine the syscall number for `fchmodat2`. + * + * On most platforms this is 452. Exceptions can be found on + * a glibc git checkout via `rg --pcre2 'define __NR_fchmodat2 (?!452)'`. + * + * The problem is that glibc 2.39 and libseccomp 2.5.5 are needed to + * get the syscall number. However, a Nix built against nixpkgs 23.11 + * (glibc 2.38) should still have the issue fixed without depending + * on the build environment. + * + * To achieve that, the macros below try to determine the platform and + * set the syscall number which is platform-specific, but + * in most cases 452. + * + * TODO: remove this when 23.11 is EOL and the entire (supported) ecosystem + * is on glibc 2.39. + */ + +#if HAVE_SECCOMP +# if defined(__alpha__) +# define NIX_SYSCALL_FCHMODAT2 562 +# elif defined(__x86_64__) && SIZE_MAX == 0xFFFFFFFF // x32 +# define NIX_SYSCALL_FCHMODAT2 1073742276 +# elif defined(__mips__) && defined(__mips64) && defined(_ABIN64) // mips64/n64 +# define NIX_SYSCALL_FCHMODAT2 5452 +# elif defined(__mips__) && defined(__mips64) && defined(_ABIN32) // mips64/n32 +# define NIX_SYSCALL_FCHMODAT2 6452 +# elif defined(__mips__) && defined(_ABIO32) // mips32 +# define NIX_SYSCALL_FCHMODAT2 4452 +# else +# define NIX_SYSCALL_FCHMODAT2 452 +# endif +#endif // HAVE_SECCOMP diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 33208a622..2d691143c 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -37,6 +37,7 @@ /* Includes required for chroot support. */ #if __linux__ +# include "linux/fchmodat2-compat.hh" # include # include # include @@ -1672,6 +1673,10 @@ void setupSeccomp() if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), NIX_SYSCALL_FCHMODAT2, 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); } /* Prevent builders from creating EAs or ACLs. Not all filesystems From fb9f4208ed371afe23307f9b6cb9ada618bada9b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 Apr 2024 16:57:07 -0400 Subject: [PATCH 207/211] Don't include `linux/` in `#include` The linux dirs are conditionally added to the `-I` path. --- src/libstore/unix/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 2d691143c..aad5173e7 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -37,7 +37,7 @@ /* Includes required for chroot support. */ #if __linux__ -# include "linux/fchmodat2-compat.hh" +# include "fchmodat2-compat.hh" # include # include # include From e3fa7c38d7af8f34de0c24766b2e8cf1cd1330f0 Mon Sep 17 00:00:00 2001 From: 0x4A6F <0x4A6F@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:10:52 +0200 Subject: [PATCH 208/211] system: build for riscv64-unknown-linux-gnu (#10228) * system: add support for riscv64-unknown-linux-gnu * maintainers: upload riscv64-linux-gnu * doc: add riscv64-linux to supported platforms --- doc/manual/src/contributing/hacking.md | 6 +++++- flake.nix | 3 +++ maintainers/upload-release.pl | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index d56ac29a4..c43149c4d 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -144,6 +144,7 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: - `aarch64-darwin` - `armv6l-linux` - `armv7l-linux` +- `riscv64-linux` In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that @@ -166,7 +167,10 @@ or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled $ nix build .#packages.aarch64-linux.default ``` -Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. ### Building for multiple platforms at once diff --git a/flake.nix b/flake.nix index 8ee46101b..07d909fcd 100644 --- a/flake.nix +++ b/flake.nix @@ -31,6 +31,7 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" + "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" "x86_64-w64-mingw32" ]; @@ -253,6 +254,7 @@ # Cross self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" ]; installerScriptForGHA = installScriptFor [ # Native @@ -261,6 +263,7 @@ # Cross self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" ]; # docker image with Nix inside diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index f2830a3af..6d1b7631d 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -171,6 +171,10 @@ eval { downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); }; warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. From f8a67d7e26296c325410febdf10cf21b27ef57d1 Mon Sep 17 00:00:00 2001 From: 0x4A6F <0x4A6F@users.noreply.github.com> Date: Thu, 18 Apr 2024 17:09:11 +0200 Subject: [PATCH 209/211] scripts/upload-release: fix riscv64 call --- maintainers/upload-release.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 6d1b7631d..9a30f8227 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -172,7 +172,7 @@ eval { }; warn "$@" if $@; eval { - downloadFile("binaryTarballCross.x86_64-linux.riscv64-linux-gnu", "1"); + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); }; warn "$@" if $@; downloadFile("installerScript", "1"); From ad643cde587805ac6e03cef4e51ca5a4268829d6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Apr 2024 17:41:23 +0200 Subject: [PATCH 210/211] C API: Add nix_init_apply Thunks are relevant when initializing attrsets and lists, passing arguments. This is an important way to produce them. --- src/libexpr-c/nix_api_expr.h | 2 + src/libexpr-c/nix_api_value.cc | 13 +++ src/libexpr-c/nix_api_value.h | 20 ++++- tests/unit/libexpr/nix_api_value.cc | 124 ++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 2 deletions(-) diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index 7504b5d7a..fd9746ab7 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -93,6 +93,8 @@ nix_err nix_expr_eval_from_string( * @param[in] arg The argument to pass to the function. * @param[out] value The result of the function call. * @return NIX_OK if the function call was successful, an error code otherwise. + * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk. + * Note the different argument order. */ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value); diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 79e62a1d2..2550e975a 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -404,6 +404,19 @@ nix_err nix_init_null(nix_c_context * context, Value * value) NIXC_CATCH_ERRS } +nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + auto & f = check_value_not_null(fn); + auto & a = check_value_not_null(arg); + v.mkApp(&f, &a); + } + NIXC_CATCH_ERRS +} + nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val) { if (context) diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index e6744e610..d8bd77c33 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -342,8 +342,24 @@ nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i); * @param[out] value Nix value to modify * @return error code, NIX_OK on success. */ - nix_err nix_init_null(nix_c_context * context, Value * value); + +/** @brief Set the value to a thunk that will perform a function application when needed. + * + * Thunks may be put into attribute sets and lists to perform some computation lazily; on demand. + * However, note that in some places, a thunk must not be returned, such as in the return value of a PrimOp. + * In such cases, you may use nix_value_call() instead (but note the different argument order). + * + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] fn function to call + * @param[in] arg argument to pass + * @return error code, NIX_OK on successful initialization. + * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value. + * Note the different argument order. + */ +nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg); + /** @brief Set an external value * @param[out] context Optional, stores error information * @param[out] value Nix value to modify @@ -421,7 +437,7 @@ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * /** @brief Insert bindings into a builder * @param[out] context Optional, stores error information * @param[in] builder BindingsBuilder to insert into - * @param[in] name attribute name, copied into the symbol store + * @param[in] name attribute name, only used for the duration of the call. * @param[in] value value to give the binding * @return error code, NIX_OK on success. */ diff --git a/tests/unit/libexpr/nix_api_value.cc b/tests/unit/libexpr/nix_api_value.cc index 7fbb2bbdc..ac0cdb9c4 100644 --- a/tests/unit/libexpr/nix_api_value.cc +++ b/tests/unit/libexpr/nix_api_value.cc @@ -8,6 +8,7 @@ #include "tests/nix_api_expr.hh" #include "tests/string_callback.hh" +#include "gmock/gmock.h" #include #include @@ -187,4 +188,127 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) free(out_name); } +TEST_F(nix_api_expr_test, nix_value_init) +{ + // Setup + + // two = 2; + // f = a: a * a; + + Value * two = nix_alloc_value(ctx, state); + nix_init_int(ctx, two, 2); + + Value * f = nix_alloc_value(ctx, state); + nix_expr_eval_from_string( + ctx, state, R"( + a: a * a + )", + "", f); + + // Test + + // r = f two; + + Value * r = nix_alloc_value(ctx, state); + nix_init_apply(ctx, r, f, two); + assert_ctx_ok(); + + ValueType t = nix_get_type(ctx, r); + assert_ctx_ok(); + + ASSERT_EQ(t, NIX_TYPE_THUNK); + + nix_value_force(ctx, state, r); + + t = nix_get_type(ctx, r); + assert_ctx_ok(); + + ASSERT_EQ(t, NIX_TYPE_INT); + + int n = nix_get_int(ctx, r); + assert_ctx_ok(); + + ASSERT_EQ(n, 4); + + // Clean up + nix_gc_decref(ctx, two); + nix_gc_decref(ctx, f); + nix_gc_decref(ctx, r); +} + +TEST_F(nix_api_expr_test, nix_value_init_apply_error) +{ + Value * some_string = nix_alloc_value(ctx, state); + nix_init_string(ctx, some_string, "some string"); + assert_ctx_ok(); + + Value * v = nix_alloc_value(ctx, state); + nix_init_apply(ctx, v, some_string, some_string); + assert_ctx_ok(); + + // All ok. Call has not been evaluated yet. + + // Evaluate it + nix_value_force(ctx, state, v); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("attempt to call something which is not a function but")); + + // Clean up + nix_gc_decref(ctx, some_string); + nix_gc_decref(ctx, v); +} + +TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) +{ + // f is a lazy function: it does not evaluate its argument before returning its return value + // g is a helper to produce e + // e is a thunk that throws an exception + // + // r = f e + // r should not throw an exception, because e is not evaluated + + Value * f = nix_alloc_value(ctx, state); + nix_expr_eval_from_string( + ctx, state, R"( + a: { foo = a; } + )", + "", f); + assert_ctx_ok(); + + Value * e = nix_alloc_value(ctx, state); + { + Value * g = nix_alloc_value(ctx, state); + nix_expr_eval_from_string( + ctx, state, R"( + _ignore: throw "error message for test case nix_value_init_apply_lazy_arg" + )", + "", g); + assert_ctx_ok(); + + nix_init_apply(ctx, e, g, g); + assert_ctx_ok(); + nix_gc_decref(ctx, g); + } + + Value * r = nix_alloc_value(ctx, state); + nix_init_apply(ctx, r, f, e); + assert_ctx_ok(); + + nix_value_force(ctx, state, r); + assert_ctx_ok(); + + auto n = nix_get_attrs_size(ctx, r); + assert_ctx_ok(); + ASSERT_EQ(1, n); + + // nix_get_attr_byname isn't lazy (it could have been) so it will throw the exception + Value * foo = nix_get_attr_byname(ctx, r, state, "foo"); + ASSERT_EQ(nullptr, foo); + ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("error message for test case nix_value_init_apply_lazy_arg")); + + // Clean up + nix_gc_decref(ctx, f); + nix_gc_decref(ctx, e); +} + } From 8c4c2156bd786156a57219e78aa14a363ca8f041 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 19 Apr 2024 15:48:56 +0200 Subject: [PATCH 211/211] doc/glossary: Define output closure (#8311) --- doc/manual/src/glossary.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index c4d9c2a52..66e4628c0 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -215,6 +215,9 @@ [output path]: #gloss-output-path +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + - [deriver]{#gloss-deriver} The [store derivation] that produced an [output path].