From 6d1aa523de8ece115eb0504b2313c12dc5302383 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 May 2023 14:41:47 -0400 Subject: [PATCH 01/78] Create escape hatch for supplementary group sandboxing woes There is no obvious good solution for this that has occured to anyone. --- src/libstore/build/local-derivation-goal.cc | 13 ++++--------- src/libstore/globals.hh | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 21cd6e7ee..6d2d458da 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -907,15 +907,10 @@ void LocalDerivationGoal::startBuilder() openSlave(); /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); + after we've created the new user namespace. */ + if (settings.dropSupplementaryGroups) + if (setgroups(0, 0) == -1) + throw SysError("setgroups failed"); ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 609cf53b8..0b429cf98 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -515,6 +515,25 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; + Setting dropSupplementaryGroups{this, getuid() == 0, "drop-supplementary-groups", + R"( + Whether to drop supplementary groups when building with sandboxing. + This is normally a good idea if we are root and have the capability to + do so. + + But if this "root" is mapped from a non-root user in a larger + namespace, we won't be able drop additional groups; they will be + mapped to nogroup in the child namespace. There does not seem to be a + workaround for this. + + (But who can tell from reading user_namespaces(7)? See also https://lwn.net/Articles/621612/.) + + TODO: It might be good to create a middle ground option that allows + `setgroups` to fail if all additional groups are "nogroup" / the value + of `/proc/sys/fs/overflowuid`. This would handle the common + nested-sandboxing case identified above. + )"}; + #if __linux__ Setting sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", From 2524a2118647a4125dcae08fe0eb20de5f79a291 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 12:38:39 -0400 Subject: [PATCH 02/78] Update src/libstore/build/local-derivation-goal.cc Co-authored-by: Guillaume Girol --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6d2d458da..a50fe1a28 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -910,7 +910,7 @@ void LocalDerivationGoal::startBuilder() after we've created the new user namespace. */ if (settings.dropSupplementaryGroups) if (setgroups(0, 0) == -1) - throw SysError("setgroups failed"); + throw SysError("setgroups failed. Set the drop-supplementary-groups option to false to skip this step."); ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; From d8ef0c949523324615b66059b3d48c4c445f478b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 May 2023 17:41:51 -0400 Subject: [PATCH 03/78] Add some tests for `drop-supplementary-groups` --- tests/common.sh | 2 +- tests/common/vars-and-functions.sh.in | 2 +- tests/hermetic.nix | 56 +++++++++++++++++++++++++++ tests/local.mk | 1 + tests/supplementary-groups.sh | 33 ++++++++++++++++ 5 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 tests/hermetic.nix create mode 100644 tests/supplementary-groups.sh diff --git a/tests/common.sh b/tests/common.sh index 8941671d6..7b0922c9f 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -4,7 +4,7 @@ if [[ -z "${COMMON_SH_SOURCED-}" ]]; then COMMON_SH_SOURCED=1 -source "$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/common/vars-and-functions.sh" +source "$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")/common/vars-and-functions.sh" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then startDaemon fi diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index a9e6c802f..dc7ce13cc 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]}:$LINENO) ' +export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export NIX_STORE_DIR diff --git a/tests/hermetic.nix b/tests/hermetic.nix new file mode 100644 index 000000000..4c9d7a51f --- /dev/null +++ b/tests/hermetic.nix @@ -0,0 +1,56 @@ +{ busybox, seed }: + +with import ./config.nix; + +let + contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1"; + caArgs = if contentAddressedByDefault then { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } else {}; + + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + } // removeAttrs args ["builder" "meta" "passthru"] + // caArgs) + // { meta = args.meta or {}; passthru = args.passthru or {}; }; + + input1 = mkDerivation { + shell = busybox; + name = "hermetic-input-1"; + buildCommand = "echo hi-input1 seed=${toString seed}; echo FOO > $out"; + }; + + input2 = mkDerivation { + shell = busybox; + name = "hermetic-input-2"; + buildCommand = "echo hi; echo BAR > $out"; + }; + + input3 = mkDerivation { + shell = busybox; + name = "hermetic-input-3"; + buildCommand = '' + echo hi-input3 + read x < ${input2} + echo $x BAZ > $out + ''; + }; + +in + + mkDerivation { + shell = busybox; + name = "hermetic"; + passthru = { inherit input1 input2 input3; }; + buildCommand = + '' + read x < ${input1} + read y < ${input3} + echo "$x $y" > $out + ''; + } diff --git a/tests/local.mk b/tests/local.mk index 9e340e2e2..9cb81e1f0 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -93,6 +93,7 @@ nix_tests = \ misc.sh \ dump-db.sh \ linux-sandbox.sh \ + supplementary-groups.sh \ build-dry.sh \ structured-attrs.sh \ shell.sh \ diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh new file mode 100644 index 000000000..fd3da2945 --- /dev/null +++ b/tests/supplementary-groups.sh @@ -0,0 +1,33 @@ +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" +if ! command -p -v unshare; then skipTest "Need unshare"; fi +needLocalStore "The test uses --store always so we would just be bypassing the daemon" + +unshare --mount --map-root-user bash < Date: Mon, 15 May 2023 17:49:04 -0400 Subject: [PATCH 04/78] Avoid out links in supplementary groups test This gets in the way of the tests running in parallel. --- tests/supplementary-groups.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh index fd3da2945..47debc5e3 100644 --- a/tests/supplementary-groups.sh +++ b/tests/supplementary-groups.sh @@ -13,7 +13,7 @@ unshare --mount --map-root-user bash < Date: Mon, 12 Jun 2023 23:42:05 -0300 Subject: [PATCH 05/78] Add test of explicit ssh control path in nix-copy test This highlights a problem caused by SSHMaster::isMasterRunning returning false when NIX_SSHOPTS contains -oControlPath. --- tests/nixos/nix-copy.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 16c477bf9..ef053de03 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -79,6 +79,15 @@ in { server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") server.succeed("systemctl restart sshd") client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") + client.succeed(f"ssh -O check {server.name}") + client.succeed(f"ssh -O exit {server.name}") + client.fail(f"ssh -O check {server.name}") + + # Check that an explicit master will work + client.succeed(f"ssh -MNfS /tmp/master {server.name}") + client.succeed(f"ssh -S /tmp/master -O check {server.name}") + client.succeed("NIX_SSHOPTS='-oControlPath=/tmp/master' nix copy --to ssh://server ${pkgA} >&2") + client.succeed(f"ssh -S /tmp/master -O exit {server.name}") # Copy the closure of package B from the server to the client, using ssh-ng. client.fail("nix-store --check-validity ${pkgB}") From d5e1eb20a2712eb17076c3ca2d30e5ac4a351d00 Mon Sep 17 00:00:00 2001 From: David McFarland Date: Mon, 12 Jun 2023 23:45:53 -0300 Subject: [PATCH 06/78] Pass common ssh options in isMasterRunning --- src/libstore/ssh.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index fae99d75b..da32f1b79 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -42,7 +42,10 @@ void SSHMaster::addCommonSSHOpts(Strings & args) } bool SSHMaster::isMasterRunning() { - auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true}); + Strings args = {"-O", "check", host}; + addCommonSSHOpts(args); + + auto res = runProgram(RunOptions {.program = "ssh", .args = args, .mergeStderrToStdout = true}); return res.first == 0; } From 903700c5e168e2ab5bd6bee5e09b5908cb2908d6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Jul 2023 18:53:44 -0400 Subject: [PATCH 07/78] Simplify `ContentAddress` Whereas `ContentAddressWithReferences` is a sum type complex because different varieties support different notions of reference, and `ContentAddressMethod` is a nested enum to support that, `ContentAddress` can be a simple pair of a method and hash. `ContentAddress` does not need to be a sum type on the outside because the choice of method doesn't effect what type of hashes we can use. Co-Authored-By: Cale Gibbard --- perl/lib/Nix/Store.xs | 6 +- src/libexpr/primops.cc | 13 +- src/libexpr/primops/fetchTree.cc | 6 +- src/libfetchers/fetchers.cc | 6 +- src/libfetchers/tarball.cc | 6 +- src/libstore/binary-cache-store.cc | 16 +-- src/libstore/build/local-derivation-goal.cc | 6 +- src/libstore/content-address.cc | 125 ++++++-------------- src/libstore/content-address.hh | 91 +++++--------- src/libstore/derivations.cc | 22 ++-- src/libstore/local-store.cc | 83 +++++++------ src/libstore/local-store.hh | 8 +- src/libstore/make-content-addressed.cc | 6 +- src/libstore/path-info.cc | 21 ++-- src/libstore/store-api.cc | 28 ++--- src/libstore/tests/derivation.cc | 6 +- src/nix-store/nix-store.cc | 6 +- src/nix/add-to-store.cc | 6 +- src/nix/prefetch.cc | 8 +- src/nix/profile.cc | 6 +- 20 files changed, 182 insertions(+), 293 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 41ecbbeb4..c38ea2d2b 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -294,10 +294,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5dfad470a..a8b4a069f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1300,9 +1300,10 @@ drvName, Bindings * attrs, Value & v) auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); DerivationOutput::CAFixed dof { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(h)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(h), + }, }; drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); @@ -2162,10 +2163,8 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = *expectedHash, - }, + .method = method, + .hash = *expectedHash, .references = {}, }); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 579a45f92..5e668c629 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -254,10 +254,8 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .hash = { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, - .hash = *expectedHash, - }, + .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .hash = *expectedHash, .references = {} }); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 2860c1ceb..f86c0604e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -217,10 +217,8 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = *narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = *narHash, .references = {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e42aca6db..a012234e0 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -77,10 +77,8 @@ DownloadFileResult downloadFile( *store, name, FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Flat, - .hash = hash, - }, + .method = FileIngestionMethod::Flat, + .hash = hash, .references = {}, }, hashString(htSHA256, sink.s), diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index fcd763a9d..b4fea693f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -309,10 +309,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = nar.first, - }, + .method = method, + .hash = nar.first, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -428,10 +426,8 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -465,8 +461,8 @@ StorePath BinaryCacheStore::addTextToStore( *this, std::string { name }, TextInfo { - { .hash = textHash }, - references, + .hash = textHash, + .references = references, }, nar.first, }; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ee66ee500..7b98a8601 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2538,16 +2538,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }, [&](const DerivationOutput::CAFixed & dof) { - auto wanted = dof.ca.getHash(); + auto & wanted = dof.ca.hash; auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.ca.getMethod(), + .method = dof.ca.method, .hashType = wanted.type, }); /* Check wanted hash */ assert(newInfo0.ca); - auto got = newInfo0.ca->getHash(); + auto & got = newInfo0.ca->hash; if (wanted != got) { /* Throw an error after registering the path as valid. */ diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 04f7ac214..080456e18 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,11 +4,6 @@ namespace nix { -std::string FixedOutputHash::printMethodAlgo() const -{ - return makeFileIngestionPrefix(method) + printHashType(hash.type); -} - std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { @@ -42,21 +37,6 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) return method; } -std::string ContentAddress::render() const -{ - return std::visit(overloaded { - [](const TextHash & th) { - return "text:" - + th.hash.to_string(Base32, true); - }, - [](const FixedOutputHash & fsh) { - return "fixed:" - + makeFileIngestionPrefix(fsh.method) - + fsh.hash.to_string(Base32, true); - } - }, raw); -} - std::string ContentAddressMethod::render(HashType ht) const { return std::visit(overloaded { @@ -69,6 +49,20 @@ std::string ContentAddressMethod::render(HashType ht) const }, raw); } +std::string ContentAddress::render() const +{ + return std::visit(overloaded { + [](const TextIngestionMethod &) -> std::string { + return "text:"; + }, + [](const FileIngestionMethod & method) { + return "fixed:" + + makeFileIngestionPrefix(method); + }, + }, method.raw) + + this->hash.to_string(Base32, true); +} + /** * Parses content address strings up to the hash. */ @@ -118,22 +112,12 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; - auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest); - auto hashType = hashType_; // work around clang bug + auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); - return std::visit(overloaded { - [&](TextIngestionMethod &) { - return ContentAddress(TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, hashType) - }); - }, - [&](FileIngestionMethod & fim) { - return ContentAddress(FixedOutputHash { - .method = fim, - .hash = Hash::parseNonSRIUnprefixed(rest, hashType), - }); - }, - }, caMethod.raw); + return ContentAddress { + .method = std::move(caMethod).raw, + .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + }; } std::pair ContentAddressMethod::parse(std::string_view caMethod) @@ -156,52 +140,10 @@ std::string renderContentAddress(std::optional ca) return ca ? ca->render() : ""; } -ContentAddress ContentAddress::fromParts( - ContentAddressMethod method, Hash hash) noexcept -{ - return std::visit(overloaded { - [&](TextIngestionMethod _) -> ContentAddress { - return TextHash { - .hash = std::move(hash), - }; - }, - [&](FileIngestionMethod m2) -> ContentAddress { - return FixedOutputHash { - .method = std::move(m2), - .hash = std::move(hash), - }; - }, - }, method.raw); -} - -ContentAddressMethod ContentAddress::getMethod() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> ContentAddressMethod { - return TextIngestionMethod {}; - }, - [](const FixedOutputHash & fsh) -> ContentAddressMethod { - return fsh.method; - }, - }, raw); -} - -const Hash & ContentAddress::getHash() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> auto & { - return th.hash; - }, - [](const FixedOutputHash & fsh) -> auto & { - return fsh.hash; - }, - }, raw); -} - std::string ContentAddress::printMethodAlgo() const { - return getMethod().renderPrefix() - + printHashType(getHash().type); + return method.renderPrefix() + + printHashType(hash.type); } bool StoreReferences::empty() const @@ -217,19 +159,20 @@ size_t StoreReferences::size() const ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept { return std::visit(overloaded { - [&](const TextHash & h) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { return TextInfo { - .hash = h, + .hash = ca.hash, .references = {}, }; }, - [&](const FixedOutputHash & h) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & method) -> ContentAddressWithReferences { return FixedOutputInfo { - .hash = h, + .method = method, + .hash = ca.hash, .references = {}, }; }, - }, ca.raw); + }, ca.method.raw); } std::optional ContentAddressWithReferences::fromPartsOpt( @@ -241,7 +184,7 @@ std::optional ContentAddressWithReferences::fromPa return std::nullopt; return ContentAddressWithReferences { TextInfo { - .hash = { .hash = std::move(hash) }, + .hash = std::move(hash), .references = std::move(refs.others), } }; @@ -249,10 +192,8 @@ std::optional ContentAddressWithReferences::fromPa [&](FileIngestionMethod m2) -> std::optional { return ContentAddressWithReferences { FixedOutputInfo { - .hash = { - .method = m2, - .hash = std::move(hash), - }, + .method = m2, + .hash = std::move(hash), .references = std::move(refs), } }; @@ -267,7 +208,7 @@ ContentAddressMethod ContentAddressWithReferences::getMethod() const return TextIngestionMethod {}; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.hash.method; + return fsh.method; }, }, raw); } @@ -276,10 +217,10 @@ Hash ContentAddressWithReferences::getHash() const { return std::visit(overloaded { [](const TextInfo & th) { - return th.hash.hash; + return th.hash; }, [](const FixedOutputInfo & fsh) { - return fsh.hash.hash; + return fsh.hash; }, }, raw); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index e1e80448b..01b771e52 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -113,37 +113,6 @@ struct ContentAddressMethod * Mini content address */ -/** - * Somewhat obscure, used by \ref Derivation derivations and - * `builtins.toFile` currently. - */ -struct TextHash { - /** - * Hash of the contents of the text/file. - */ - Hash hash; - - GENERATE_CMP(TextHash, me->hash); -}; - -/** - * Used by most store objects that are content-addressed. - */ -struct FixedOutputHash { - /** - * How the file system objects are serialized - */ - FileIngestionMethod method; - /** - * Hash of that serialization - */ - Hash hash; - - std::string printMethodAlgo() const; - - GENERATE_CMP(FixedOutputHash, me->method, me->hash); -}; - /** * We've accumulated several types of content-addressed paths over the * years; fixed-output derivations support multiple hash algorithms and @@ -158,19 +127,17 @@ struct FixedOutputHash { */ struct ContentAddress { - typedef std::variant< - TextHash, - FixedOutputHash - > Raw; + /** + * How the file system objects are serialized + */ + ContentAddressMethod method; - Raw raw; + /** + * Hash of that serialization + */ + Hash hash; - GENERATE_CMP(ContentAddress, me->raw); - - /* The moral equivalent of `using Raw::Raw;` */ - ContentAddress(auto &&... arg) - : raw(std::forward(arg)...) - { } + GENERATE_CMP(ContentAddress, me->method, me->hash); /** * Compute the content-addressability assertion @@ -183,20 +150,6 @@ struct ContentAddress static std::optional parseOpt(std::string_view rawCaOpt); - /** - * Create a `ContentAddress` from 2 parts: - * - * @param method Way ingesting the file system data. - * - * @param hash Hash of ingested file system data. - */ - static ContentAddress fromParts( - ContentAddressMethod method, Hash hash) noexcept; - - ContentAddressMethod getMethod() const; - - const Hash & getHash() const; - std::string printMethodAlgo() const; }; @@ -219,7 +172,8 @@ std::string renderContentAddress(std::optional ca); * References to other store objects are tracked with store paths, self * references however are tracked with a boolean. */ -struct StoreReferences { +struct StoreReferences +{ /** * References to other store objects */ @@ -246,8 +200,13 @@ struct StoreReferences { }; // This matches the additional info that we need for makeTextPath -struct TextInfo { - TextHash hash; +struct TextInfo +{ + /** + * Hash of the contents of the text/file. + */ + Hash hash; + /** * References to other store objects only; self references * disallowed @@ -257,8 +216,18 @@ struct TextInfo { GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo { - FixedOutputHash hash; +struct FixedOutputInfo +{ + /** + * How the file system objects are serialized + */ + FileIngestionMethod method; + + /** + * Hash of that serialization + */ + Hash hash; + /** * References to other store objects or this one. */ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6f63685d4..b831b2fe5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -232,9 +232,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, validatePath(pathS); auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); return DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(hash)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(hash), + }, }; } else { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -395,7 +396,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -628,7 +629,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut auto & dof = std::get(i.second.raw()); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.getHash().to_string(Base16, false) + ":" + + dof.ca.hash.to_string(Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -780,7 +781,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.getHash().to_string(Base16, false); + << dof.ca.hash.to_string(Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -970,7 +971,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.getHash().to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { @@ -1017,9 +1018,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "path", "hashAlgo", "hash" })) { auto [method, hashType] = methodAlgo(); auto dof = DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)), + .ca = ContentAddress { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, }; if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) throw Error("Path doesn't match derivation output"); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e69460e6c..c468cb51e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1249,27 +1249,17 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&info.ca->raw)) { - auto actualFoHash = hashCAPath( - foHash->method, - foHash->hash.type, - info.path - ); - if (foHash->hash != actualFoHash.hash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - foHash->hash.to_string(Base32, true), - actualFoHash.hash.to_string(Base32, true)); - } - } - if (auto textHash = std::get_if(&info.ca->raw)) { - auto actualTextHash = hashString(htSHA256, readFile(realPath)); - if (textHash->hash != actualTextHash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - textHash->hash.to_string(Base32, true), - actualTextHash.to_string(Base32, true)); - } + auto & specified = *info.ca; + auto actualHash = hashCAPath( + specified.method, + specified.hash.type, + info.path + ); + if (specified.hash != actualHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + specified.hash.to_string(Base32, true), + actualHash.hash.to_string(Base32, true)); } } @@ -1349,10 +1339,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -1428,8 +1416,8 @@ StorePath LocalStore::addTextToStore( { auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, TextInfo { - { .hash = hash }, - references, + .hash = hash, + .references = references, }); addTempRoot(dstPath); @@ -1459,7 +1447,10 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); info.references = references; - info.ca = TextHash { .hash = hash }; + info.ca = { + .method = TextIngestionMethod {}, + .hash = hash, + }; registerValidPath(info); } @@ -1856,33 +1847,39 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, const HashType & hashType, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path) { return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash ) { HashModuloSink caSink ( hashType, std::string(pathHash) ); - switch (method) { - case FileIngestionMethod::Recursive: - dumpPath(path, caSink); - break; - case FileIngestionMethod::Flat: - readFile(path, caSink); - break; - } - auto hash = caSink.finish().first; - return FixedOutputHash{ + std::visit(overloaded { + [&](const TextIngestionMethod &) { + readFile(path, caSink); + }, + [&](const FileIngestionMethod & m2) { + switch (m2) { + case FileIngestionMethod::Recursive: + dumpPath(path, caSink); + break; + case FileIngestionMethod::Flat: + readFile(path, caSink); + break; + } + }, + }, method.raw); + return ContentAddress { .method = method, - .hash = hash, + .hash = caSink.finish().first, }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8a3b0b43f..d76662f33 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -345,13 +345,13 @@ private: void signRealisation(Realisation &); // XXX: Make a generic `Store` method - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path); - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 53fe04704..349c2f187 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,10 +52,8 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, .references = std::move(refs), }, Hash::dummy, diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 981bbfb14..ccb57104f 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -29,14 +29,14 @@ std::optional ValidPathInfo::contentAddressWithRef return std::nullopt; return std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - .hash = th, + .hash = ca->hash, .references = references, }; }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { @@ -44,14 +44,15 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - .hash = foh, + .method = m2, + .hash = ca->hash, .references = { .others = std::move(refs), .self = hasSelfReference, }, }; }, - }, ca->raw); + }, ca->method.raw); } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -110,13 +111,19 @@ ValidPathInfo::ValidPathInfo( std::visit(overloaded { [this](TextInfo && ti) { this->references = std::move(ti.references); - this->ca = std::move((TextHash &&) ti); + this->ca = ContentAddress { + .method = TextIngestionMethod {}, + .hash = std::move(ti.hash), + }; }, [this](FixedOutputInfo && foi) { this->references = std::move(foi.references.others); if (foi.references.self) this->references.insert(path); - this->ca = std::move((FixedOutputHash &&) foi); + this->ca = ContentAddress { + .method = std::move(foi.method), + .hash = std::move(foi.hash), + }; }, }, std::move(ca).raw); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5bee1af9f..b16f5a6b7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -184,15 +184,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.hash.method) - + info.hash.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } @@ -200,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.hash.type == htSHA256); + assert(info.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash.hash, + info.hash, name); } @@ -232,10 +232,8 @@ std::pair Store::computeStorePathForPath(std::string_view name, ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); FixedOutputInfo caInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); @@ -248,8 +246,8 @@ StorePath Store::computeStorePathForText( const StorePathSet & references) const { return makeTextPath(name, TextInfo { - { .hash = hashString(htSHA256, s) }, - references, + .hash = hashString(htSHA256, s), + .references = references, }); } @@ -441,10 +439,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = {}, }, narHash, diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 6328ad370..0e28c1f08 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -81,7 +81,7 @@ TEST_JSON(DerivationTest, caFixedFlat, "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Flat, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -95,7 +95,7 @@ TEST_JSON(DerivationTest, caFixedNAR, "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Recursive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -109,7 +109,7 @@ TEST_JSON(DynDerivationTest, caFixedText, "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = TextHash { + .ca = { .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index caa0248f1..94956df66 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,10 +220,8 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = Hash::parseAny(hash, hashAlgo), - }, + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 16e48a39b..39e5cc99d 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -45,10 +45,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand *store, std::move(*namePart), FixedOutputInfo { - .hash = { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - }, + .method = std::move(ingestionMethod), + .hash = std::move(hash), .references = {}, }, narHash, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3b2e225f6..b67d381ca 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -71,10 +71,8 @@ std::tuple prefetchFile( if (expectedHash) { hashType = expectedHash->type; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - .hash = { - .method = ingestionMethod, - .hash = *expectedHash, - }, + .method = ingestionMethod, + .hash = *expectedHash, .references = {}, }); if (store->isValidPath(*storePath)) @@ -127,7 +125,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = info.ca->getHash(); + hash = info.ca->hash; } return {storePath.value(), hash.value()}; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b833b5192..476ddcd60 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -222,10 +222,8 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narHash, .references = { .others = std::move(references), // profiles never refer to themselves From 87dcd090470ed6e56a2744cbe1490d2cf235d5c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Jun 2023 12:31:09 -0400 Subject: [PATCH 08/78] Clean up `resolveSearchPathElem` We should use `std::optional` not `std::pair` for an optional string. --- src/libexpr/eval.cc | 14 +++++++------- src/libexpr/eval.hh | 8 ++++++-- src/libexpr/parser.y | 45 +++++++++++++++++++++++--------------------- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 706a19024..97a264085 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -571,22 +571,22 @@ EvalState::EvalState( allowedPaths = PathSet(); for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; + auto r = resolveSearchPathElem(i.path); + if (!r) continue; - auto path = r.second; + auto path = *std::move(r); - if (store->isInStore(r.second)) { + if (store->isInStore(path)) { try { StorePathSet closure; - store->computeFSClosure(store->toStorePath(r.second).first, closure); + store->computeFSClosure(store->toStorePath(path).first, closure); for (auto & path : closure) allowPath(path); } catch (InvalidPath &) { - allowPath(r.second); + allowPath(path); } } else - allowPath(r.second); + allowPath(path); } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e3676c1b7..e1a540a7f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -317,7 +317,7 @@ private: SearchPath searchPath; - std::map> searchPathResolved; + std::map> searchPathResolved; /** * Cache used by checkSourcePath(). @@ -434,9 +434,13 @@ public: SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** + * Try to resolve a search path value (not the optinal key part) + * * If the specified search path element is a URI, download it. + * + * If it is not found, return `std::nullopt` */ - std::pair resolveSearchPathElem(const SearchPathElem & elem); + std::optional resolveSearchPathElem(const std::string & value); /** * Evaluate an expression to normal form diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0d0004f9f..f839e804c 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -772,9 +772,9 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p continue; suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); } - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - Path res = r.second + suffix; + auto r = resolveSearchPathElem(i.path); + if (!r) continue; + Path res = *r + suffix; if (pathExists(res)) return CanonPath(canonPath(res)); } @@ -791,49 +791,52 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p } -std::pair EvalState::resolveSearchPathElem(const SearchPathElem & elem) +std::optional EvalState::resolveSearchPathElem(const std::string & value) { - auto i = searchPathResolved.find(elem.path); + auto i = searchPathResolved.find(value); if (i != searchPathResolved.end()) return i->second; - std::pair res; + std::optional res; - if (EvalSettings::isPseudoUrl(elem.path)) { + if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath; - res = { true, store->toRealPath(storePath) }; + store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - else if (hasPrefix(elem.path, "flake:")) { + else if (hasPrefix(value, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", elem.path); + auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", value); auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; - res = { true, store->toRealPath(storePath) }; + res = { store->toRealPath(storePath) }; } else { - auto path = absPath(elem.path); + auto path = absPath(value); if (pathExists(path)) - res = { true, path }; + res = { path }; else { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - debug("resolved search path element '%s' to '%s'", elem.path, res.second); + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + else + debug("failed to resolve search path element '%s'", value); - searchPathResolved[elem.path] = res; + searchPathResolved[value] = res; return res; } From be518e73ae331ac2f46e6b3a0ffdfeead26e3186 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Jun 2023 13:51:25 -0400 Subject: [PATCH 09/78] Clean up `SearchPath` - Better types - Own header / C++ file pair - Test factored out methods - Pass parsed thing around more than strings Co-authored-by: Robert Hensing --- src/libcmd/common-eval-args.cc | 4 +- src/libcmd/common-eval-args.hh | 3 +- src/libcmd/repl.cc | 8 +-- src/libcmd/repl.hh | 2 +- src/libexpr/eval.cc | 12 ++-- src/libexpr/eval.hh | 18 ++---- src/libexpr/parser.y | 47 +++++--------- src/libexpr/primops.cc | 14 ++-- src/libexpr/search-path.cc | 56 ++++++++++++++++ src/libexpr/search-path.hh | 108 +++++++++++++++++++++++++++++++ src/libexpr/tests/search-path.cc | 90 ++++++++++++++++++++++++++ src/nix/upgrade-nix.cc | 2 +- 12 files changed, 300 insertions(+), 64 deletions(-) create mode 100644 src/libexpr/search-path.cc create mode 100644 src/libexpr/search-path.hh create mode 100644 src/libexpr/tests/search-path.cc diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 7f97364a1..3df2c71a5 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -105,7 +105,9 @@ MixEvalArgs::MixEvalArgs() )", .category = category, .labels = {"path"}, - .handler = {[&](std::string s) { searchPath.push_back(s); }} + .handler = {[&](std::string s) { + searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + }} }); addFlag({ diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index b65cb5b20..6359b2579 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -3,6 +3,7 @@ #include "args.hh" #include "common-args.hh" +#include "search-path.hh" namespace nix { @@ -19,7 +20,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - Strings searchPath; + SearchPath searchPath; std::optional evalStoreUrl; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 4b160a100..f9e9c2bf8 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -68,7 +68,7 @@ struct NixRepl const Path historyFile; - NixRepl(const Strings & searchPath, nix::ref store,ref state, + NixRepl(const SearchPath & searchPath, nix::ref store,ref state, std::function getValues); virtual ~NixRepl(); @@ -104,7 +104,7 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const Strings & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) @@ -1024,7 +1024,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m std::unique_ptr AbstractNixRepl::create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( @@ -1044,7 +1044,7 @@ void AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - const Strings & searchPath = {}; + SearchPath searchPath = {}; auto repl = std::make_unique( searchPath, openStore(), diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 731c8e6db..6d88883fe 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -25,7 +25,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues); static void runSimple( diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 97a264085..be1bdb806 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -498,7 +498,7 @@ ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) EvalState::EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore) : sWith(symbols.create("")) @@ -563,15 +563,17 @@ EvalState::EvalState( /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); + for (auto & i : _searchPath.elements) + addToSearchPath(SearchPath::Elem {i}); + for (auto & i : evalSettings.nixPath.get()) + addToSearchPath(SearchPath::Elem::parse(i)); } if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i.path); + for (auto & i : searchPath.elements) { + auto r = resolveSearchPathPath(i.path); if (!r) continue; auto path = *std::move(r); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e1a540a7f..277e77ad5 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -9,6 +9,7 @@ #include "config.hh" #include "experimental-features.hh" #include "input-accessor.hh" +#include "search-path.hh" #include #include @@ -122,15 +123,6 @@ std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -struct SearchPathElem -{ - std::string prefix; - // FIXME: maybe change this to an std::variant. - std::string path; -}; -typedef std::list SearchPath; - - /** * Initialise the Boehm GC, if applicable. */ @@ -344,12 +336,12 @@ private: public: EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore = nullptr); ~EvalState(); - void addToSearchPath(const std::string & s); + void addToSearchPath(SearchPath::Elem && elem); SearchPath getSearchPath() { return searchPath; } @@ -431,7 +423,7 @@ public: * Look up a file in the search path. */ SourcePath findFile(const std::string_view path); - SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); + SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** * Try to resolve a search path value (not the optinal key part) @@ -440,7 +432,7 @@ public: * * If it is not found, return `std::nullopt` */ - std::optional resolveSearchPathElem(const std::string & value); + std::optional resolveSearchPathPath(const SearchPath::Path & path); /** * Evaluate an expression to normal form diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index f839e804c..77bcc31e0 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -734,22 +734,9 @@ Expr * EvalState::parseStdin() } -void EvalState::addToSearchPath(const std::string & s) +void EvalState::addToSearchPath(SearchPath::Elem && elem) { - size_t pos = s.find('='); - std::string prefix; - Path path; - if (pos == std::string::npos) { - path = s; - } else { - prefix = std::string(s, 0, pos); - path = std::string(s, pos + 1); - } - - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, - }); + searchPath.elements.emplace_back(std::move(elem)); } @@ -759,22 +746,19 @@ SourcePath EvalState::findFile(const std::string_view path) } -SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) +SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) { - for (auto & i : searchPath) { - std::string suffix; - if (i.prefix.empty()) - suffix = concatStrings("/", path); - else { - auto s = i.prefix.size(); - if (path.compare(0, s, i.prefix) != 0 || - (path.size() > s && path[s] != '/')) - continue; - suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); - } - auto r = resolveSearchPathElem(i.path); - if (!r) continue; - Path res = *r + suffix; + for (auto & i : searchPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveSearchPathPath(i.path); + if (!rOpt) continue; + auto r = *rOpt; + + Path res = suffix == "" ? r : concatStrings(r, "/", suffix); if (pathExists(res)) return CanonPath(canonPath(res)); } @@ -791,8 +775,9 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p } -std::optional EvalState::resolveSearchPathElem(const std::string & value) +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) { + auto & value = value0.s; auto i = searchPathResolved.find(value); if (i != searchPathResolved.end()) return i->second; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5dfad470a..b98b06db9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1656,9 +1656,9 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V })); } - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, + searchPath.elements.emplace_back(SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = prefix }, + .path = SearchPath::Path { .s = path }, }); } @@ -4319,12 +4319,12 @@ void EvalState::createBaseEnv() }); /* Add a value containing the current Nix expression search path. */ - mkList(v, searchPath.size()); + mkList(v, searchPath.elements.size()); int n = 0; - for (auto & i : searchPath) { + for (auto & i : searchPath.elements) { auto attrs = buildBindings(2); - attrs.alloc("path").mkString(i.path); - attrs.alloc("prefix").mkString(i.prefix); + attrs.alloc("path").mkString(i.path.s); + attrs.alloc("prefix").mkString(i.prefix.s); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } addConstant("__nixPath", v, { diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc new file mode 100644 index 000000000..36bb4c3a5 --- /dev/null +++ b/src/libexpr/search-path.cc @@ -0,0 +1,56 @@ +#include "search-path.hh" +#include "util.hh" + +namespace nix { + +std::optional SearchPath::Prefix::suffixIfPotentialMatch( + std::string_view path) const +{ + auto n = s.size(); + + /* Non-empty prefix and suffix must be separated by a /, or the + prefix is not a valid path prefix. */ + bool needSeparator = n > 0 && (path.size() - n) > 0; + + if (needSeparator && path[n] != '/') { + return std::nullopt; + } + + /* Prefix must be prefix of this path. */ + if (path.compare(0, n, s) != 0) { + return std::nullopt; + } + + /* Skip next path separator. */ + return { + path.substr(needSeparator ? n + 1 : n) + }; +} + + +SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem) +{ + size_t pos = rawElem.find('='); + + return SearchPath::Elem { + .prefix = Prefix { + .s = pos == std::string::npos + ? std::string { "" } + : std::string { rawElem.substr(0, pos) }, + }, + .path = Path { + .s = std::string { rawElem.substr(pos + 1) }, + }, + }; +} + + +SearchPath parseSearchPath(const Strings & rawElems) +{ + SearchPath res; + for (auto & rawElem : rawElems) + res.elements.emplace_back(SearchPath::Elem::parse(rawElem)); + return res; +} + +} diff --git a/src/libexpr/search-path.hh b/src/libexpr/search-path.hh new file mode 100644 index 000000000..ce78135b5 --- /dev/null +++ b/src/libexpr/search-path.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "comparator.hh" + +namespace nix { + +/** + * A "search path" is a list of ways look for something, used with + * `builtins.findFile` and `< >` lookup expressions. + */ +struct SearchPath +{ + /** + * A single element of a `SearchPath`. + * + * Each element is tried in succession when looking up a path. The first + * element to completely match wins. + */ + struct Elem; + + /** + * The first part of a `SearchPath::Elem` pair. + * + * Called a "prefix" because it takes the form of a prefix of a file + * path (first `n` path components). When looking up a path, to use + * a `SearchPath::Elem`, its `Prefix` must match the path. + */ + struct Prefix; + + /** + * The second part of a `SearchPath::Elem` pair. + * + * It is either a path or a URL (with certain restrictions / extra + * structure). + * + * If the prefix of the path we are looking up matches, we then + * check if the rest of the path points to something that exists + * within the directory denoted by this. If so, the + * `SearchPath::Elem` as a whole matches, and that *something* being + * pointed to by the rest of the path we are looking up is the + * result. + */ + struct Path; + + /** + * The list of search path elements. Each one is checked for a path + * when looking up. (The actual lookup entry point is in `EvalState` + * not in this class.) + */ + std::list elements; + + /** + * Parse a string into a `SearchPath` + */ + static SearchPath parse(const Strings & rawElems); +}; + +struct SearchPath::Prefix +{ + /** + * Underlying string + * + * @todo Should we normalize this when constructing a `SearchPath::Prefix`? + */ + std::string s; + + GENERATE_CMP(SearchPath::Prefix, me->s); + + /** + * If the path possibly matches this search path element, return the + * suffix that we should look for inside the resolved value of the + * element + * Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist. + */ + std::optional suffixIfPotentialMatch(std::string_view path) const; +}; + +struct SearchPath::Path +{ + /** + * The location of a search path item, as a path or URL. + * + * @todo Maybe change this to `std::variant`. + */ + std::string s; + + GENERATE_CMP(SearchPath::Path, me->s); +}; + +struct SearchPath::Elem +{ + + Prefix prefix; + Path path; + + GENERATE_CMP(SearchPath::Elem, me->prefix, me->path); + + /** + * Parse a string into a `SearchPath::Elem` + */ + static SearchPath::Elem parse(std::string_view rawElem); +}; + +} diff --git a/src/libexpr/tests/search-path.cc b/src/libexpr/tests/search-path.cc new file mode 100644 index 000000000..dbe7ab95f --- /dev/null +++ b/src/libexpr/tests/search-path.cc @@ -0,0 +1,90 @@ +#include +#include + +#include "search-path.hh" + +namespace nix { + +TEST(SearchPathElem, parse_justPath) { + ASSERT_EQ( + SearchPath::Elem::parse("foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_emptyPrefix) { + ASSERT_EQ( + SearchPath::Elem::parse("=foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_oneEq) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar" }, + })); +} + +TEST(SearchPathElem, parse_twoEqs) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar=baz"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar=baz" }, + })); +} + + +TEST(SearchPathElem, suffixIfPotentialMatch_justPath) { + SearchPath::Prefix prefix { .s = "" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) { + SearchPath::Prefix prefix { .s = "fooX" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) { + SearchPath::Prefix prefix { .s = "foo/bar" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" }); +} + +} diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 3997c98bf..d05c23fb7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -146,7 +146,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest(storePathsUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(Strings(), store); + auto state = std::make_unique(SearchPath{}, store); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); From c2c8187118c4bf2961aa175ff8667e1402a767c7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 21 Jun 2023 23:15:24 -0400 Subject: [PATCH 10/78] Fix test file name It's UTF-8, not UFT-8. --- tests/lang/{parse-fail-uft8.nix => parse-fail-utf8.nix} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/lang/{parse-fail-uft8.nix => parse-fail-utf8.nix} (100%) diff --git a/tests/lang/parse-fail-uft8.nix b/tests/lang/parse-fail-utf8.nix similarity index 100% rename from tests/lang/parse-fail-uft8.nix rename to tests/lang/parse-fail-utf8.nix From 07dabcc90ed8f2a2e7b98d858a47de3e75d2c3a2 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Jul 2023 10:44:03 +0100 Subject: [PATCH 11/78] Always attempt setgroups but allow failure to be ignored. --- src/libstore/build/local-derivation-goal.cc | 9 ++++++--- src/libstore/globals.hh | 2 +- tests/supplementary-groups.sh | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 53e6998e8..068b47f93 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -909,9 +909,12 @@ void LocalDerivationGoal::startBuilder() /* Drop additional groups here because we can't do it after we've created the new user namespace. */ - if (settings.dropSupplementaryGroups) - if (setgroups(0, 0) == -1) - throw SysError("setgroups failed. Set the drop-supplementary-groups option to false to skip this step."); + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a19b43086..dbabf116a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -524,7 +524,7 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting dropSupplementaryGroups{this, getuid() == 0, "drop-supplementary-groups", + Setting requireDropSupplementaryGroups{this, true, "require-drop-supplementary-groups", R"( Whether to drop supplementary groups when building with sandboxing. This is normally a good idea if we are root and have the capability to diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh index 47debc5e3..47c6ef605 100644 --- a/tests/supplementary-groups.sh +++ b/tests/supplementary-groups.sh @@ -20,14 +20,14 @@ unshare --mount --map-root-user bash < Date: Tue, 11 Jul 2023 10:57:03 +0100 Subject: [PATCH 12/78] Update description for require-drop-supplementary-groups. --- src/libstore/globals.hh | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index dbabf116a..601626d00 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -526,21 +526,15 @@ public: Setting requireDropSupplementaryGroups{this, true, "require-drop-supplementary-groups", R"( - Whether to drop supplementary groups when building with sandboxing. - This is normally a good idea if we are root and have the capability to - do so. + Following the principle of least privilege, + Nix will attempt to drop supplementary groups when building with sandboxing. - But if this "root" is mapped from a non-root user in a larger - namespace, we won't be able drop additional groups; they will be - mapped to nogroup in the child namespace. There does not seem to be a - workaround for this. + However this can fail under some circumstances. + For example, if the user lacks the CAP_SETGID capability. + Search setgroups(2) for EPERM to find more detailed information on this. - (But who can tell from reading user_namespaces(7)? See also https://lwn.net/Articles/621612/.) - - TODO: It might be good to create a middle ground option that allows - `setgroups` to fail if all additional groups are "nogroup" / the value - of `/proc/sys/fs/overflowuid`. This would handle the common - nested-sandboxing case identified above. + If you encounter such a failure, + you can instruct Nix to continue without dropping supplementary groups by setting this option to `false`. )"}; #if __linux__ From 2b4c59dd997c72069b6039783fea4c3b35f5cee7 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Jul 2023 11:09:25 +0100 Subject: [PATCH 13/78] Be clearer about the security implications. --- src/libstore/globals.hh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 601626d00..dec132ff0 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -533,8 +533,9 @@ public: For example, if the user lacks the CAP_SETGID capability. Search setgroups(2) for EPERM to find more detailed information on this. - If you encounter such a failure, - you can instruct Nix to continue without dropping supplementary groups by setting this option to `false`. + If you encounter such a failure, setting this option to `false` will let you ignore it and continue. + But before doing so, you should consider the security implications carefully. + Not dropping supplementary groups means the build sandbox will be less restricted than intended. )"}; #if __linux__ From a193ec4052d9efa895681c438cc335296c7affea Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Jul 2023 11:13:39 +0100 Subject: [PATCH 14/78] Default should depend on whether we are root. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index dec132ff0..9a9b4903f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -524,7 +524,7 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting requireDropSupplementaryGroups{this, true, "require-drop-supplementary-groups", + Setting requireDropSupplementaryGroups{this, getuid() == 0, "require-drop-supplementary-groups", R"( Following the principle of least privilege, Nix will attempt to drop supplementary groups when building with sandboxing. From b8e8dfc3e8581a08bc179f75fbe61e04030088de Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Jul 2023 11:24:11 +0100 Subject: [PATCH 15/78] Say a bit about default value in setting description. --- src/libstore/globals.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9a9b4903f..81fa154bb 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -536,6 +536,10 @@ public: If you encounter such a failure, setting this option to `false` will let you ignore it and continue. But before doing so, you should consider the security implications carefully. Not dropping supplementary groups means the build sandbox will be less restricted than intended. + + This option defaults to `true` when the user is root + (since root usually has permissions to call setgroups) + and `false` otherwise. )"}; #if __linux__ From c1d39de1fbceebbb6b46abc4a21fb8e34423df99 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 12:48:06 +0100 Subject: [PATCH 16/78] Check _NIX_TEST_NO_SANDBOX when setting _canUseSandbox. --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index dc7ce13cc..ad0623871 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -141,7 +141,7 @@ restartDaemon() { startDaemon } -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then +if [[ -z "${_NIX_TEST_NO_SANDBOX:-}" ]] && [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then _canUseSandbox=1 fi From 41412dc4ae0fec2d08335c19724276d99e0c6056 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Thu, 15 Jun 2023 12:58:59 +0100 Subject: [PATCH 17/78] Skip build-remote-trustless unless sandbox is supported. --- tests/build-remote-trustless-should-fail-0.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh index fad1def59..b14101f83 100644 --- a/tests/build-remote-trustless-should-fail-0.sh +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -2,6 +2,7 @@ source common.sh enableFeatures "daemon-trust-override" +requireSandboxSupport restartDaemon [[ $busybox =~ busybox ]] || skipTest "no busybox" From c70484454f52a3bba994ac0c155b4a6f35a5013c Mon Sep 17 00:00:00 2001 From: Mathnerd314 Date: Fri, 4 Sep 2015 14:23:08 -0600 Subject: [PATCH 18/78] Expanded test suite * Lang now verifies errors and parse output * Some new miscellaneous tests * Easy way to update the tests * Document workflow in manual * Use `!` not `~` as separater char for sed It is confusing to use `~` when we are talking about paths and home directories! * Test test suite itself (`test/lang-test/infra.sh`) Additionally, run shellcheck on `tests/lang.sh` to help ensure it is correct, now that is is more complex. Co-authored-by: Robert Hensing Co-authored-by: Valentin Gagarin --- .gitignore | 1 + doc/manual/src/contributing/testing.md | 25 ++++ tests/dependencies.sh | 3 + tests/lang-test-infra.sh | 86 ++++++++++++ tests/lang.sh | 123 +++++++++++++----- tests/lang/empty.exp | 0 tests/lang/eval-fail-abort.err.exp | 10 ++ tests/lang/eval-fail-antiquoted-path.err.exp | 1 + tests/lang/eval-fail-assert.err.exp | 36 +++++ tests/lang/eval-fail-bad-antiquote-1.err.exp | 10 ++ tests/lang/eval-fail-bad-antiquote-2.err.exp | 1 + tests/lang/eval-fail-bad-antiquote-3.err.exp | 10 ++ ...al-fail-bad-string-interpolation-1.err.exp | 10 ++ ...al-fail-bad-string-interpolation-2.err.exp | 1 + ...al-fail-bad-string-interpolation-3.err.exp | 10 ++ tests/lang/eval-fail-blackhole.err.exp | 18 +++ tests/lang/eval-fail-deepseq.err.exp | 26 ++++ ...-foldlStrict-strict-op-application.err.exp | 38 ++++++ .../eval-fail-fromTOML-timestamps.err.exp | 12 ++ tests/lang/eval-fail-hashfile-missing.err.exp | 19 +++ tests/lang/eval-fail-list.err.exp | 10 ++ tests/lang/eval-fail-list.nix | 1 + tests/lang/eval-fail-missing-arg.err.exp | 16 +++ tests/lang/eval-fail-nonexist-path.err.exp | 1 + tests/lang/eval-fail-path-slash.err.exp | 8 ++ tests/lang/eval-fail-recursion.err.exp | 16 +++ tests/lang/eval-fail-recursion.nix | 1 + tests/lang/eval-fail-remove.err.exp | 19 +++ tests/lang/eval-fail-scope-5.err.exp | 36 +++++ tests/lang/eval-fail-seq.err.exp | 18 +++ tests/lang/eval-fail-set-override.err.exp | 6 + tests/lang/eval-fail-set-override.nix | 1 + tests/lang/eval-fail-set.err.exp | 7 + tests/lang/eval-fail-set.nix | 1 + tests/lang/eval-fail-substring.err.exp | 12 ++ tests/lang/eval-fail-to-path.err.exp | 14 ++ tests/lang/eval-fail-undeclared-arg.err.exp | 17 +++ tests/lang/eval-okay-fromjson.nix | 14 +- tests/lang/eval-okay-overrides.nix | 2 +- tests/lang/eval-okay-print.err.exp | 1 + tests/lang/eval-okay-print.exp | 1 + tests/lang/eval-okay-print.nix | 1 + tests/lang/eval-okay-search-path.flags | 2 +- tests/lang/framework.sh | 33 +++++ tests/lang/parse-fail-dup-attrs-1.err.exp | 7 + tests/lang/parse-fail-dup-attrs-2.err.exp | 7 + tests/lang/parse-fail-dup-attrs-3.err.exp | 7 + tests/lang/parse-fail-dup-attrs-4.err.exp | 7 + tests/lang/parse-fail-dup-attrs-6.err.exp | 1 + tests/lang/parse-fail-dup-attrs-7.err.exp | 7 + tests/lang/parse-fail-dup-formals.err.exp | 6 + tests/lang/parse-fail-eof-in-string.err.exp | 7 + .../parse-fail-mixed-nested-attrs1.err.exp | 8 ++ .../parse-fail-mixed-nested-attrs2.err.exp | 8 ++ tests/lang/parse-fail-patterns-1.err.exp | 7 + .../parse-fail-regression-20060610.err.exp | 8 ++ tests/lang/parse-fail-undef-var-2.err.exp | 7 + tests/lang/parse-fail-undef-var.err.exp | 7 + tests/lang/parse-fail-utf8.err.exp | 6 + tests/lang/parse-okay-1.exp | 1 + tests/lang/parse-okay-crlf.exp | 1 + tests/lang/parse-okay-dup-attrs-5.exp | 1 + tests/lang/parse-okay-dup-attrs-6.exp | 1 + .../lang/parse-okay-mixed-nested-attrs-1.exp | 1 + .../lang/parse-okay-mixed-nested-attrs-2.exp | 1 + .../lang/parse-okay-mixed-nested-attrs-3.exp | 1 + tests/lang/parse-okay-regression-20041027.exp | 1 + tests/lang/parse-okay-regression-751.exp | 1 + tests/lang/parse-okay-subversion.exp | 1 + tests/lang/parse-okay-url.exp | 1 + tests/local.mk | 1 + tests/misc.sh | 6 + tests/restricted.sh | 2 + 73 files changed, 762 insertions(+), 36 deletions(-) create mode 100644 tests/lang-test-infra.sh mode change 100644 => 100755 tests/lang.sh create mode 100644 tests/lang/empty.exp create mode 100644 tests/lang/eval-fail-abort.err.exp create mode 100644 tests/lang/eval-fail-antiquoted-path.err.exp create mode 100644 tests/lang/eval-fail-assert.err.exp create mode 100644 tests/lang/eval-fail-bad-antiquote-1.err.exp create mode 100644 tests/lang/eval-fail-bad-antiquote-2.err.exp create mode 100644 tests/lang/eval-fail-bad-antiquote-3.err.exp create mode 100644 tests/lang/eval-fail-bad-string-interpolation-1.err.exp create mode 100644 tests/lang/eval-fail-bad-string-interpolation-2.err.exp create mode 100644 tests/lang/eval-fail-bad-string-interpolation-3.err.exp create mode 100644 tests/lang/eval-fail-blackhole.err.exp create mode 100644 tests/lang/eval-fail-deepseq.err.exp create mode 100644 tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp create mode 100644 tests/lang/eval-fail-fromTOML-timestamps.err.exp create mode 100644 tests/lang/eval-fail-hashfile-missing.err.exp create mode 100644 tests/lang/eval-fail-list.err.exp create mode 100644 tests/lang/eval-fail-list.nix create mode 100644 tests/lang/eval-fail-missing-arg.err.exp create mode 100644 tests/lang/eval-fail-nonexist-path.err.exp create mode 100644 tests/lang/eval-fail-path-slash.err.exp create mode 100644 tests/lang/eval-fail-recursion.err.exp create mode 100644 tests/lang/eval-fail-recursion.nix create mode 100644 tests/lang/eval-fail-remove.err.exp create mode 100644 tests/lang/eval-fail-scope-5.err.exp create mode 100644 tests/lang/eval-fail-seq.err.exp create mode 100644 tests/lang/eval-fail-set-override.err.exp create mode 100644 tests/lang/eval-fail-set-override.nix create mode 100644 tests/lang/eval-fail-set.err.exp create mode 100644 tests/lang/eval-fail-set.nix create mode 100644 tests/lang/eval-fail-substring.err.exp create mode 100644 tests/lang/eval-fail-to-path.err.exp create mode 100644 tests/lang/eval-fail-undeclared-arg.err.exp create mode 100644 tests/lang/eval-okay-print.err.exp create mode 100644 tests/lang/eval-okay-print.exp create mode 100644 tests/lang/eval-okay-print.nix create mode 100644 tests/lang/framework.sh create mode 100644 tests/lang/parse-fail-dup-attrs-1.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-2.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-3.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-4.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-6.err.exp create mode 100644 tests/lang/parse-fail-dup-attrs-7.err.exp create mode 100644 tests/lang/parse-fail-dup-formals.err.exp create mode 100644 tests/lang/parse-fail-eof-in-string.err.exp create mode 100644 tests/lang/parse-fail-mixed-nested-attrs1.err.exp create mode 100644 tests/lang/parse-fail-mixed-nested-attrs2.err.exp create mode 100644 tests/lang/parse-fail-patterns-1.err.exp create mode 100644 tests/lang/parse-fail-regression-20060610.err.exp create mode 100644 tests/lang/parse-fail-undef-var-2.err.exp create mode 100644 tests/lang/parse-fail-undef-var.err.exp create mode 100644 tests/lang/parse-fail-utf8.err.exp create mode 100644 tests/lang/parse-okay-1.exp create mode 100644 tests/lang/parse-okay-crlf.exp create mode 100644 tests/lang/parse-okay-dup-attrs-5.exp create mode 100644 tests/lang/parse-okay-dup-attrs-6.exp create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-1.exp create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-2.exp create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-3.exp create mode 100644 tests/lang/parse-okay-regression-20041027.exp create mode 100644 tests/lang/parse-okay-regression-751.exp create mode 100644 tests/lang/parse-okay-subversion.exp create mode 100644 tests/lang/parse-okay-url.exp diff --git a/.gitignore b/.gitignore index 969194650..93a9ff9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -95,6 +95,7 @@ perl/Makefile.config # /tests/lang/ /tests/lang/*.out /tests/lang/*.out.xml +/tests/lang/*.err /tests/lang/*.ast /perl/lib/Nix/Config.pm diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index e5f80a928..a5253997d 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -86,6 +86,31 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. +### Characterization testing + +Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. +This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward. + +For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. + +It is frequently useful to regenerate the expected output. +To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`. +(At least, this is the convention we've used for `tests/lang.sh`. +If we add more characterization testing we should always strive to be consistent.) + +An interesting situation to document is the case when these tests are "overfitted". +The language tests are, again, an example of this. +The expected successful output of evaluation is supposed to be highly stable – we do not intend to make breaking changes to (the stable parts of) the Nix language. +However, the errors and warnings during evaluation (successful or not) are not stable in this way. +We are free to change how they are displayed at any time. + +It may be surprising that we would test non-normative behavior like diagnostic outputs. +Diagnostic outputs are indeed not a stable interface, but they still are important to users. +By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review. +Regressions are caught, and improvements always show up in code review. + +To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. diff --git a/tests/dependencies.sh b/tests/dependencies.sh index f9da0c6bc..d5cd30396 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -15,6 +15,9 @@ if test -n "$dot"; then $dot < $TEST_ROOT/graph fi +# Test GraphML graph generation +nix-store -q --graphml "$drvPath" > $TEST_ROOT/graphml + outPath=$(nix-store -rvv "$drvPath") || fail "build failed" # Test Graphviz graph generation. diff --git a/tests/lang-test-infra.sh b/tests/lang-test-infra.sh new file mode 100644 index 000000000..30da8977b --- /dev/null +++ b/tests/lang-test-infra.sh @@ -0,0 +1,86 @@ +# Test the function for lang.sh +source common.sh + +source lang/framework.sh + +# We are testing this, so don't want outside world to affect us. +unset _NIX_TEST_ACCEPT + +# We'll only modify this in subshells so we don't need to reset it. +badDiff=0 + +# matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# matches empty, non-existant file is the same as empty file +echo -n > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 0 )) +) + +# doesn't matches non-empty, non-existant file is the same as empty file +echo Hi! > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 1 )) +) + +# doesn't match, `badDiff` set, file unchanged +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Bye! )" == $(< "$TEST_ROOT/expected") ]] + +# _NIX_TEST_ACCEPT=1 matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT doesn't match, `badDiff=1` set, file changed (was previously non-empty) +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Hi! )" == $(< "$TEST_ROOT/expected") ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT matches empty, non-existant file not created +echo -n > "$TEST_ROOT/got" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists" + (( "$badDiff" == 0 )) +) +[[ ! -f "$TEST_ROOT/does-not-exist" ]] + +# _NIX_TEST_ACCEPT doesn't match, output empty, file deleted +echo -n > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +badDiff=0 +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ ! -f "$TEST_ROOT/expected" ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) diff --git a/tests/lang.sh b/tests/lang.sh old mode 100644 new mode 100755 index 8170cb39d..bfb942d3d --- a/tests/lang.sh +++ b/tests/lang.sh @@ -1,5 +1,17 @@ source common.sh +set -o pipefail + +source lang/framework.sh + +# specialize function a bit +function diffAndAccept() { + local -r testName="$1" + local -r got="lang/$testName.$2" + local -r expected="lang/$testName.$3" + diffAndAcceptInner "$testName" "$got" "$expected" +} + export TEST_VAR=foo # for eval-okay-getenv.nix export NIX_REMOTE=dummy:// export NIX_STORE_DIR=/nix/store @@ -20,63 +32,114 @@ nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x t set +x -fail=0 +badDiff=0 +badExitCode=0 for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then + i=$(basename "$i" .nix) + if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else echo "FAIL: $i shouldn't parse" - fail=1 + badExitCode=1 fi done for i in lang/parse-okay-*.nix; do echo "parsing $i (should succeed)"; - i=$(basename $i .nix) - if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then + i=$(basename "$i" .nix) + if + expect 0 nix-instantiate --parse - < "lang/$i.nix" \ + 1> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.out") \ + 2> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.err") + then + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else echo "FAIL: $i should parse" - fail=1 + badExitCode=1 fi done for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --eval lang/$i.nix; then + i=$(basename "$i" .nix) + if + expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \ + | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else echo "FAIL: $i shouldn't evaluate" - fail=1 + badExitCode=1 fi done for i in lang/eval-okay-*.nix; do echo "evaluating $i (should succeed)"; - i=$(basename $i .nix) + i=$(basename "$i" .nix) - if test -e lang/$i.exp; then - flags= - if test -e lang/$i.flags; then - flags=$(cat lang/$i.flags) - fi - if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then + if test -e "lang/$i.exp.xml"; then + if expect 0 nix-instantiate --eval --xml --no-location --strict \ + "lang/$i.nix" > "lang/$i.out.xml" + then + diffAndAccept "$i" out.xml exp.xml + else echo "FAIL: $i should evaluate" - fail=1 - elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then - echo "FAIL: evaluation result of $i not as expected" - fail=1 + badExitCode=1 + fi + elif test ! -e "lang/$i.exp-disabled"; then + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi - fi - if test -e lang/$i.exp.xml; then - if ! expect 0 nix-instantiate --eval --xml --no-location --strict \ - lang/$i.nix > lang/$i.out.xml; then + if + expect 0 env \ + NIX_PATH=lang/dir3:lang/dir4 \ + HOME=/fake-home \ + nix-instantiate "${flags[@]}" --eval --strict "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else echo "FAIL: $i should evaluate" - fail=1 - elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then - echo "FAIL: XML evaluation result of $i not as expected" - fail=1 + badExitCode=1 fi fi done -exit $fail +if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + else + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi +else + if (( "$badDiff" )); then + echo '' + echo 'You can rerun this test with:' + echo '' + echo ' _NIX_TEST_ACCEPT=1 make tests/lang.sh.test' + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + fi + exit $(( "$badExitCode" + "$badDiff" )) +fi diff --git a/tests/lang/empty.exp b/tests/lang/empty.exp new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lang/eval-fail-abort.err.exp b/tests/lang/eval-fail-abort.err.exp new file mode 100644 index 000000000..345232d3f --- /dev/null +++ b/tests/lang/eval-fail-abort.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-abort.nix:1:14: + + 1| if true then abort "this should fail" else 1 + | ^ + 2| + + error: evaluation aborted with the following error message: 'this should fail' diff --git a/tests/lang/eval-fail-antiquoted-path.err.exp b/tests/lang/eval-fail-antiquoted-path.err.exp new file mode 100644 index 000000000..425deba42 --- /dev/null +++ b/tests/lang/eval-fail-antiquoted-path.err.exp @@ -0,0 +1 @@ +error: getting attributes of path ‘PWD/lang/fnord’: No such file or directory diff --git a/tests/lang/eval-fail-assert.err.exp b/tests/lang/eval-fail-assert.err.exp new file mode 100644 index 000000000..aeecd8167 --- /dev/null +++ b/tests/lang/eval-fail-assert.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-assert.nix:4:3: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … from call site + + at /pwd/lang/eval-fail-assert.nix:4:10: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … while calling 'x' + + at /pwd/lang/eval-fail-assert.nix:2:7: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| + + error: assertion '(arg == "y")' failed + + at /pwd/lang/eval-fail-assert.nix:2:12: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| diff --git a/tests/lang/eval-fail-bad-antiquote-1.err.exp b/tests/lang/eval-fail-bad-antiquote-1.err.exp new file mode 100644 index 000000000..cf94f53bc --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-1.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-antiquote-1.nix:1:2: + + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-antiquote-2.err.exp b/tests/lang/eval-fail-bad-antiquote-2.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-2.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-bad-antiquote-3.err.exp b/tests/lang/eval-fail-bad-antiquote-3.err.exp new file mode 100644 index 000000000..fbefbc826 --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-3.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-antiquote-3.nix:1:3: + + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/lang/eval-fail-bad-string-interpolation-1.err.exp new file mode 100644 index 000000000..eb73e9a52 --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2: + + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/lang/eval-fail-bad-string-interpolation-2.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/lang/eval-fail-bad-string-interpolation-3.err.exp new file mode 100644 index 000000000..ac14f329b --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3: + + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-blackhole.err.exp b/tests/lang/eval-fail-blackhole.err.exp new file mode 100644 index 000000000..f0618d8ac --- /dev/null +++ b/tests/lang/eval-fail-blackhole.err.exp @@ -0,0 +1,18 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-blackhole.nix:2:3: + + 1| let { + 2| body = x; + | ^ + 3| x = y; + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-blackhole.nix:3:7: + + 2| body = x; + 3| x = y; + | ^ + 4| y = x; diff --git a/tests/lang/eval-fail-deepseq.err.exp b/tests/lang/eval-fail-deepseq.err.exp new file mode 100644 index 000000000..5e204ba73 --- /dev/null +++ b/tests/lang/eval-fail-deepseq.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'deepSeq' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:1: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while evaluating the attribute 'x' + + at /pwd/lang/eval-fail-deepseq.nix:1:20: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:24: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp new file mode 100644 index 000000000..0069285fb --- /dev/null +++ b/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp @@ -0,0 +1,38 @@ +error: + … while calling the 'foldl'' builtin + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1: + + 1| # Tests that the result of applying op is forced even if the value is never used + 2| builtins.foldl' + | ^ + 3| (_: f: f null) + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … from call site + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6: + + 4| null + 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] + | ^ + 6| + + error: Not the final value, but is still forced! diff --git a/tests/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/lang/eval-fail-fromTOML-timestamps.err.exp new file mode 100644 index 000000000..f6bd19f5a --- /dev/null +++ b/tests/lang/eval-fail-fromTOML-timestamps.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'fromTOML' builtin + + at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1: + + 1| builtins.fromTOML '' + | ^ + 2| key = "value" + + error: while parsing a TOML string: Dates and times are not supported + + at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-hashfile-missing.err.exp b/tests/lang/eval-fail-hashfile-missing.err.exp new file mode 100644 index 000000000..8e77dec1e --- /dev/null +++ b/tests/lang/eval-fail-hashfile-missing.err.exp @@ -0,0 +1,19 @@ +error: + … while calling the 'toString' builtin + + at /pwd/lang/eval-fail-hashfile-missing.nix:4:3: + + 3| in + 4| toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) + | ^ + 5| + + … while evaluating the first argument passed to builtins.toString + + at «none»:0: (source not available) + + … while calling the 'hashFile' builtin + + at «none»:0: (source not available) + + error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory diff --git a/tests/lang/eval-fail-list.err.exp b/tests/lang/eval-fail-list.err.exp new file mode 100644 index 000000000..24d682118 --- /dev/null +++ b/tests/lang/eval-fail-list.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating one of the elements to concatenate + + at /pwd/lang/eval-fail-list.nix:1:2: + + 1| 8++1 + | ^ + 2| + + error: value is an integer while a list was expected diff --git a/tests/lang/eval-fail-list.nix b/tests/lang/eval-fail-list.nix new file mode 100644 index 000000000..fa749f2f7 --- /dev/null +++ b/tests/lang/eval-fail-list.nix @@ -0,0 +1 @@ +8++1 diff --git a/tests/lang/eval-fail-missing-arg.err.exp b/tests/lang/eval-fail-missing-arg.err.exp new file mode 100644 index 000000000..61fabf0d5 --- /dev/null +++ b/tests/lang/eval-fail-missing-arg.err.exp @@ -0,0 +1,16 @@ +error: + … from call site + + at /pwd/lang/eval-fail-missing-arg.nix:1:1: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called without required argument 'y' + + at /pwd/lang/eval-fail-missing-arg.nix:1:2: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| diff --git a/tests/lang/eval-fail-nonexist-path.err.exp b/tests/lang/eval-fail-nonexist-path.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-nonexist-path.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-path-slash.err.exp b/tests/lang/eval-fail-path-slash.err.exp new file mode 100644 index 000000000..f0011c97f --- /dev/null +++ b/tests/lang/eval-fail-path-slash.err.exp @@ -0,0 +1,8 @@ +error: path has a trailing slash + + at /pwd/lang/eval-fail-path-slash.nix:6:12: + + 5| # and https://nixos.org/nix-dev/2016-June/020829.html + 6| /nix/store/ + | ^ + 7| diff --git a/tests/lang/eval-fail-recursion.err.exp b/tests/lang/eval-fail-recursion.err.exp new file mode 100644 index 000000000..af64133cb --- /dev/null +++ b/tests/lang/eval-fail-recursion.err.exp @@ -0,0 +1,16 @@ +error: + … in the right operand of the update (//) operator + + at /pwd/lang/eval-fail-recursion.nix:1:12: + + 1| let a = {} // a; in a.foo + | ^ + 2| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-recursion.nix:1:15: + + 1| let a = {} // a; in a.foo + | ^ + 2| diff --git a/tests/lang/eval-fail-recursion.nix b/tests/lang/eval-fail-recursion.nix new file mode 100644 index 000000000..075b5ed06 --- /dev/null +++ b/tests/lang/eval-fail-recursion.nix @@ -0,0 +1 @@ +let a = {} // a; in a.foo diff --git a/tests/lang/eval-fail-remove.err.exp b/tests/lang/eval-fail-remove.err.exp new file mode 100644 index 000000000..e82cdac98 --- /dev/null +++ b/tests/lang/eval-fail-remove.err.exp @@ -0,0 +1,19 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-remove.nix:4:3: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + + error: attribute 'x' missing + + at /pwd/lang/eval-fail-remove.nix:4:10: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + Did you mean y? diff --git a/tests/lang/eval-fail-scope-5.err.exp b/tests/lang/eval-fail-scope-5.err.exp new file mode 100644 index 000000000..22b6166f8 --- /dev/null +++ b/tests/lang/eval-fail-scope-5.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-scope-5.nix:8:3: + + 7| + 8| body = f {}; + | ^ + 9| + + … from call site + + at /pwd/lang/eval-fail-scope-5.nix:8:10: + + 7| + 8| body = f {}; + | ^ + 9| + + … while calling 'f' + + at /pwd/lang/eval-fail-scope-5.nix:6:7: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-scope-5.nix:6:12: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| diff --git a/tests/lang/eval-fail-seq.err.exp b/tests/lang/eval-fail-seq.err.exp new file mode 100644 index 000000000..33a7e9491 --- /dev/null +++ b/tests/lang/eval-fail-seq.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'seq' builtin + + at /pwd/lang/eval-fail-seq.nix:1:1: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-seq.nix:1:15: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-set-override.err.exp b/tests/lang/eval-fail-set-override.err.exp new file mode 100644 index 000000000..beb29d678 --- /dev/null +++ b/tests/lang/eval-fail-set-override.err.exp @@ -0,0 +1,6 @@ +error: + … while evaluating the `__overrides` attribute + + at «none»:0: (source not available) + + error: value is an integer while a set was expected diff --git a/tests/lang/eval-fail-set-override.nix b/tests/lang/eval-fail-set-override.nix new file mode 100644 index 000000000..03551c186 --- /dev/null +++ b/tests/lang/eval-fail-set-override.nix @@ -0,0 +1 @@ +rec { __overrides = 1; } diff --git a/tests/lang/eval-fail-set.err.exp b/tests/lang/eval-fail-set.err.exp new file mode 100644 index 000000000..0d0140508 --- /dev/null +++ b/tests/lang/eval-fail-set.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'x' + + at /pwd/lang/eval-fail-set.nix:1:3: + + 1| 8.x + | ^ + 2| diff --git a/tests/lang/eval-fail-set.nix b/tests/lang/eval-fail-set.nix new file mode 100644 index 000000000..c6b7980b6 --- /dev/null +++ b/tests/lang/eval-fail-set.nix @@ -0,0 +1 @@ +8.x diff --git a/tests/lang/eval-fail-substring.err.exp b/tests/lang/eval-fail-substring.err.exp new file mode 100644 index 000000000..dc26a00bd --- /dev/null +++ b/tests/lang/eval-fail-substring.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'substring' builtin + + at /pwd/lang/eval-fail-substring.nix:1:1: + + 1| builtins.substring (builtins.sub 0 1) 1 "x" + | ^ + 2| + + error: negative start position in 'substring' + + at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-to-path.err.exp b/tests/lang/eval-fail-to-path.err.exp new file mode 100644 index 000000000..43ed2bdfc --- /dev/null +++ b/tests/lang/eval-fail-to-path.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'toPath' builtin + + at /pwd/lang/eval-fail-to-path.nix:1:1: + + 1| builtins.toPath "foo/bar" + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + at «none»:0: (source not available) + + error: string 'foo/bar' doesn't represent an absolute path diff --git a/tests/lang/eval-fail-undeclared-arg.err.exp b/tests/lang/eval-fail-undeclared-arg.err.exp new file mode 100644 index 000000000..30db743c7 --- /dev/null +++ b/tests/lang/eval-fail-undeclared-arg.err.exp @@ -0,0 +1,17 @@ +error: + … from call site + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:1: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called with unexpected argument 'y' + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:2: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + Did you mean one of x or z? diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/lang/eval-okay-fromjson.nix index e1c0f86cc..4c526b9ae 100644 --- a/tests/lang/eval-okay-fromjson.nix +++ b/tests/lang/eval-okay-fromjson.nix @@ -11,9 +11,12 @@ builtins.fromJSON "Width": 200, "Height": 250 }, + "Animated" : false, + "IDs": [116, 943, 234, 38793, true ,false,null, -100], + "Escapes": "\"\\\/\t\n\r\t", "Subtitle" : false, - "Latitude": 46.2051, - "Longitude": 6.0723 + "Latitude": 37.7668, + "Longitude": -122.3959 } } '' @@ -28,8 +31,11 @@ builtins.fromJSON Width = 200; Height = 250; }; + Animated = false; + IDs = [ 116 943 234 38793 true false null (0-100) ]; + Escapes = "\"\\\/\t\n\r\t"; # supported in JSON but not Nix: \b\f Subtitle = false; - Latitude = 46.2051; - Longitude = 6.0723; + Latitude = 37.7668; + Longitude = -122.3959; }; } diff --git a/tests/lang/eval-okay-overrides.nix b/tests/lang/eval-okay-overrides.nix index 358742b36..719bdc9c0 100644 --- a/tests/lang/eval-okay-overrides.nix +++ b/tests/lang/eval-okay-overrides.nix @@ -1,6 +1,6 @@ let - overrides = { a = 2; }; + overrides = { a = 2; b = 3; }; in (rec { __overrides = overrides; diff --git a/tests/lang/eval-okay-print.err.exp b/tests/lang/eval-okay-print.err.exp new file mode 100644 index 000000000..3fc99be3e --- /dev/null +++ b/tests/lang/eval-okay-print.err.exp @@ -0,0 +1 @@ +trace: [ ] diff --git a/tests/lang/eval-okay-print.exp b/tests/lang/eval-okay-print.exp new file mode 100644 index 000000000..0d960fb70 --- /dev/null +++ b/tests/lang/eval-okay-print.exp @@ -0,0 +1 @@ +[ null [ [ «repeated» ] ] ] diff --git a/tests/lang/eval-okay-print.nix b/tests/lang/eval-okay-print.nix new file mode 100644 index 000000000..d36ba4da3 --- /dev/null +++ b/tests/lang/eval-okay-print.nix @@ -0,0 +1 @@ +with builtins; trace [(1+1)] [ null toString (deepSeq "x") (a: a) (let x=[x]; in x) ] diff --git a/tests/lang/eval-okay-search-path.flags b/tests/lang/eval-okay-search-path.flags index a28e68210..dfad1c611 100644 --- a/tests/lang/eval-okay-search-path.flags +++ b/tests/lang/eval-okay-search-path.flags @@ -1 +1 @@ --I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 \ No newline at end of file +-I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 diff --git a/tests/lang/framework.sh b/tests/lang/framework.sh new file mode 100644 index 000000000..d6499e0e9 --- /dev/null +++ b/tests/lang/framework.sh @@ -0,0 +1,33 @@ +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=lang/empty.exp + fi + + # Diff so we get a nice message + if ! diff "$got" "$expectedOrEmpty"; then + echo "FAIL: evaluation result of $testName not as expected" + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} diff --git a/tests/lang/parse-fail-dup-attrs-1.err.exp b/tests/lang/parse-fail-dup-attrs-1.err.exp new file mode 100644 index 000000000..4fe6b7a1f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-1.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:1:3 + + at «stdin»:3:3: + + 2| y = 456; + 3| x = 789; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-2.err.exp b/tests/lang/parse-fail-dup-attrs-2.err.exp new file mode 100644 index 000000000..3aba2891f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-2.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-3.err.exp b/tests/lang/parse-fail-dup-attrs-3.err.exp new file mode 100644 index 000000000..3aba2891f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-3.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-4.err.exp b/tests/lang/parse-fail-dup-attrs-4.err.exp new file mode 100644 index 000000000..ff68446a1 --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-4.err.exp @@ -0,0 +1,7 @@ +error: attribute 'services.ssh.port' already defined at «stdin»:2:3 + + at «stdin»:3:3: + + 2| services.ssh.port = 22; + 3| services.ssh.port = 23; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-6.err.exp b/tests/lang/parse-fail-dup-attrs-6.err.exp new file mode 100644 index 000000000..74823fc25 --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-6.err.exp @@ -0,0 +1 @@ +error: attribute ‘services.ssh’ at (string):3:3 already defined at (string):2:3 diff --git a/tests/lang/parse-fail-dup-attrs-7.err.exp b/tests/lang/parse-fail-dup-attrs-7.err.exp new file mode 100644 index 000000000..512a499ca --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-7.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:6:12 + + at «stdin»:7:12: + + 6| inherit x; + 7| inherit x; + | ^ diff --git a/tests/lang/parse-fail-dup-formals.err.exp b/tests/lang/parse-fail-dup-formals.err.exp new file mode 100644 index 000000000..1d566fb33 --- /dev/null +++ b/tests/lang/parse-fail-dup-formals.err.exp @@ -0,0 +1,6 @@ +error: duplicate formal function argument 'x' + + at «stdin»:1:8: + + 1| {x, y, x}: x + | ^ diff --git a/tests/lang/parse-fail-eof-in-string.err.exp b/tests/lang/parse-fail-eof-in-string.err.exp new file mode 100644 index 000000000..f9fa72312 --- /dev/null +++ b/tests/lang/parse-fail-eof-in-string.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected end of file, expecting '"' + + at «stdin»:3:5: + + 2| # Note that this file must not end with a newline. + 3| a 1"$ + | ^ diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/lang/parse-fail-mixed-nested-attrs1.err.exp new file mode 100644 index 000000000..32f776795 --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -0,0 +1,8 @@ +error: attribute 'z' already defined at «stdin»:3:16 + + at «stdin»:2:3: + + 1| { + 2| x.z = 3; + | ^ + 3| x = { y = 3; z = 3; }; diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/lang/parse-fail-mixed-nested-attrs2.err.exp new file mode 100644 index 000000000..0437cd50c --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -0,0 +1,8 @@ +error: attribute 'y' already defined at «stdin»:3:9 + + at «stdin»:2:3: + + 1| { + 2| x.y.y = 3; + | ^ + 3| x = { y.y= 3; z = 3; }; diff --git a/tests/lang/parse-fail-patterns-1.err.exp b/tests/lang/parse-fail-patterns-1.err.exp new file mode 100644 index 000000000..634a04aaa --- /dev/null +++ b/tests/lang/parse-fail-patterns-1.err.exp @@ -0,0 +1,7 @@ +error: duplicate formal function argument 'args' + + at «stdin»:1:1: + + 1| args@{args, x, y, z}: x + | ^ + 2| diff --git a/tests/lang/parse-fail-regression-20060610.err.exp b/tests/lang/parse-fail-regression-20060610.err.exp new file mode 100644 index 000000000..167d01e85 --- /dev/null +++ b/tests/lang/parse-fail-regression-20060610.err.exp @@ -0,0 +1,8 @@ +error: undefined variable 'gcc' + + at «stdin»:8:12: + + 7| + 8| body = ({ + | ^ + 9| inherit gcc; diff --git a/tests/lang/parse-fail-undef-var-2.err.exp b/tests/lang/parse-fail-undef-var-2.err.exp new file mode 100644 index 000000000..77c96bbd2 --- /dev/null +++ b/tests/lang/parse-fail-undef-var-2.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected ':', expecting '}' + + at «stdin»:3:13: + + 2| + 3| f = {x, y : + | ^ diff --git a/tests/lang/parse-fail-undef-var.err.exp b/tests/lang/parse-fail-undef-var.err.exp new file mode 100644 index 000000000..48e88747f --- /dev/null +++ b/tests/lang/parse-fail-undef-var.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'y' + + at «stdin»:1:4: + + 1| x: y + | ^ + 2| diff --git a/tests/lang/parse-fail-utf8.err.exp b/tests/lang/parse-fail-utf8.err.exp new file mode 100644 index 000000000..6087479a3 --- /dev/null +++ b/tests/lang/parse-fail-utf8.err.exp @@ -0,0 +1,6 @@ +error: syntax error, unexpected invalid token, expecting end of file + + at «stdin»:1:5: + + 1| 123 + | ^ diff --git a/tests/lang/parse-okay-1.exp b/tests/lang/parse-okay-1.exp new file mode 100644 index 000000000..d5ab5f18a --- /dev/null +++ b/tests/lang/parse-okay-1.exp @@ -0,0 +1 @@ +({ x, y, z }: ((x + y) + z)) diff --git a/tests/lang/parse-okay-crlf.exp b/tests/lang/parse-okay-crlf.exp new file mode 100644 index 000000000..4213609fc --- /dev/null +++ b/tests/lang/parse-okay-crlf.exp @@ -0,0 +1 @@ +rec { foo = "multi\nline\n string\n test\r"; x = y; y = 123; z = 456; } diff --git a/tests/lang/parse-okay-dup-attrs-5.exp b/tests/lang/parse-okay-dup-attrs-5.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/lang/parse-okay-dup-attrs-5.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-dup-attrs-6.exp b/tests/lang/parse-okay-dup-attrs-6.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/lang/parse-okay-dup-attrs-6.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.exp b/tests/lang/parse-okay-mixed-nested-attrs-1.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-1.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.exp b/tests/lang/parse-okay-mixed-nested-attrs-2.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-2.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.exp b/tests/lang/parse-okay-mixed-nested-attrs-3.exp new file mode 100644 index 000000000..b89a59734 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-3.exp @@ -0,0 +1 @@ +{ services = { httpd = { enable = true; }; ssh = { enable = true; port = 123; }; }; } diff --git a/tests/lang/parse-okay-regression-20041027.exp b/tests/lang/parse-okay-regression-20041027.exp new file mode 100644 index 000000000..9df7219e4 --- /dev/null +++ b/tests/lang/parse-okay-regression-20041027.exp @@ -0,0 +1 @@ +({ fetchurl, stdenv }: ((stdenv).mkDerivation { name = "libXi-6.0.1"; src = (fetchurl { md5 = "7e935a42428d63a387b3c048be0f2756"; url = "http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2"; }); })) diff --git a/tests/lang/parse-okay-regression-751.exp b/tests/lang/parse-okay-regression-751.exp new file mode 100644 index 000000000..e2ed886fe --- /dev/null +++ b/tests/lang/parse-okay-regression-751.exp @@ -0,0 +1 @@ +(let const = (a: "const"); in ((const { x = "q"; }))) diff --git a/tests/lang/parse-okay-subversion.exp b/tests/lang/parse-okay-subversion.exp new file mode 100644 index 000000000..4168ee8bf --- /dev/null +++ b/tests/lang/parse-okay-subversion.exp @@ -0,0 +1 @@ +({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { builder = /foo/bar; db4 = (if localServer then db4 else null); inherit expat ; inherit httpServer ; httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); inherit javaSwigBindings ; inherit javahlBindings ; inherit localServer ; name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); inherit pythonBindings ; src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); inherit sslSupport ; swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) diff --git a/tests/lang/parse-okay-url.exp b/tests/lang/parse-okay-url.exp new file mode 100644 index 000000000..e5f0829b0 --- /dev/null +++ b/tests/lang/parse-okay-url.exp @@ -0,0 +1 @@ +[ ("x:x") ("https://svn.cs.uu.nl:12443/repos/trace/trunk") ("http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2") ("http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz") ("http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz") ("https://ftp5.gwdg.de/pub/linux/archlinux/extra/os/x86_64/unzip-6.0-14-x86_64.pkg.tar.zst") ("ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz") ] diff --git a/tests/local.mk b/tests/local.mk index 88848926b..bb80c5317 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -20,6 +20,7 @@ nix_tests = \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ + lang-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ diff --git a/tests/misc.sh b/tests/misc.sh index 60d58310e..af96d20bd 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -24,3 +24,9 @@ eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - echo $eval_stdin_res | grep "at «stdin»:1:15:" echo $eval_stdin_res | grep "infinite recursion encountered" +# Attribute path errors +expectStderr 1 nix-instantiate --eval -E '{}' -A '"x' | grepQuiet "missing closing quote in selection path" +expectStderr 1 nix-instantiate --eval -E '[]' -A 'x' | grepQuiet "should be a set" +expectStderr 1 nix-instantiate --eval -E '{}' -A '1' | grepQuiet "should be a list" +expectStderr 1 nix-instantiate --eval -E '{}' -A '.' | grepQuiet "empty attribute name" +expectStderr 1 nix-instantiate --eval -E '[]' -A '1' | grepQuiet "out of range" diff --git a/tests/restricted.sh b/tests/restricted.sh index 776893a56..17f310a4b 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -49,3 +49,5 @@ output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ 2>&1 || :)" echo "$output" | grep "is forbidden" echo "$output" | grepInverse -F restricted-secret + +expectStderr 1 nix-instantiate --restrict-eval true ./dependencies.nix | grepQuiet "forbidden in restricted mode" From 0309f6b5b8ca354e1a43d320928d3d546ce7f878 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Wed, 12 Jul 2023 12:32:57 +0100 Subject: [PATCH 19/78] Update src/libstore/globals.hh Co-authored-by: Valentin Gagarin --- src/libstore/globals.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 81fa154bb..d8fda9707 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -530,8 +530,8 @@ public: Nix will attempt to drop supplementary groups when building with sandboxing. However this can fail under some circumstances. - For example, if the user lacks the CAP_SETGID capability. - Search setgroups(2) for EPERM to find more detailed information on this. + For example, if the user lacks the `CAP_SETGID` capability. + Search `setgroups(2)` for `EPERM` to find more detailed information on this. If you encounter such a failure, setting this option to `false` will let you ignore it and continue. But before doing so, you should consider the security implications carefully. From a2acd23466108a1a10ed3b6b874c558bd95e7645 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Wed, 12 Jul 2023 12:33:05 +0100 Subject: [PATCH 20/78] Update src/libstore/globals.hh Co-authored-by: Valentin Gagarin --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d8fda9707..d4b8fb1f9 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -538,7 +538,7 @@ public: Not dropping supplementary groups means the build sandbox will be less restricted than intended. This option defaults to `true` when the user is root - (since root usually has permissions to call setgroups) + (since `root` usually has permissions to call setgroups) and `false` otherwise. )"}; From 2c3fb0eb33d205d1937b7ed801bdb36bb301d1a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jul 2023 22:22:44 -0400 Subject: [PATCH 21/78] Move `BuiltPath` to its own header/C++ file in libcmd It is less important, and used less widely, than `DerivedPath`. --- src/libcmd/built-path.cc | 67 ++++++++++++++++++++++++++++++++++++ src/libcmd/built-path.hh | 47 +++++++++++++++++++++++++ src/libcmd/installables.hh | 1 + src/libstore/derived-path.cc | 56 ------------------------------ src/libstore/derived-path.hh | 41 ---------------------- 5 files changed, 115 insertions(+), 97 deletions(-) create mode 100644 src/libcmd/built-path.cc create mode 100644 src/libcmd/built-path.hh diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc new file mode 100644 index 000000000..db9c440e3 --- /dev/null +++ b/src/libcmd/built-path.cc @@ -0,0 +1,67 @@ +#include "built-path.hh" +#include "derivations.hh" +#include "store-api.hh" + +#include + +#include + +namespace nix { + +nlohmann::json BuiltPath::Built::toJSON(ref store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + for (const auto& [output, path] : outputs) { + res["outputs"][output] = store->printStorePath(path); + } + return res; +} + +StorePathSet BuiltPath::outPaths() const +{ + return std::visit( + overloaded{ + [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, + [](const BuiltPath::Built & b) { + StorePathSet res; + for (auto & [_, path] : b.outputs) + res.insert(path); + return res; + }, + }, raw() + ); +} + +RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +{ + RealisedPath::Set res; + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, + [&](const BuiltPath::Built & p) { + auto drvHashes = + staticOutputHashes(store, store.readDerivation(p.drvPath)); + for (auto& [outputName, outputPath] : p.outputs) { + if (experimentalFeatureSettings.isEnabled( + Xp::CaDerivations)) { + auto drvOutput = get(drvHashes, outputName); + if (!drvOutput) + throw Error( + "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", + store.printStorePath(p.drvPath), outputName); + auto thisRealisation = store.queryRealisation( + DrvOutput{*drvOutput, outputName}); + assert(thisRealisation); // We’ve built it, so we must + // have the realisation + res.insert(*thisRealisation); + } else { + res.insert(outputPath); + } + } + }, + }, + raw()); + return res; +} + +} diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh new file mode 100644 index 000000000..c563a46e9 --- /dev/null +++ b/src/libcmd/built-path.hh @@ -0,0 +1,47 @@ +#include "derived-path.hh" + +namespace nix { + +/** + * A built derived path with hints in the form of optional concrete output paths. + * + * See 'BuiltPath' for more an explanation. + */ +struct BuiltPathBuilt { + StorePath drvPath; + std::map outputs; + + nlohmann::json toJSON(ref store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view); + + GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); +}; + +using _BuiltPathRaw = std::variant< + DerivedPath::Opaque, + BuiltPathBuilt +>; + +/** + * A built path. Similar to a DerivedPath, but enriched with the corresponding + * output path(s). + */ +struct BuiltPath : _BuiltPathRaw { + using Raw = _BuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = BuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePathSet outPaths() const; + RealisedPath::Set toRealisedPaths(Store & store) const; + +}; + +typedef std::vector BuiltPaths; + +} diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 42d6c7c7c..b0dc0dc02 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -5,6 +5,7 @@ #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" +#include "built-path.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 9a2ffda39..52d073f81 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,5 +1,4 @@ #include "derived-path.hh" -#include "derivations.hh" #include "store-api.hh" #include @@ -30,30 +29,6 @@ nlohmann::json DerivedPath::Built::toJSON(ref store) const { return res; } -nlohmann::json BuiltPath::Built::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); - } - return res; -} - -StorePathSet BuiltPath::outPaths() const -{ - return std::visit( - overloaded{ - [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, - [](const BuiltPath::Built & b) { - StorePathSet res; - for (auto & [_, path] : b.outputs) - res.insert(path); - return res; - }, - }, raw() - ); -} - std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); @@ -121,35 +96,4 @@ DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) return parseWith(store, s, "!"); } -RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const -{ - RealisedPath::Set res; - std::visit( - overloaded{ - [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, - [&](const BuiltPath::Built & p) { - auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); - for (auto& [outputName, outputPath] : p.outputs) { - if (experimentalFeatureSettings.isEnabled( - Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); - auto thisRealisation = store.queryRealisation( - DrvOutput{*drvOutput, outputName}); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(*thisRealisation); - } else { - res.insert(outputPath); - } - } - }, - }, - raw()); - return res; -} } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 5f7acbebc..6ea80c92e 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -109,47 +109,6 @@ struct DerivedPath : _DerivedPathRaw { static DerivedPath parseLegacy(const Store & store, std::string_view); }; -/** - * A built derived path with hints in the form of optional concrete output paths. - * - * See 'BuiltPath' for more an explanation. - */ -struct BuiltPathBuilt { - StorePath drvPath; - std::map outputs; - - nlohmann::json toJSON(ref store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); - - GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); -}; - -using _BuiltPathRaw = std::variant< - DerivedPath::Opaque, - BuiltPathBuilt ->; - -/** - * A built path. Similar to a DerivedPath, but enriched with the corresponding - * output path(s). - */ -struct BuiltPath : _BuiltPathRaw { - using Raw = _BuiltPathRaw; - using Raw::Raw; - - using Opaque = DerivedPathOpaque; - using Built = BuiltPathBuilt; - - inline const Raw & raw() const { - return static_cast(*this); - } - - StorePathSet outPaths() const; - RealisedPath::Set toRealisedPaths(Store & store) const; - -}; - typedef std::vector DerivedPaths; -typedef std::vector BuiltPaths; } From caabc4f64889d5a4c47d6102b3aa1d3c80bbc107 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 12 Jul 2023 23:33:43 -0400 Subject: [PATCH 22/78] Feature gate `DownstreamPlaceholder::unknownCaOutput` This is a part of CA derivations that we forgot to put behind the experimental feature. This was caught by @fricklerhandwerk in https://github.com/NixOS/nix/pull/8369#discussion_r1258133719 --- src/libexpr/eval.cc | 5 ++-- src/libexpr/eval.hh | 5 +++- src/libexpr/primops.cc | 25 ++++++++++---------- src/libexpr/tests/derived-path.cc | 9 ++++++- src/libstore/derivations.cc | 8 ++++--- src/libstore/downstream-placeholder.cc | 4 +++- src/libstore/downstream-placeholder.hh | 5 +++- src/libstore/tests/downstream-placeholder.cc | 16 +++++++++---- src/nix/app.cc | 11 +++++---- 9 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index be1bdb806..7071815d4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1068,14 +1068,15 @@ void EvalState::mkOutputString( Value & value, const StorePath & drvPath, const std::string outputName, - std::optional optOutputPath) + std::optional optOutputPath, + const ExperimentalFeatureSettings & xpSettings) { value.mkString( optOutputPath ? store->printStorePath(*std::move(optOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ - : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(), + : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(), NixStringContext { NixStringContextElem::Built { .drvPath = drvPath, diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 277e77ad5..7f11ce71d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -689,12 +689,15 @@ public: * be passed if and only if output store object is input-addressed. * Will be printed to form string if passed, otherwise a placeholder * will be used (see `DownstreamPlaceholder`). + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ void mkOutputString( Value & value, const StorePath & drvPath, const std::string outputName, - std::optional optOutputPath); + std::optional optOutputPath, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a61e57cc..5dab06f26 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -83,22 +83,21 @@ StringMap EvalState::realiseContext(const NixStringContext & context) for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->buildPaths(buildReqs); - /* Get all the output paths corresponding to the placeholders we had */ for (auto & drv : drvs) { auto outputs = resolveDerivedPath(*store, drv); for (auto & [outputName, outputPath] : outputs) { - res.insert_or_assign( - DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), - store->printStorePath(outputPath) - ); - } - } - - /* Add the output of this derivations to the allowed - paths. */ - if (allowedPaths) { - for (auto & [_placeholder, outputPath] : res) { - allowPath(store->toRealPath(outputPath)); + /* Add the output of this derivations to the allowed + paths. */ + if (allowedPaths) { + allowPath(outputPath); + } + /* Get all the output paths corresponding to the placeholders we had */ + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + res.insert_or_assign( + DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), + store->printStorePath(outputPath) + ); + } } } diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index 8210efef2..c713fe28a 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -37,8 +37,15 @@ RC_GTEST_FIXTURE_PROP( prop_built_path_placeholder_round_trip, (const StorePath & drvPath, const StorePathName & outputName)) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + auto * v = state.allocValue(); - state.mkOutputString(*v, drvPath, outputName.name, std::nullopt); + state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings); auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); DerivedPath::Built b { .drvPath = drvPath, diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6f63685d4..234d70ec5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -878,9 +878,11 @@ std::optional Derivation::tryResolve( for (auto & [inputDrv, inputOutputs] : inputDrvs) { for (auto & outputName : inputOutputs) { if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - inputRewrites.emplace( - DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), - store.printStorePath(*actualPath)); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + inputRewrites.emplace( + DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), + store.printStorePath(*actualPath)); + } resolved.inputSrcs.insert(*actualPath); } else { warn("output '%s' of input '%s' missing, aborting the resolving", diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 1752738f2..d623c05e2 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -11,8 +11,10 @@ std::string DownstreamPlaceholder::render() const DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( const StorePath & drvPath, - std::string_view outputName) + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings) { + xpSettings.require(Xp::CaDerivations); auto drvNameWithExtension = drvPath.name(); auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index f0c0dee77..97f77e6b8 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -52,10 +52,13 @@ public: * * The derivation itself is known (we have a store path for it), but * the output doesn't yet have a known store path. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DownstreamPlaceholder unknownCaOutput( const StorePath & drvPath, - std::string_view outputName); + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Create a placehold for the output of an unknown derivation. diff --git a/src/libstore/tests/downstream-placeholder.cc b/src/libstore/tests/downstream-placeholder.cc index ec3e1000f..fd29530ac 100644 --- a/src/libstore/tests/downstream-placeholder.cc +++ b/src/libstore/tests/downstream-placeholder.cc @@ -5,17 +5,24 @@ namespace nix { TEST(DownstreamPlaceholder, unknownCaOutput) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + ASSERT_EQ( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, - "out").render(), + "out", + mockXpSettings).render(), "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz"); } TEST(DownstreamPlaceholder, unknownDerivation) { /** - * We set these in tests rather than the regular globals so we don't have - * to worry about race conditions if the tests run concurrently. + * Same reason as above */ ExperimentalFeatureSettings mockXpSettings; mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); @@ -24,7 +31,8 @@ TEST(DownstreamPlaceholder, unknownDerivation) { DownstreamPlaceholder::unknownDerivation( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" }, - "out"), + "out", + mockXpSettings), "out", mockXpSettings).render(), "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w"); diff --git a/src/nix/app.cc b/src/nix/app.cc index e678b54f0..e0f68b4fc 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -22,11 +22,12 @@ StringPairs resolveRewrites( StringPairs res; for (auto & dep : dependencies) if (auto drvDep = std::get_if(&dep.path)) - for (auto & [ outputName, outputPath ] : drvDep->outputs) - res.emplace( - DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), - store.printStorePath(outputPath) - ); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + for (auto & [ outputName, outputPath ] : drvDep->outputs) + res.emplace( + DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), + store.printStorePath(outputPath) + ); return res; } From e072e18475bc461e522f34d63cc74fc5fbf1640e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jul 2023 08:03:42 -0400 Subject: [PATCH 23/78] Fix race condition in the language tests When we pipe to `>(...)` like that, we unfortunately don't wait for the process to finish. Better to just substitute the file. Also, use the "unified" diff output that people (including myself) are more familiar with, thanks to Git. --- tests/lang.sh | 5 +++-- tests/lang/framework.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/lang.sh b/tests/lang.sh index bfb942d3d..75dbbc38e 100755 --- a/tests/lang.sh +++ b/tests/lang.sh @@ -52,9 +52,10 @@ for i in lang/parse-okay-*.nix; do i=$(basename "$i" .nix) if expect 0 nix-instantiate --parse - < "lang/$i.nix" \ - 1> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.out") \ - 2> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.err") + 1> "lang/$i.out" \ + 2> "lang/$i.err" then + sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else diff --git a/tests/lang/framework.sh b/tests/lang/framework.sh index d6499e0e9..516bff8ad 100644 --- a/tests/lang/framework.sh +++ b/tests/lang/framework.sh @@ -16,7 +16,7 @@ function diffAndAcceptInner() { fi # Diff so we get a nice message - if ! diff "$got" "$expectedOrEmpty"; then + if ! diff --unified "$got" "$expectedOrEmpty"; then echo "FAIL: evaluation result of $testName not as expected" badDiff=1 fi From 9e64f243406ff2b3fbe06adbc762cf965e6825a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jul 2023 15:06:34 -0400 Subject: [PATCH 24/78] Revert "Check _NIX_TEST_NO_SANDBOX when setting _canUseSandbox." This reverts commit c1d39de1fbceebbb6b46abc4a21fb8e34423df99. --- tests/common/vars-and-functions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index ad0623871..dc7ce13cc 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -141,7 +141,7 @@ restartDaemon() { startDaemon } -if [[ -z "${_NIX_TEST_NO_SANDBOX:-}" ]] && [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then _canUseSandbox=1 fi From 84c4e6f0acfc902955d580aa090aa52f20ee82ad Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Jul 2023 15:06:50 -0400 Subject: [PATCH 25/78] Revert "Skip build-remote-trustless unless sandbox is supported." This reverts commit 41412dc4ae0fec2d08335c19724276d99e0c6056. --- tests/build-remote-trustless-should-fail-0.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh index b14101f83..fad1def59 100644 --- a/tests/build-remote-trustless-should-fail-0.sh +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -2,7 +2,6 @@ source common.sh enableFeatures "daemon-trust-override" -requireSandboxSupport restartDaemon [[ $busybox =~ busybox ]] || skipTest "no busybox" From 1a13757880479921966adeca839cf182b3ffcbd0 Mon Sep 17 00:00:00 2001 From: cidkidnix Date: Thu, 13 Jul 2023 14:17:22 -0500 Subject: [PATCH 26/78] Add comment regarding the unset of NIX_STORE_DIR in build-remote.sh and supplementary-groups.sh --- tests/build-remote.sh | 1 + tests/supplementary-groups.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 78e12b477..d2a2132c1 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -1,6 +1,7 @@ requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# Avoid store dir being inside sandbox build-dir unset NIX_STORE_DIR unset NIX_STATE_DIR diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh index 47c6ef605..474ba7503 100644 --- a/tests/supplementary-groups.sh +++ b/tests/supplementary-groups.sh @@ -5,6 +5,10 @@ requireSandboxSupport if ! command -p -v unshare; then skipTest "Need unshare"; fi needLocalStore "The test uses --store always so we would just be bypassing the daemon" +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + unshare --mount --map-root-user bash < Date: Thu, 13 Jul 2023 14:23:24 -0500 Subject: [PATCH 27/78] move unset NIX_STORE_DIR in supplementary-groups.sh to inside the unshare --- tests/supplementary-groups.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh index 474ba7503..d18fb2414 100644 --- a/tests/supplementary-groups.sh +++ b/tests/supplementary-groups.sh @@ -5,13 +5,13 @@ requireSandboxSupport if ! command -p -v unshare; then skipTest "Need unshare"; fi needLocalStore "The test uses --store always so we would just be bypassing the daemon" -# Avoid store dir being inside sandbox build-dir -unset NIX_STORE_DIR -unset NIX_STATE_DIR - unshare --mount --map-root-user bash < Date: Thu, 13 Jul 2023 13:17:17 -0400 Subject: [PATCH 28/78] Test nested sandboxing, and make nicer error We were bedeviled by sandboxing issues when working on the layered store. The problem ended up being that when we have nested nix builds, and the inner store is inside the build dir (e.g. store is `/build/nix-test/$name/store`, build dir is `/build`) bind mounts clobber each other and store paths cannot be found. After thoroughly cleaning up `local-derivation-goal.cc`, we might be able to make that work. But that is a lot of work. For now, we just fail earlier with a proper error message. Finally, test this: nested sandboxing without the problematic store dir should work, and with should fail with the expected error message. Co-authored-by: Dylan Green <67574902+cidkidnix@users.noreply.github.com> Co-authored-by: Robert Hensing --- src/libstore/build/local-derivation-goal.cc | 4 +++ tests/local.mk | 3 ++- tests/nested-sandboxing.sh | 11 ++++++++ tests/nested-sandboxing/command.sh | 29 +++++++++++++++++++++ tests/nested-sandboxing/runner.nix | 24 +++++++++++++++++ 5 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 tests/nested-sandboxing.sh create mode 100644 tests/nested-sandboxing/command.sh create mode 100644 tests/nested-sandboxing/runner.nix diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ee66ee500..e22a522a2 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -594,6 +594,10 @@ void LocalDerivationGoal::startBuilder() else dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } + if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) + { + throw Error("`sandbox-build-dir` must not contain the storeDir"); + } dirsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ diff --git a/tests/local.mk b/tests/local.mk index bb80c5317..173bc84b3 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -138,7 +138,8 @@ nix_tests = \ path-from-hash-part.sh \ test-libstoreconsumer.sh \ toString-path.sh \ - read-only-store.sh + read-only-store.sh \ + nested-sandboxing.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/nested-sandboxing.sh b/tests/nested-sandboxing.sh new file mode 100644 index 000000000..d9fa788aa --- /dev/null +++ b/tests/nested-sandboxing.sh @@ -0,0 +1,11 @@ +source common.sh +# This test is run by `tests/nested-sandboxing/runner.nix` in an extra layer of sandboxing. +[[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" + +requireSandboxSupport + +source ./nested-sandboxing/command.sh + +expectStderr 100 runNixBuild badStoreUrl 2 | grepQuiet '`sandbox-build-dir` must not contain' + +runNixBuild goodStoreUrl 5 diff --git a/tests/nested-sandboxing/command.sh b/tests/nested-sandboxing/command.sh new file mode 100644 index 000000000..69366486c --- /dev/null +++ b/tests/nested-sandboxing/command.sh @@ -0,0 +1,29 @@ +export NIX_BIN_DIR=$(dirname $(type -p nix)) +# TODO Get Nix and its closure more flexibly +export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" + +badStoreUrl () { + local altitude=$1 + echo $TEST_ROOT/store-$altitude +} + +goodStoreUrl () { + local altitude=$1 + echo $("badStoreUrl" "$altitude")?store=/foo-$altitude +} + +# The non-standard sandbox-build-dir helps ensure that we get the same behavior +# whether this test is being run in a derivation as part of the nix build or +# being manually run by a developer outside a derivation +runNixBuild () { + local storeFun=$1 + local altitude=$2 + nix-build \ + --no-substitute --no-out-link \ + --store "$("$storeFun" "$altitude")" \ + --extra-sandbox-paths "$EXTRA_SANDBOX" \ + ./nested-sandboxing/runner.nix \ + --arg altitude "$((altitude - 1))" \ + --argstr storeFun "$storeFun" \ + --sandbox-build-dir /build-non-standard +} diff --git a/tests/nested-sandboxing/runner.nix b/tests/nested-sandboxing/runner.nix new file mode 100644 index 000000000..9a5822c88 --- /dev/null +++ b/tests/nested-sandboxing/runner.nix @@ -0,0 +1,24 @@ +{ altitude, storeFun }: + +with import ../config.nix; + +mkDerivation { + name = "nested-sandboxing"; + busybox = builtins.getEnv "busybox"; + EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX"; + buildCommand = if altitude == 0 then '' + echo Deep enough! > $out + '' else '' + cp -r ${../common} ./common + cp ${../common.sh} ./common.sh + cp ${../config.nix} ./config.nix + cp -r ${./.} ./nested-sandboxing + + export PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + source common.sh + source ./nested-sandboxing/command.sh + + runNixBuild ${storeFun} ${toString altitude} >> $out + ''; +} From a5c88f860987bd5dec8c96efed1e6c9d8ce7a669 Mon Sep 17 00:00:00 2001 From: Sinan Mohd <69694713+sinanmohd@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:25:11 +0000 Subject: [PATCH 29/78] Nix Reference Manual: keep nix expressions uptodate with nixpkgs (#8703) --- doc/manual/src/language/constructs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index c53eb8889..2482f2d17 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -92,10 +92,10 @@ In this fragment from `all-packages.nix`, ```nix graphviz = (import ../tools/graphics/graphviz) { inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xlibs) libXaw; + inherit (xorg) libXaw; }; -xlibs = { +xorg = { libX11 = ...; libXaw = ...; ... @@ -109,7 +109,7 @@ libjpg = ...; the set used in the function call to the function defined in `../tools/graphics/graphviz` inherits a number of variables from the surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set. +(the X Athena Widgets) from the `xorg` set. Summarizing the fragment From 259e328de81a91cf824efb0603f57fcde94ad3ff Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Jul 2023 22:24:51 -0400 Subject: [PATCH 30/78] Introduce notion of a test group, use for CA tests Grouping our tests should make it easier to understand the intent than one long poorly-arranged list. It also is convenient for running just the tests for a specific component when working on that component. We need at least one test group so this isn't dead code; I decided to collect the tests for the `ca-derivations` and `dynamic-derivations` experimental features in groups. Do ```bash make ca.test-group -jN ``` and ```bash make dyn-drv.test-group -jN ``` to try running just them. I originally did this as part of #8397 for being able to just the local overlay store alone. I am PRing it separately now so we can separate general infra from new features. Co-authored-by: Valentin Gagarin --- Makefile | 2 ++ doc/manual/src/contributing/testing.md | 29 ++++++++++++++++++++++++++ mk/lib.mk | 20 +++++++++++++++++- mk/tests.mk | 8 +++++-- tests/ca/local.mk | 27 ++++++++++++++++++++++++ tests/dyn-drv/local.mk | 11 ++++++++++ tests/local.mk | 29 +++----------------------- 7 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 tests/ca/local.mk create mode 100644 tests/dyn-drv/local.mk diff --git a/Makefile b/Makefile index c6220482a..31b54b93d 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,8 @@ makefiles += \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ tests/local.mk \ + tests/ca/local.mk \ + tests/dyn-drv/local.mk \ tests/test-libstoreconsumer/local.mk \ tests/plugins/local.mk else diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index a5253997d..c3c82e3c0 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. Each test is a bash script. +### Running the whole test suite + The whole test suite can be run with: ```shell-session @@ -23,6 +25,33 @@ ran test tests/bar.sh... [PASS] ... ``` +### Grouping tests + +Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. +Each test group is in a subdirectory of `tests`. +For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. + +That test group can be run like this: + +```shell-session +$ make ca.test-group -j50 +ran test tests/ca/nix-run.sh... [PASS] +ran test tests/ca/import-derivation.sh... [PASS] +... +``` + +The test group is defined in Make like this: +```makefile +$(test-group-name)-tests := \ + $(d)/test0.sh \ + $(d)/test1.sh \ + ... + +install-tests-groups += $(test-group-name) +``` + +### Running individual tests + Individual tests can be run with `make`: ```shell-session diff --git a/mk/lib.mk b/mk/lib.mk index 34fa624d8..e86a7f1a4 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,6 +10,7 @@ bin-scripts := noinst-scripts := man-pages := install-tests := +install-tests-groups := ifdef HOST_OS HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) @@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +$(foreach test, $(install-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval installcheck: $(test).test)) +$(foreach test-group, $(install-tests-groups), \ + $(eval $(call run-install-test-group,$(test-group))) \ + $(eval installcheck: $(test-group).test-group) \ + $(foreach test, $($(test-group)-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval $(test-group).test-group: $(test).test))) + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) @@ -151,6 +161,14 @@ ifdef libs-list @echo "The following libraries can be built:" @echo "" @for i in $(libs-list); do echo " $$i"; done +endif +ifdef install-tests-groups + @echo "" + @echo "The following groups of functional tests can be run:" + @echo "" + @for i in $(install-tests-groups); do echo " $$i.test-group"; done + @echo "" + @echo "(installcheck includes tests in test groups too.)" endif @echo "" @echo "The following variables control the build:" diff --git a/mk/tests.mk b/mk/tests.mk index 3ebbd86e3..ec8128bdf 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -4,8 +4,6 @@ test-deps = define run-install-test - installcheck: $1.test - .PHONY: $1.test $1.test: $1 $(test-deps) @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null @@ -16,6 +14,12 @@ define run-install-test endef +define run-install-test-group + + .PHONY: $1.test-group + +endef + .PHONY: check installcheck print-top-help += \ diff --git a/tests/ca/local.mk b/tests/ca/local.mk new file mode 100644 index 000000000..d15312708 --- /dev/null +++ b/tests/ca/local.mk @@ -0,0 +1,27 @@ +ca-tests := \ + $(d)/build-with-garbage-path.sh \ + $(d)/build.sh \ + $(d)/concurrent-builds.sh \ + $(d)/derivation-json.sh \ + $(d)/duplicate-realisation-in-closure.sh \ + $(d)/gc.sh \ + $(d)/import-derivation.sh \ + $(d)/new-build-cmd.sh \ + $(d)/nix-copy.sh \ + $(d)/nix-run.sh \ + $(d)/nix-shell.sh \ + $(d)/post-hook.sh \ + $(d)/recursive.sh \ + $(d)/repl.sh \ + $(d)/selfref-gc.sh \ + $(d)/signatures.sh \ + $(d)/substitute.sh \ + $(d)/why-depends.sh + +install-tests-groups += ca + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/ca/config.nix diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk new file mode 100644 index 000000000..f065a5627 --- /dev/null +++ b/tests/dyn-drv/local.mk @@ -0,0 +1,11 @@ +dyn-drv-tests := \ + $(d)/text-hashed-output.sh \ + $(d)/recursive-mod-json.sh + +install-tests-groups += dyn-drv + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/dyn-drv/config.nix diff --git a/tests/local.mk b/tests/local.mk index df20f3dd7..2afe91220 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -14,7 +14,6 @@ nix_tests = \ flakes/absolute-paths.sh \ flakes/build-paths.sh \ flakes/flake-in-submodule.sh \ - ca/gc.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ @@ -28,8 +27,6 @@ nix_tests = \ user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ - ca/build.sh \ - ca/new-build-cmd.sh \ nix-build.sh \ gc-concurrent.sh \ repair.sh \ @@ -47,24 +44,17 @@ nix_tests = \ referrers.sh \ optimise-store.sh \ substitute-with-invalid-ca.sh \ - ca/concurrent-builds.sh \ signing.sh \ - ca/build-with-garbage-path.sh \ hash.sh \ gc-non-blocking.sh \ check.sh \ - ca/substitute.sh \ nix-shell.sh \ - ca/signatures.sh \ - ca/nix-shell.sh \ - ca/nix-copy.sh \ check-refs.sh \ build-remote-input-addressed.sh \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ flakes/search-root.sh \ - ca/duplicate-realisation-in-closure.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -80,10 +70,7 @@ nix_tests = \ nar-access.sh \ pure-eval.sh \ eval.sh \ - ca/post-hook.sh \ repl.sh \ - ca/repl.sh \ - ca/recursive.sh \ binary-cache-build-remote.sh \ search.sh \ logging.sh \ @@ -109,13 +96,8 @@ nix_tests = \ fmt.sh \ eval-store.sh \ why-depends.sh \ - ca/why-depends.sh \ derivation-json.sh \ - ca/derivation-json.sh \ import-derivation.sh \ - ca/import-derivation.sh \ - dyn-drv/text-hashed-output.sh \ - dyn-drv/recursive-mod-json.sh \ nix_path.sh \ case-hack.sh \ placeholders.sh \ @@ -124,8 +106,7 @@ nix_tests = \ build.sh \ build-delete.sh \ output-normalization.sh \ - ca/nix-run.sh \ - selfref-gc.sh ca/selfref-gc.sh \ + selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ @@ -150,16 +131,12 @@ install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) clean-files += \ $(d)/common/vars-and-functions.sh \ - $(d)/config.nix \ - $(d)/ca/config.nix \ - $(d)/dyn-drv/config.nix + $(d)/config.nix test-deps += \ tests/common/vars-and-functions.sh \ tests/config.nix \ - tests/ca/config.nix \ - tests/test-libstoreconsumer/test-libstoreconsumer \ - tests/dyn-drv/config.nix + tests/test-libstoreconsumer/test-libstoreconsumer ifeq ($(BUILD_SHARED_LIBS), 1) test-deps += tests/plugins/libplugintest.$(SO_EXT) From ee72ede389a7a8b9ac2d0f908b2524950d72f303 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 5 Mar 2023 03:59:17 +0100 Subject: [PATCH 31/78] remove the Channels section this is a how-to guide which should not be in the reference manual. it also refers to `nix-env`, which should not be the first thing readers of the reference manual encounter, as it behaves very differently in spirit from the rest of Nix. slightly reword the documentation to be more concise and informative. --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/command-ref/nix-channel.md | 70 +++++++++++-------- .../package-management/basic-package-mgmt.md | 2 +- doc/manual/src/package-management/channels.md | 50 ------------- 4 files changed, 41 insertions(+), 82 deletions(-) delete mode 100644 doc/manual/src/package-management/channels.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 1bd8fa774..f2e69178f 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -21,7 +21,6 @@ - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - - [Channels](package-management/channels.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 025f758e7..8c0dcdfaa 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -8,36 +8,41 @@ # Description -A Nix channel is a mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing a set of Nix expressions. +Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. +For the list of official channels, visit . -To see the list of official NixOS channels, visit -. +> **Note** +> +> The state of a subscribed channel is external to the Nix expressions relying on it. +> This may limit reproducibility. +> +> Dependencies on other Nix expressions can be declared explicitly with: +> - [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl), [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball), or [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) in Nix expressions +> - the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) in command line invocations This command has the following operations: - `--add` *url* \[*name*\]\ - Adds a channel named *name* with URL *url* to the list of subscribed - channels. If *name* is omitted, it defaults to the last component of - *url*, with the suffixes `-stable` or `-unstable` removed. + Add a channel *name* located at *url* to the list of subscribed channels. + If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. + + > **Note** + > + > `--add` does not automatically perform an update. + > Use `--update` explicitly. A channel URL must point to a directory containing a file `nixexprs.tar.gz`. At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. - `--remove` *name*\ - Removes the channel named *name* from the list of subscribed - channels. + Remove the channel *name* from the list of subscribed channels. - `--list`\ - Prints the names and URLs of all subscribed channels on standard - output. + Print the names and URLs of all subscribed channels on standard output. - `--update` \[*names*…\]\ - Downloads the Nix expressions of all subscribed channels (or only - those included in *names* if specified) and makes them the default - for `nix-env` operations (by symlinking them from the directory - `~/.nix-defexpr`). + Download the Nix expressions of subscribed channels and create a new generation. + Update all channels if none is specified, and only those included in *names* otherwise. - `--list-generations`\ Prints a list of all the current existing generations for the @@ -49,13 +54,8 @@ This command has the following operations: ``` - `--rollback` \[*generation*\]\ - Reverts the previous call to `nix-channel - --update`. Optionally, you can specify a specific channel generation - number to restore. - -Note that `--add` does not automatically perform an update. - -The list of subscribed channels is stored in `~/.nix-channels`. + Revert channels to the state before the last call to `nix-channel --update`. + Optionally, you can specify a specific channel *generation* number to restore. {{#include ./opt-common.md}} @@ -69,23 +69,33 @@ The list of subscribed channels is stored in `~/.nix-channels`. # Examples -To subscribe to the Nixpkgs channel and install the GNU Hello package: +Subscribe to the Nixpkgs channel and run `hello` from the GNU Hello package: ```console $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --list +nixpkgs https://nixos.org/channels/nixpkgs $ nix-channel --update -$ nix-env --install --attr nixpkgs.hello +$ nix-shell -p hello --run hello +hello ``` -You can revert channel updates using `--rollback`: +Revert channel updates using `--rollback`: ```console -$ nix-instantiate --eval --expr '(import {}).lib.version' -"14.04.527.0e935f1" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre296212.530a53dcbc9" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval --expr '(import {}).lib.version' -"14.04.526.dbadfad" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre281526.d0419badfad" +``` + +Remove a channel: + +```console +$ nix-channel --remove nixpkgs +$ nix-channel --list ``` diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md index 6b86e763e..07b92fb76 100644 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ b/doc/manual/src/package-management/basic-package-mgmt.md @@ -25,7 +25,7 @@ or completely new ones.) You can manually download the latest version of Nixpkgs from . However, it’s much more -convenient to use the Nixpkgs [*channel*](channels.md), since it makes +convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is automatically added to your list of “subscribed” channels when you install Nix. If this is not the case for some reason, you can add it diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md deleted file mode 100644 index 8e4da180b..000000000 --- a/doc/manual/src/package-management/channels.md +++ /dev/null @@ -1,50 +0,0 @@ -# Channels - -If you want to stay up to date with a set of packages, it’s not very -convenient to manually download the latest set of Nix expressions for -those packages and upgrade using `nix-env`. Fortunately, there’s a -better way: *Nix channels*. - -A Nix channel is just a URL that points to a place that contains a set -of Nix expressions and a manifest. Using the command -[`nix-channel`](../command-ref/nix-channel.md) you can automatically -stay up to date with whatever is available at that URL. - -To see the list of official NixOS channels, visit -. - -You can “subscribe” to a channel using `nix-channel --add`, e.g., - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -``` - -subscribes you to a channel that always contains that latest version of -the Nix Packages collection. (Subscribing really just means that the URL -is added to the file `~/.nix-channels`, where it is read by subsequent -calls to `nix-channel ---update`.) You can “unsubscribe” using `nix-channel ---remove`: - -```console -$ nix-channel --remove nixpkgs -``` - -To obtain the latest Nix expressions available in a channel, do - -```console -$ nix-channel --update -``` - -This downloads and unpacks the Nix expressions in every channel -(downloaded from `url/nixexprs.tar.bz2`). It also makes the union of -each channel’s Nix expressions available by default to `nix-env` -operations (via the symlink `~/.nix-defexpr/channels`). Consequently, -you can then say - -```console -$ nix-env --upgrade -``` - -to upgrade all packages in your profile to the latest versions available -in the subscribed channels. From cd0e39bd899b51d0b1f139117b1fd6c1280caa70 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 21 Jun 2023 11:54:23 +0200 Subject: [PATCH 32/78] remove redundant information from channel profile description --- doc/manual/src/command-ref/files/channels.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/files/channels.md b/doc/manual/src/command-ref/files/channels.md index 7b1f27128..691468cf8 100644 --- a/doc/manual/src/command-ref/files/channels.md +++ b/doc/manual/src/command-ref/files/channels.md @@ -1,11 +1,10 @@ ## Channels -A directory containing symlinks to Nix channels, managed by [`nix-channel`]: +[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels: - `$XDG_STATE_HOME/nix/profiles/channels` for regular users - `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` -[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels. This profile contains symlinks to the contents of those channels. ## Subscribed channels From 4bab5a62081a792530de8c4ae2cd163c1bb581a5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 09:42:53 +0200 Subject: [PATCH 33/78] revert channel files overview --- doc/manual/src/command-ref/files/channels.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/files/channels.md b/doc/manual/src/command-ref/files/channels.md index 691468cf8..7b1f27128 100644 --- a/doc/manual/src/command-ref/files/channels.md +++ b/doc/manual/src/command-ref/files/channels.md @@ -1,10 +1,11 @@ ## Channels -[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels: +A directory containing symlinks to Nix channels, managed by [`nix-channel`]: - `$XDG_STATE_HOME/nix/profiles/channels` for regular users - `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` +[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels. This profile contains symlinks to the contents of those channels. ## Subscribed channels From e14c8a359efcacd267064b9c7d28adb0f41a168e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 10:25:05 +0200 Subject: [PATCH 34/78] list moving parts of channels --- doc/manual/src/command-ref/nix-channel.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 8c0dcdfaa..cebbc7b00 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -9,7 +9,12 @@ # Description Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. -For the list of official channels, visit . + +The moving parts of channels are: +- The official channels listed at +- The user-specific list of [subscribed channels](#subscribed-channels) +- The [downloaded channel contents](#channels) +- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH) > **Note** > From 68b7bb1a062c6471b678d5488a71324e0486d405 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 11 May 2023 15:51:48 +0200 Subject: [PATCH 35/78] add information on the system type string --- configure.ac | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index bb3f92e4d..26df39d54 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,12 @@ AC_CONFIG_AUX_DIR(config) AC_PROG_SED -# Construct a Nix system name (like "i686-linux"). +# Construct a Nix system name (like "i686-linux"): +# https://www.gnu.org/software/autoconf/manual/html_node/Canonicalizing.html#index-AC_005fCANONICAL_005fHOST-1 +# The inital value is produced by the `config.guess` script: +# https://git.savannah.gnu.org/cgit/config.git/tree/config.guess +# It has the following form, which is not documented anywhere: +# --[][-] AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) From 0751c1bfc6375ffbe94e90cfcfdb987e8b286f2e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 31 May 2023 03:21:11 +0200 Subject: [PATCH 36/78] one line per sentence for easier review --- src/libstore/globals.hh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d4b8fb1f9..76afeb5f5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -193,18 +193,12 @@ public: Setting thisSystem{ this, SYSTEM, "system", R"( - This option specifies the canonical Nix system name of the current - installation, such as `i686-linux` or `x86_64-darwin`. Nix can only - build derivations whose `system` attribute equals the value - specified here. In general, it never makes sense to modify this - value from its default, since you can use it to ‘lie’ about the - platform you are building on (e.g., perform a Mac OS build on a - Linux machine; the result would obviously be wrong). It only makes - sense if the Nix binaries can run on multiple platforms, e.g., - ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. + This option specifies the canonical Nix system name of the current installation, such as `i686-linux` or `x86_64-darwin`. + Nix can only build derivations whose `system` attribute equals the value specified here. + In general, it never makes sense to modify this value from its default, since you can use it to ‘lie’ about the platform you are building on (e.g., perform a Mac OS build on a Linux machine; the result would obviously be wrong). + It only makes sense if the Nix binaries can run on multiple platforms, e.g., ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. - It defaults to the canonical Nix system name detected by `configure` - at build time. + It defaults to the canonical Nix system name detected by `configure` at build time. )"}; Setting maxSilentTime{ From c8a42039eae4c046297517b67a3ce43685d3e756 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 31 May 2023 03:48:05 +0200 Subject: [PATCH 37/78] move docs of the current system to the system setting add information what happens when Nix itself is cross-compiled --- configure.ac | 2 ++ src/libstore/globals.hh | 45 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 26df39d54..cc7009859 100644 --- a/configure.ac +++ b/configure.ac @@ -11,6 +11,8 @@ AC_PROG_SED # https://git.savannah.gnu.org/cgit/config.git/tree/config.guess # It has the following form, which is not documented anywhere: # --[][-] +# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config.sub` instead: +# https://git.savannah.gnu.org/cgit/config.git/tree/config.sub AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 76afeb5f5..1fb5927d0 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -193,12 +193,49 @@ public: Setting thisSystem{ this, SYSTEM, "system", R"( - This option specifies the canonical Nix system name of the current installation, such as `i686-linux` or `x86_64-darwin`. - Nix can only build derivations whose `system` attribute equals the value specified here. - In general, it never makes sense to modify this value from its default, since you can use it to ‘lie’ about the platform you are building on (e.g., perform a Mac OS build on a Linux machine; the result would obviously be wrong). + The system type of the current Nix installation. + + Nix can only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. + In general, it never makes sense to modify this value, since you can use it to ‘lie’ about the system you are building on (e.g., perform a macOS build on a Linux machine; the result would obviously be wrong). It only makes sense if the Nix binaries can run on multiple platforms, e.g., ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. - It defaults to the canonical Nix system name detected by `configure` at build time. + The default value is set when Nix itself is compiled for the system it will run on. + The following system types are widely used, as Nix is actively supported on these platforms: + + - `x86_64-linux` + - `x86_64-darwin` + - `i686-linux` + - `aarch64-linux` + - `aarch64-darwin` + + The concrete value is based on the output of [`config.guess`](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess): + + ``` + --[][-] + ``` + + When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub): + + ``` + -[-]- + ``` + + Nix only uses the CPU and OS identifiers: + + ``` + -[-] + ``` + + For historic reasons and backwards-compatibility, some CPU and OS identifiers are transformed as follows: + + | `config.guess` | Nix | + |----------------------------|---------------------| + | `amd64` | `x86_64` | + | `i*86` | `i686` | + | `arm6` | `arm6l` | + | `arm7` | `arm7l` | + | `linux-gnu*` | `linux` | + | `linux-musl*` | `linux` | )"}; Setting maxSilentTime{ From 3763c7bb5ef5a7a1ddcf47169c9733edb7989e98 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 13:42:23 +0200 Subject: [PATCH 38/78] shorten `system` setting description --- src/libstore/globals.hh | 38 ++++++-------------------------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1fb5927d0..442d72911 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -194,12 +194,12 @@ public: this, SYSTEM, "system", R"( The system type of the current Nix installation. - - Nix can only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. - In general, it never makes sense to modify this value, since you can use it to ‘lie’ about the system you are building on (e.g., perform a macOS build on a Linux machine; the result would obviously be wrong). - It only makes sense if the Nix binaries can run on multiple platforms, e.g., ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. + Nix will only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. The default value is set when Nix itself is compiled for the system it will run on. + In general, it never makes sense to modify this value. + While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. + The following system types are widely used, as Nix is actively supported on these platforms: - `x86_64-linux` @@ -207,35 +207,9 @@ public: - `i686-linux` - `aarch64-linux` - `aarch64-darwin` + - `armv6l-linux` + - `armv7l-linux` - The concrete value is based on the output of [`config.guess`](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess): - - ``` - --[][-] - ``` - - When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub): - - ``` - -[-]- - ``` - - Nix only uses the CPU and OS identifiers: - - ``` - -[-] - ``` - - For historic reasons and backwards-compatibility, some CPU and OS identifiers are transformed as follows: - - | `config.guess` | Nix | - |----------------------------|---------------------| - | `amd64` | `x86_64` | - | `i*86` | `i686` | - | `arm6` | `arm6l` | - | `arm7` | `arm7l` | - | `linux-gnu*` | `linux` | - | `linux-musl*` | `linux` | )"}; Setting maxSilentTime{ From 4944e37ec04cc8c62f35aa9b1f4e97ee8f233eb8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 20 Jun 2023 14:10:30 +0200 Subject: [PATCH 39/78] expand on the system type in hacking guide --- configure.ac | 8 +-- doc/manual/src/contributing/hacking.md | 69 +++++++++++++++++++------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/configure.ac b/configure.ac index cc7009859..6d78237f0 100644 --- a/configure.ac +++ b/configure.ac @@ -7,12 +7,12 @@ AC_PROG_SED # Construct a Nix system name (like "i686-linux"): # https://www.gnu.org/software/autoconf/manual/html_node/Canonicalizing.html#index-AC_005fCANONICAL_005fHOST-1 -# The inital value is produced by the `config.guess` script: -# https://git.savannah.gnu.org/cgit/config.git/tree/config.guess +# The inital value is produced by the `config/config.guess` script: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.guess # It has the following form, which is not documented anywhere: # --[][-] -# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config.sub` instead: -# https://git.savannah.gnu.org/cgit/config.git/tree/config.sub +# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config/config.sub` instead: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.sub AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 7b2440971..c7cb0d980 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -110,41 +110,72 @@ You can also build Nix for one of the [supported platforms](#platforms). ## Platforms -As specified in [`flake.nix`], Nix can be built for various platforms: - -- `aarch64-linux` -- `i686-linux` -- `x86_64-darwin` -- `x86_64-linux` +Nix can be built for various platforms, as specified in [`flake.nix`]: [`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix +- `x86_64-linux` +- `x86_64-darwin` +- `i686-linux` +- `aarch64-linux` +- `aarch64-darwin` +- `armv6l-linux` +- `armv7l-linux` + In order to build Nix for a different platform than the one you're currently -on, you need to have some way for your system Nix to build code for that -platform. Common solutions include [remote builders] and [binfmt emulation] +on, you need a way for your current Nix installation to build code for that +platform. Common solutions include [remote builders] and [binary format emulation] (only supported on NixOS). [remote builders]: ../advanced-topics/distributed-builds.md [binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems -These solutions let Nix perform builds as if you're on the native platform, so -executing the build is as simple as - -```console -$ nix build .#packages.aarch64-linux.default -``` - -for flake-enabled Nix, or +Given such a setup, executing the build only requires selecting the respective output attribute. +For example, to compile for `aarch64-linux`: ```console $ nix-build --attr packages.aarch64-linux.default ``` -for classic Nix. +or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled: -You can use any of the other supported platforms in place of `aarch64-linux`. +```console +$ nix build .#packages.aarch64-linux.default +``` -Cross-compiled builds are available for ARMv6 and ARMv7, and Nix on unsupported platforms can be bootstrapped by adding more `crossSystems` in `flake.nix`. +Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. + +## System type + +Nix uses a string with he following format to identify the *system type* or *platform* it runs on: + +``` +-[-] +``` + +It is set when Nix is compiled for the given system, and based on the output of [`config.guess`](https://github.com/nixos/nix/blob/master/config/config.guess) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess)): + +``` +--[][-] +``` + +When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://github.com/nixos/nix/blob/master/config/config.sub) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub)): + +``` +-[-]- +``` + +For historic reasons and backward-compatibility, some CPU and OS identifiers are transformed as follows in [`configure.ac`](https://github.com/nixos/nix/blob/master/config/config.sub): + +| `config.guess` | Nix | +|----------------------------|---------------------| +| `amd64` | `x86_64` | +| `i*86` | `i686` | +| `arm6` | `arm6l` | +| `arm7` | `arm7l` | +| `linux-gnu*` | `linux` | +| `linux-musl*` | `linux` | ## Compilation environments From 32de11923e932b5bbda8ce4877542fb299de9b03 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 22 Jun 2023 13:50:00 +0200 Subject: [PATCH 40/78] add cross-links --- src/libstore/globals.hh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 442d72911..760655302 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -197,10 +197,7 @@ public: Nix will only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. The default value is set when Nix itself is compiled for the system it will run on. - In general, it never makes sense to modify this value. - While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. - - The following system types are widely used, as Nix is actively supported on these platforms: + The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): - `x86_64-linux` - `x86_64-darwin` @@ -210,6 +207,10 @@ public: - `armv6l-linux` - `armv7l-linux` + In general, it never makes sense to modify this value when building derivations. + While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. + + This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem). )"}; Setting maxSilentTime{ From c8f04e2024d3e66f620ae38258fa86eb6103ce05 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 13 Jul 2023 22:00:16 +0200 Subject: [PATCH 41/78] note that naming convention is from Autotools Co-authored-by: Robert Hensing --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index c7cb0d980..ff4690f1e 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -166,7 +166,7 @@ When Nix is built such that `./configure` is passed any of the `--host`, `--buil -[-]- ``` -For historic reasons and backward-compatibility, some CPU and OS identifiers are transformed as follows in [`configure.ac`](https://github.com/nixos/nix/blob/master/config/config.sub): +For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/config/config.sub) as follows: | `config.guess` | Nix | |----------------------------|---------------------| From 1a220bed9361be2c360f83eb42025f9765c96f34 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 23:21:43 +0200 Subject: [PATCH 42/78] do not mention output attributes Co-authored-by: Robert Hensing --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index ff4690f1e..b8ea977d7 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -130,7 +130,7 @@ platform. Common solutions include [remote builders] and [binary format emulatio [remote builders]: ../advanced-topics/distributed-builds.md [binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems -Given such a setup, executing the build only requires selecting the respective output attribute. +Given such a setup, executing the build only requires selecting the respective attribute. For example, to compile for `aarch64-linux`: ```console From aba32def7390432855401b815ff21915aba3eb81 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 23:26:36 +0200 Subject: [PATCH 43/78] fix wording Co-authored-by: Robert Hensing --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 760655302..63ac54e97 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -207,7 +207,7 @@ public: - `armv6l-linux` - `armv7l-linux` - In general, it never makes sense to modify this value when building derivations. + In general, you do not have to modify this setting. While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem). From fcadac0a02b30d1d876c804edf7f2745d880a10c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 10:33:41 +0200 Subject: [PATCH 44/78] mention `extra-platforms` --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 63ac54e97..ad02bfb42 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -194,7 +194,7 @@ public: this, SYSTEM, "system", R"( The system type of the current Nix installation. - Nix will only build [derivations](@docroot@/language/derivations.md) whose `system` attribute equals the value specified here. + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals the value specified here or in [`extra-platforms`](@docroot@/command-ref/conf-file.html#conf-extra-platforms). The default value is set when Nix itself is compiled for the system it will run on. The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): From 0779005f4945d373b81c933a6b7390c8bc7ad6cb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 10:57:37 +0200 Subject: [PATCH 45/78] expand on the `extra-platforms` option --- src/libstore/globals.hh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ad02bfb42..723d18d74 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -194,7 +194,7 @@ public: this, SYSTEM, "system", R"( The system type of the current Nix installation. - Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals the value specified here or in [`extra-platforms`](@docroot@/command-ref/conf-file.html#conf-extra-platforms). + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms). The default value is set when Nix itself is compiled for the system it will run on. The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): @@ -676,18 +676,20 @@ public: getDefaultExtraPlatforms(), "extra-platforms", R"( - Platforms other than the native one which this machine is capable of - building for. This can be useful for supporting additional - architectures on compatible machines: i686-linux can be built on - x86\_64-linux machines (and the default for this setting reflects - this); armv7 is backwards-compatible with armv6 and armv5tel; some - aarch64 machines can also natively run 32-bit ARM code; and - qemu-user may be used to support non-native platforms (though this - may be slow and buggy). Most values for this are not enabled by - default because build systems will often misdetect the target - platform and generate incompatible code, so you may wish to - cross-check the results of using this option against proper - natively-built versions of your derivations. + System types of executables that can be run on this machine. + + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in the [`system` option](#conf-system). + + Setting this can be useful to build derivations locally on compatible machines: + - `i686-linux` executables can be run on `x86_64-linux` machines (set by default) + - `x86_64-darwin` executables can be run on macOS `aarch64-darwin` with Rosetta 2 (set by default where applicable) + - `armv6` and `armv5tel` executables can be run on `armv7` + - some `aarch64` machines can also natively run 32-bit ARM code + - `qemu-user` may be used to support non-native platforms (though this + may be slow and buggy) + + Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation. + You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation. )", {}, false}; Setting systemFeatures{ From 6c3cd429a6aabe8673af99d6bc0653d376dd69b8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 11:01:48 +0200 Subject: [PATCH 46/78] fix broken links --- doc/manual/redirects.js | 2 +- src/libstore/globals.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index dcdb5d6e9..b43622ed6 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -281,7 +281,7 @@ const redirects = { "chap-introduction": "introduction.html", "ch-basic-package-mgmt": "package-management/basic-package-mgmt.html", "ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html", - "sec-channels": "package-management/channels.html", + "sec-channels": "command-ref/nix-channel.html", "ssec-copy-closure": "package-management/copy-closure.html", "sec-garbage-collection": "package-management/garbage-collection.html", "ssec-gc-roots": "package-management/garbage-collector-roots.html", diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d4b8fb1f9..2d8cac656 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1010,7 +1010,7 @@ public: | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | - If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option. + If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/command-ref/nix-channel.md), you should migrate manually when you enable this option. If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`. This can be achieved with the following shell commands: From b0173716f6b27b4fb307ac9ded544e46e712ad22 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 15:07:07 +0200 Subject: [PATCH 47/78] clarify wording on args@ default handling (#8596) * clarify wording on args@ default handling Most importantly use shorter sentences and emphasize the key point that defaults aren't taken into account Co-authored-by: Robert Hensing Co-authored-by: John Ericson --- doc/manual/src/language/constructs.md | 45 +++++++++++++++++---------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index 2482f2d17..a3590f55d 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -208,30 +208,41 @@ three kinds of patterns: ```nix { x, y, z, ... } @ args: z + y + x + args.a ``` - - Here `args` is bound to the entire argument, which is further - matched against the pattern `{ x, y, z, - ... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as you can access attribute names as `a`, using `args.a`, which was given as an additional attribute to the function. - + > **Warning** - > - > The `args@` expression is bound to the argument passed to the - > function which means that attributes with defaults that aren't - > explicitly specified in the function call won't cause an - > evaluation error, but won't exist in `args`. - > + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > > For instance - > + > > ```nix > let - > function = args@{ a ? 23, ... }: args; + > f = args@{ a ? 23, ... }: [ a args ]; > in - > function {} - > ```` - > - > will evaluate to an empty attribute set. + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` Note that functions do not have names. If you want to give them a name, you can bind them to an attribute, e.g., From 0e4f6dfcf7303bfe3c5d8af6863d57b3e70d370a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 20 Jul 2023 10:27:38 +0200 Subject: [PATCH 48/78] revert anchor prefix for builtin constants the original change broke many pre-existing anchor links. also change formatting of the constants listing slightly: - the type should not be part of the anchor - add highlight to the "impure only" note --- doc/manual/generate-builtin-constants.nix | 8 +++++--- src/libexpr/eval.hh | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix index 3fc1fae42..8af80a02c 100644 --- a/doc/manual/generate-builtin-constants.nix +++ b/doc/manual/generate-builtin-constants.nix @@ -10,12 +10,14 @@ let type' = optionalString (type != null) " (${type})"; impureNotice = optionalString impure-only '' - Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + > **Note** + > + > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). ''; in squash '' -
- ${name}${type'} +
+ ${name}${type'}
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 277e77ad5..46fa96d05 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -806,7 +806,7 @@ struct EvalSettings : Config List of directories to be searched for `<...>` file references In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath). + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). )"}; Setting restrictEval{ From 85d0eb63165e7d7f441fe3dd94bb548a40502e52 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 20 Jul 2023 17:58:14 +0200 Subject: [PATCH 49/78] fix broken links (#8722) --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index b8ea977d7..4b0a3a3e5 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -128,7 +128,7 @@ platform. Common solutions include [remote builders] and [binary format emulatio (only supported on NixOS). [remote builders]: ../advanced-topics/distributed-builds.md -[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems +[binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems Given such a setup, executing the build only requires selecting the respective attribute. For example, to compile for `aarch64-linux`: @@ -166,7 +166,7 @@ When Nix is built such that `./configure` is passed any of the `--host`, `--buil -[-]- ``` -For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/config/config.sub) as follows: +For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows: | `config.guess` | Nix | |----------------------------|---------------------| From 7b30293d389ea75ee24ec1f2a4a4187f175757ab Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Jul 2023 15:50:25 -0400 Subject: [PATCH 50/78] Tighten `#include`s: `DerivedPath` doesn't care about `Realisation` --- src/libcmd/built-path.hh | 1 + src/libstore/derived-path.hh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index c563a46e9..744e8090b 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -1,4 +1,5 @@ #include "derived-path.hh" +#include "realisation.hh" namespace nix { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 6ea80c92e..7a4261ce0 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,7 +3,6 @@ #include "util.hh" #include "path.hh" -#include "realisation.hh" #include "outputs-spec.hh" #include "comparator.hh" From f62543fe1cc544a5684af327f63a1aeb1bdeba94 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 20 Jul 2023 09:58:25 -0400 Subject: [PATCH 51/78] Remove unneeded copy It appeared in 8eb73a87245acf9d93dc401831b629981864fa58 (by me!) without justification. --- src/libstore/local-store.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e69460e6c..892ff2295 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1022,9 +1022,8 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) +LocalStore::queryPartialDerivationOutputMap(const StorePath & path) { - auto path = path_; auto outputs = retrySQLite>>([&]() { auto state(_state.lock()); std::map> outputs; From 6bc98c7fba9f783414692fcef41d90ed80928b6c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Jul 2023 14:52:35 -0400 Subject: [PATCH 52/78] Give `queryPartialDerivationOutputMap` an `evalStore` parameter This makes it more useful. In general, the derivation will be in one store, and the realisation info is in another. This also helps us avoid duplication. See how `resolveDerivedPath` is now simpler because it uses `queryPartialDerivationOutputMap`. In #8369 we get more flavors of derived path, and need more code to resolve them all, and this problem only gets worse. The fact that we need a new method to deal with the multiple dispatch is unfortunate, but this generally relates to the fact that `Store` is a sub-par interface, too bulky/unwieldy and conflating separate concerns. Solving that is out of scope of this PR. This is part of the RFC 92 work. See tracking issue #6316 --- src/libstore/build/local-derivation-goal.cc | 6 ++- src/libstore/local-store.cc | 19 +------- src/libstore/local-store.hh | 2 +- src/libstore/misc.cc | 53 +++++++++------------ src/libstore/realisation.hh | 9 +++- src/libstore/remote-store.cc | 33 ++++++++----- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 34 +++++++++++-- src/libstore/store-api.hh | 15 +++++- 9 files changed, 103 insertions(+), 70 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2935b9da9..0d982bbff 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1251,11 +1251,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - std::map> queryPartialDerivationOutputMap(const StorePath & path) override + std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); + return next->queryPartialDerivationOutputMap(path, evalStore); } std::optional queryPathFromHashPart(const std::string & hashPart) override diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 892ff2295..010e52019 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1022,9 +1022,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath & path) +LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path) { - auto outputs = retrySQLite>>([&]() { + return retrySQLite>>([&]() { auto state(_state.lock()); std::map> outputs; uint64_t drvId; @@ -1036,21 +1036,6 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path) return outputs; }); - - if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - return outputs; - - auto drv = readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto& [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); - if (realisation) - outputs.insert_or_assign(outputName, realisation->outPath); - else - outputs.insert({outputName, std::nullopt}); - } - - return outputs; } std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8a3b0b43f..d20e9cc4f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -165,7 +165,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryStaticPartialDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 50336c779..14160dc8b 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -310,43 +310,34 @@ std::map drvOutputReferences( OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { - auto & evalStore = evalStore_ ? *evalStore_ : store; + auto outputsOpt_ = store.queryPartialDerivationOutputMap(bfd.drvPath, evalStore_); - OutputPathMap outputs; - auto drv = evalStore.readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(store, drv); - auto drvOutputs = drv.outputsAndOptPaths(store); - auto outputNames = std::visit(overloaded { + auto outputsOpt = std::visit(overloaded { [&](const OutputsSpec::All &) { - StringSet names; - for (auto & [outputName, _] : drv.outputs) - names.insert(outputName); - return names; + // Keep all outputs + return std::move(outputsOpt_); }, [&](const OutputsSpec::Names & names) { - return static_cast>(names); + // Get just those mentioned by name + std::map> outputsOpt; + for (auto & output : names) { + auto * pOutputPathOpt = get(outputsOpt_, output); + if (!pOutputPathOpt) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store.printStorePath(bfd.drvPath), output); + outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); + } + return outputsOpt; }, }, bfd.outputs.raw()); - for (auto & output : outputNames) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - DrvOutput outputId { *outputHash, output }; - auto realisation = store.queryRealisation(outputId); - if (!realisation) - throw MissingRealisation(outputId); - outputs.insert_or_assign(output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); - outputs.insert_or_assign(output, *drvOutput->second); - } + + OutputPathMap outputs; + for (auto & [outputName, outputPathOpt] : outputsOpt) { + if (!outputPathOpt) + throw MissingRealisation(store.printStorePath(bfd.drvPath), outputName); + auto & outputPath = *outputPathOpt; + outputs.insert_or_assign(outputName, outputPath); } return outputs; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 2a093c128..0548b30c1 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" #include #include "comparator.hh" #include "crypto.hh" @@ -143,9 +144,13 @@ class MissingRealisation : public Error { public: MissingRealisation(DrvOutput & outputId) - : Error( "cannot operate on an output of the " + : MissingRealisation(outputId.outputName, outputId.strHash()) + {} + MissingRealisation(std::string_view drv, std::string outputName) + : Error( "cannot operate on output '%s' of the " "unbuilt derivation '%s'", - outputId.to_string()) + outputName, + drv) {} }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1e2104e1f..bfe2258a4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -378,27 +378,36 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) +std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_) { if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { - auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return WorkerProto::Serialise>>::read(*this, *conn); + if (!evalStore_) { + auto conn(getConnection()); + conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return WorkerProto::Serialise>>::read(*this, *conn); + } else { + auto & evalStore = *evalStore_; + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + // union with the first branch overriding the statically-known ones + // when non-`std::nullopt`. + for (auto && [outputName, optPath] : queryPartialDerivationOutputMap(path, nullptr)) { + if (optPath) + outputs.insert_or_assign(std::move(outputName), std::move(optPath)); + else + outputs.insert({std::move(outputName), std::nullopt}); + } + return outputs; + } } else { + auto & evalStore = evalStore_ ? *evalStore_ : *this; // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an // under-approximation as it only returns the paths that can be inferred // from the derivation itself (and not the ones that are known because // the have been built), but as old stores don't handle floating-CA // derivations this shouldn't matter - auto derivation = readDerivation(path); - auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); - std::map> ret; - for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { - ret.emplace(outputName, outputAndPath.second); - } - return ret; + return evalStore.queryStaticPartialDerivationOutputMap(path); } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index cb7a71acf..b12f5437f 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -63,7 +63,7 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5bee1af9f..a581edcc0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -496,22 +496,50 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) +std::map> Store::queryStaticPartialDerivationOutputMap(const StorePath & path) { std::map> outputs; auto drv = readInvalidDerivation(path); - for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) { + for (auto & [outputName, output] : drv.outputsAndOptPaths(*this)) { outputs.emplace(outputName, output.second); } return outputs; } +std::map> Store::queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : *this; + + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + return outputs; + + auto drv = evalStore.readInvalidDerivation(path); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto & [outputName, hash] : drvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); + if (realisation) { + outputs.insert_or_assign(outputName, realisation->outPath); + } else { + // queryStaticPartialDerivationOutputMap is not guaranteed + // to return std::nullopt for outputs which are not + // statically known. + outputs.insert({outputName, std::nullopt}); + } + } + + return outputs; +} + OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path)); + throw MissingRealisation(printStorePath(path), outName); result.insert_or_assign(outName, *optOutPath); } return result; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 14a862eef..7b412d2dd 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -425,7 +425,20 @@ public: * derivation. All outputs are mentioned so ones mising the mapping * are mapped to `std::nullopt`. */ - virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); + virtual std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr); + + /** + * Like `queryPartialDerivationOutputMap` but only considers + * statically known output paths (i.e. those that can be gotten from + * the derivation itself. + * + * Just a helper function for implementing + * `queryPartialDerivationOutputMap`. + */ + virtual std::map> queryStaticPartialDerivationOutputMap( + const StorePath & path); /** * Query the mapping outputName=>outputPath for the given derivation. From 570a1a3ad773d93aa2128a8fba49f98e1e115d5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sat, 8 Jul 2023 12:28:13 +0200 Subject: [PATCH 53/78] parser: merge nested dynamic attributes Fixes https://github.com/NixOS/nix/issues/7115 --- doc/manual/src/release-notes/rl-next.md | 19 +++++++++++++++++++ src/libexpr/parser.y | 1 + .../lang/eval-fail-dup-dynamic-attrs.err.exp | 8 ++++++++ tests/lang/eval-fail-dup-dynamic-attrs.nix | 4 ++++ tests/lang/eval-okay-merge-dynamic-attrs.exp | 1 + tests/lang/eval-okay-merge-dynamic-attrs.nix | 13 +++++++++++++ 6 files changed, 46 insertions(+) create mode 100644 tests/lang/eval-fail-dup-dynamic-attrs.err.exp create mode 100644 tests/lang/eval-fail-dup-dynamic-attrs.nix create mode 100644 tests/lang/eval-okay-merge-dynamic-attrs.exp create mode 100644 tests/lang/eval-okay-merge-dynamic-attrs.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 139d07188..160245a31 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -6,3 +6,22 @@ - Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. + +- Nested dynamic attributes are now merged correctly by the parser. For example: + + ```nix + { + nested = { foo = 1; }; + nested = { ${"ba" + "r"} = 2; }; + } + ``` + + This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: + + ```nix + { nested = { bar = 2; foo = 1; }; } + ``` + + Note that the feature of merging multiple attribute set declarations is of questionable value. + It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. + This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0a1ad9967..217c17382 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -137,6 +137,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, dupAttr(state, ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs.emplace(ad.first, ad.second); } + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); } else { dupAttr(state, attrPath, pos, j->second.pos); } diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/lang/eval-fail-dup-dynamic-attrs.err.exp new file mode 100644 index 000000000..e01f8e6d0 --- /dev/null +++ b/tests/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -0,0 +1,8 @@ +error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: + + 2| set = { "${"" + "b"}" = 1; }; + 3| set = { "${"b" + ""}" = 2; }; + | ^ + 4| } diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.nix b/tests/lang/eval-fail-dup-dynamic-attrs.nix new file mode 100644 index 000000000..7ea17f6c8 --- /dev/null +++ b/tests/lang/eval-fail-dup-dynamic-attrs.nix @@ -0,0 +1,4 @@ +{ + set = { "${"" + "b"}" = 1; }; + set = { "${"b" + ""}" = 2; }; +} diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.exp b/tests/lang/eval-okay-merge-dynamic-attrs.exp new file mode 100644 index 000000000..157d677ce --- /dev/null +++ b/tests/lang/eval-okay-merge-dynamic-attrs.exp @@ -0,0 +1 @@ +{ set1 = { a = 1; b = 2; }; set2 = { a = 1; b = 2; }; set3 = { a = 1; b = 2; }; set4 = { a = 1; b = 2; }; } diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.nix b/tests/lang/eval-okay-merge-dynamic-attrs.nix new file mode 100644 index 000000000..f459a554f --- /dev/null +++ b/tests/lang/eval-okay-merge-dynamic-attrs.nix @@ -0,0 +1,13 @@ +{ + set1 = { a = 1; }; + set1 = { "${"b" + ""}" = 2; }; + + set2 = { "${"b" + ""}" = 2; }; + set2 = { a = 1; }; + + set3.a = 1; + set3."${"b" + ""}" = 2; + + set4."${"b" + ""}" = 2; + set4.a = 1; +} From 0a30b072779aa605b92f1961fe94218ccd183669 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 12:17:35 -0400 Subject: [PATCH 54/78] Move `Store::Params` typedef to `StoreConfig::Params` This is because `StoreConfig` also uses it. --- src/libstore/store-api.hh | 6 ++---- src/libstore/uds-remote-store.hh | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7b412d2dd..3758c730f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -99,6 +99,8 @@ typedef std::map> StorePathCAMap; struct StoreConfig : public Config { + typedef std::map Params; + using Config::Config; StoreConfig() = delete; @@ -153,10 +155,6 @@ struct StoreConfig : public Config class Store : public std::enable_shared_from_this, public virtual StoreConfig { -public: - - typedef std::map Params; - protected: struct PathInfoCacheValue { diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 2bd6517fa..b9d9a39b5 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -9,7 +9,7 @@ namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Store::Params & params) + UDSRemoteStoreConfig(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) From 13269ba93b7453def7084b00eb4a34ad787a7c45 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 12:14:15 -0400 Subject: [PATCH 55/78] Make `RemoteStore::ConnectionHandle` part of class and expose Will need to do subclass-specific implementations in the next commit. This isn't because there will be multiple variations of the daemon protocol (whew!) but because different clients pick and choose different parts to use. --- src/libstore/remote-store-connection.hh | 31 +++++++++++++++ src/libstore/remote-store.cc | 52 +++++++------------------ src/libstore/remote-store.hh | 4 +- 3 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index d32d91a60..ce4740a9c 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -1,5 +1,6 @@ #include "remote-store.hh" #include "worker-protocol.hh" +#include "pool.hh" namespace nix { @@ -94,4 +95,34 @@ struct RemoteStore::Connection std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); }; +/** + * A wrapper around Pool::Handle that marks + * the connection as bad (causing it to be closed) if a non-daemon + * exception is thrown before the handle is closed. Such an exception + * causes a deviation from the expected protocol and therefore a + * desynchronization between the client and daemon. + */ +struct RemoteStore::ConnectionHandle +{ + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle(); + + RemoteStore::Connection & operator * () { return *handle; } + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + + void withFramedSink(std::function fun); +}; + } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bfe2258a4..926ccd9d1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -159,49 +159,25 @@ void RemoteStore::setOptions(Connection & conn) } -/* A wrapper around Pool::Handle that marks - the connection as bad (causing it to be closed) if a non-daemon - exception is thrown before the handle is closed. Such an exception - causes a deviation from the expected protocol and therefore a - desynchronization between the client and daemon. */ -struct ConnectionHandle +RemoteStore::ConnectionHandle::~ConnectionHandle() { - Pool::Handle handle; - bool daemonException = false; - - ConnectionHandle(Pool::Handle && handle) - : handle(std::move(handle)) - { } - - ConnectionHandle(ConnectionHandle && h) - : handle(std::move(h.handle)) - { } - - ~ConnectionHandle() - { - if (!daemonException && std::uncaught_exceptions()) { - handle.markBad(); - debug("closing daemon connection because of an exception"); - } + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); } +} - RemoteStore::Connection * operator -> () { return &*handle; } - RemoteStore::Connection & operator * () { return *handle; } - - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) - { - auto ex = handle->processStderr(sink, source, flush); - if (ex) { - daemonException = true; - std::rethrow_exception(ex); - } +void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush) +{ + auto ex = handle->processStderr(sink, source, flush); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); } - - void withFramedSink(std::function fun); -}; +} -ConnectionHandle RemoteStore::getConnection() +RemoteStore::ConnectionHandle RemoteStore::getConnection() { return ConnectionHandle(connections->get()); } @@ -1099,7 +1075,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -void ConnectionHandle::withFramedSink(std::function fun) +void RemoteStore::ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b12f5437f..68ec29e6c 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -17,7 +17,6 @@ class Pid; struct FdSink; struct FdSource; template class Pool; -struct ConnectionHandle; struct RemoteStoreConfig : virtual StoreConfig { @@ -182,6 +181,8 @@ protected: void setOptions() override; + struct ConnectionHandle; + ConnectionHandle getConnection(); friend struct ConnectionHandle; @@ -199,5 +200,4 @@ private: std::shared_ptr evalStore); }; - } From 60d8dd7aeaf7fc022a1b207012c94180f6732b45 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 Mar 2023 10:06:45 -0400 Subject: [PATCH 56/78] Clean up store hierarchy with `IndirectRootStore` See the API doc comments for details. --- src/libstore/build/local-derivation-goal.cc | 4 +- src/libstore/daemon.cc | 5 ++- src/libstore/gc-store.hh | 35 +++++++++++---- src/libstore/gc.cc | 3 +- src/libstore/indirect-root-store.hh | 48 +++++++++++++++++++++ src/libstore/local-fs-store.hh | 16 ++++++- src/libstore/local-store.hh | 13 ++++-- src/libstore/remote-store.cc | 9 ---- src/libstore/remote-store.hh | 2 - src/libstore/ssh-store.cc | 12 ++++-- src/libstore/uds-remote-store.cc | 10 +++++ src/libstore/uds-remote-store.hh | 16 ++++++- 12 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 src/libstore/indirect-root-store.hh diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 214f3d49e..b7a27490c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,5 +1,5 @@ #include "local-derivation-goal.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" @@ -1200,7 +1200,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore { ref next; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ad3dee1a2..8cbf6f044 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -7,6 +7,7 @@ #include "store-cast.hh" #include "gc-store.hh" #include "log-store.hh" +#include "indirect-root-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" @@ -675,8 +676,8 @@ static void performOp(TunnelLogger * logger, ref store, Path path = absPath(readString(from)); logger->startWork(); - auto & gcStore = require(*store); - gcStore.addIndirectRoot(path); + auto & indirectRootStore = require(*store); + indirectRootStore.addIndirectRoot(path); logger->stopWork(); to << 1; diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 2c26c65c4..ab1059fb1 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -71,19 +71,36 @@ struct GCResults }; +/** + * Mix-in class for \ref Store "stores" which expose a notion of garbage + * collection. + * + * Garbage collection will allow deleting paths which are not + * transitively "rooted". + * + * The notion of GC roots actually not part of this class. + * + * - The base `Store` class has `Store::addTempRoot()` because for a store + * that doesn't support garbage collection at all, a temporary GC root is + * safely implementable as no-op. + * + * @todo actually this is not so good because stores are *views*. + * Some views have only a no-op temp roots even though others to the + * same store allow triggering GC. For instance one can't add a root + * over ssh, but that doesn't prevent someone from gc-ing that store + * accesed via SSH locally). + * + * - The derived `LocalFSStore` class has `LocalFSStore::addPermRoot`, + * which is not part of this class because it relies on the notion of + * an ambient file system. There are stores (`ssh-ng://`, for one), + * that *do* support garbage collection but *don't* expose any file + * system, and `LocalFSStore::addPermRoot` thus does not make sense + * for them. + */ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /** - * Add an indirect root, which is merely a symlink to `path` from - * `/nix/var/nix/gcroots/auto/`. `path` is supposed - * to be a symlink to a store path. The garbage collector will - * automatically remove the indirect root when it finds that - * `path` has disappeared. - */ - virtual void addIndirectRoot(const Path & path) = 0; - /** * Find the roots of the garbage collector. Each root is a pair * `(link, storepath)` where `link` is the path of the symlink diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 20720fb99..26c87391c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,7 +1,6 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" -#include "local-fs-store.hh" #include "finally.hh" #include @@ -50,7 +49,7 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) +Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) { Path gcRoot(canonPath(_gcRoot)); diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh new file mode 100644 index 000000000..59e45af45 --- /dev/null +++ b/src/libstore/indirect-root-store.hh @@ -0,0 +1,48 @@ +#pragma once +///@file + +#include "local-fs-store.hh" + +namespace nix { + +/** + * Mix-in class for implementing permanent roots as a pair of a direct + * (strong) reference and indirect weak reference to the first + * reference. + * + * See methods for details on the operations it represents. + */ +struct IndirectRootStore : public virtual LocalFSStore +{ + inline static std::string operationName = "Indirect GC roots registration"; + + /** + * Implementation of `LocalFSStore::addPermRoot` where the permanent + * root is a pair of + * + * - The user-facing symlink which all implementations must create + * + * - An additional weak reference known as the "indirect root" that + * points to that symlink. + * + * The garbage collector will automatically remove the indirect root + * when it finds that the symlink has disappeared. + * + * The implementation of this method is concrete, but it delegates + * to `addIndirectRoot()` which is abstract. + */ + Path addPermRoot(const StorePath & storePath, const Path & gcRoot) override final; + + /** + * Add an indirect root, which is a weak reference to the + * user-facing symlink created by `addPermRoot()`. + * + * @param path user-facing and user-controlled symlink to a store + * path. + * + * The form this weak-reference takes is implementation-specific. + */ + virtual void addIndirectRoot(const Path & path) = 0; +}; + +} diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 2ee2ef0c8..488109501 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -40,6 +40,7 @@ class LocalFSStore : public virtual LocalFSStoreConfig, public virtual LogStore { public: + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; @@ -49,9 +50,20 @@ public: ref getFSAccessor() override; /** - * Register a permanent GC root. + * Creates symlink from the `gcRoot` to the `storePath` and + * registers the `gcRoot` as a permanent GC root. The `gcRoot` + * symlink lives outside the store and is created and owned by the + * user. + * + * @param gcRoot The location of the symlink. + * + * @param storePath The store object being rooted. The symlink will + * point to `toRealPath(store.printStorePath(storePath))`. + * + * How the permanent GC root corresponding to this symlink is + * managed is implementation-specific. */ - Path addPermRoot(const StorePath & storePath, const Path & gcRoot); + virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 2bdd46fa2..4125cacf4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,8 +5,7 @@ #include "pathlocks.hh" #include "store-api.hh" -#include "local-fs-store.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "sync.hh" #include "util.hh" @@ -68,7 +67,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig std::string doc() override; }; -class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore +class LocalStore : public virtual LocalStoreConfig + , public virtual IndirectRootStore + , public virtual GcStore { private: @@ -209,6 +210,12 @@ private: public: + /** + * Implementation of IndirectRootStore::addIndirectRoot(). + * + * The weak reference merely is a symlink to `path' from + * /nix/var/nix/gcroots/auto/. + */ void addIndirectRoot(const Path & path) override; private: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 926ccd9d1..21258daec 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -822,15 +822,6 @@ void RemoteStore::addTempRoot(const StorePath & path) } -void RemoteStore::addIndirectRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << WorkerProto::Op::AddIndirectRoot << path; - conn.processStderr(); - readInt(conn->from); -} - - Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 68ec29e6c..a1ae82a0f 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -126,8 +126,6 @@ public: void addTempRoot(const StorePath & path) override; - void addIndirectRoot(const Path & path) override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 0200076c0..9c6c42ef4 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,5 +1,6 @@ #include "ssh-store-config.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" #include "remote-fs-accessor.hh" @@ -61,7 +62,7 @@ public: std::optional getBuildLogExact(const StorePath & path) override { unsupported("getBuildLogExact"); } -private: +protected: struct Connection : RemoteStore::Connection { @@ -93,9 +94,12 @@ private: ref SSHStore::openConnection() { auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --stdio", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + + std::string command = remoteProgram + " --stdio"; + if (remoteStore.get() != "") + command += " --store " + shellEscape(remoteStore.get()); + + conn->sshConn = master.startCommand(command); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); return conn; diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 69dae2da5..99589f8b2 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "worker-protocol.hh" #include #include @@ -77,6 +78,15 @@ ref UDSRemoteStore::openConnection() } +void UDSRemoteStore::addIndirectRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index b9d9a39b5..cdb28a001 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -3,7 +3,7 @@ #include "remote-store.hh" #include "remote-store-connection.hh" -#include "local-fs-store.hh" +#include "indirect-root-store.hh" namespace nix { @@ -21,7 +21,9 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon std::string doc() override; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore +class UDSRemoteStore : public virtual UDSRemoteStoreConfig + , public virtual IndirectRootStore + , public virtual RemoteStore { public: @@ -39,6 +41,16 @@ public: void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } + /** + * Implementation of `IndirectRootStore::addIndirectRoot()` which + * delegates to the remote store. + * + * The idea is that the client makes the direct symlink, so it is + * owned managed by the client's user account, and the server makes + * the indirect symlink. + */ + void addIndirectRoot(const Path & path) override; + private: struct Connection : RemoteStore::Connection From c51be0345eda59b93dc1bda09d1c125f3f84d716 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jul 2023 17:19:31 +0200 Subject: [PATCH 57/78] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.17.md | 29 +++++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 26 ---------------------- 3 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.17.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f2e69178f..6c599abcf 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -109,6 +109,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/src/release-notes/rl-2.17.md new file mode 100644 index 000000000..125a93cfd --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.17.md @@ -0,0 +1,29 @@ +# Release 2.17 (2023-07-24) + +* [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand. + +* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. + +* Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. + Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. + +* Nested dynamic attributes are now merged correctly by the parser. For example: + + ```nix + { + nested = { foo = 1; }; + nested = { ${"ba" + "r"} = 2; }; + } + ``` + + This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: + + ```nix + { nested = { bar = 2; foo = 1; }; } + ``` + + Note that the feature of merging multiple attribute set declarations is of questionable value. + It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. + This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. + +* Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 160245a31..c869b5e2f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,27 +1 @@ # Release X.Y (202?-??-??) - -- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand - -* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. - -- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. - Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. - -- Nested dynamic attributes are now merged correctly by the parser. For example: - - ```nix - { - nested = { foo = 1; }; - nested = { ${"ba" + "r"} = 2; }; - } - ``` - - This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: - - ```nix - { nested = { bar = 2; foo = 1; }; } - ``` - - Note that the feature of merging multiple attribute set declarations is of questionable value. - It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. - This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. From 0c275558e75b62beb53e8ada2c116643c7ecb0ba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jul 2023 21:30:33 +0200 Subject: [PATCH 58/78] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index d76bd2ba3..cf8690732 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.17.0 +2.18.0 From 1b756e300f98afe6a583f8bed3acd687d4a8abf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 25 Jul 2023 15:51:15 +0200 Subject: [PATCH 59/78] doc: clarify release notes about nested attribute merges --- doc/manual/src/release-notes/rl-2.17.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/src/release-notes/rl-2.17.md index 125a93cfd..0b861aecc 100644 --- a/doc/manual/src/release-notes/rl-2.17.md +++ b/doc/manual/src/release-notes/rl-2.17.md @@ -11,8 +11,12 @@ ```nix { - nested = { foo = 1; }; - nested = { ${"ba" + "r"} = 2; }; + nested = { + foo = 1; + }; + nested = { + ${"ba" + "r"} = 2; + }; } ``` @@ -22,8 +26,17 @@ { nested = { bar = 2; foo = 1; }; } ``` - Note that the feature of merging multiple attribute set declarations is of questionable value. + Note that the feature of merging multiple *full declarations* of attribute sets like `nested` in the example is of questionable value. It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. + Instead, consider using the *nested attribute path* syntax: + + ```nix + { + nested.foo = 1; + nested.${"ba" + "r"} = 2; + } + ``` + * Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details. From 2d1d81114d72ace89ce08cd3bc93f4eb27a2975d Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Tue, 25 Jul 2023 12:43:33 -0500 Subject: [PATCH 60/78] Add `parseFlakeRef` and `flakeRefToString` builtins (#8670) Over the last year or so I've run into several use cases where I need to parse and/or serialize URLs for use by `builtins.fetchTree` or `builtins.getFlake`, largely in order to produce _lockfile-like_ files for lang2nix frameworks or tools which use `nix` internally to drive builds. I've gone through the painstaking process of emulating `nix::FlakeRef::fromAttrs` and `nix::parseFlakeRef` several times with mixed success; but these are difficult to create and even harder to maintain if I hope to stay aligned with changes to the real parser/serializer. I understand why adding new `builtins` isn't something we want to do flagrantly. I'm recommending this addition simply because I keep encountering use cases where I need to parse/serialize these URIs in `nix` expressions, and I want a reliable solution. Co-authored-by: Eelco Dolstra Co-authored-by: John Ericson --- doc/manual/src/release-notes/rl-next.md | 7 ++ src/libexpr/flake/flake.cc | 95 ++++++++++++++++++++ tests/lang/eval-okay-flake-ref-to-string.exp | 1 + tests/lang/eval-okay-flake-ref-to-string.nix | 7 ++ tests/lang/eval-okay-parse-flake-ref.exp | 1 + tests/lang/eval-okay-parse-flake-ref.nix | 1 + 6 files changed, 112 insertions(+) create mode 100644 tests/lang/eval-okay-flake-ref-to-string.exp create mode 100644 tests/lang/eval-okay-flake-ref-to-string.nix create mode 100644 tests/lang/eval-okay-parse-flake-ref.exp create mode 100644 tests/lang/eval-okay-parse-flake-ref.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2f..065364453 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,8 @@ # Release X.Y (202?-??-??) + +- Two new builtin functions, + [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) + and + [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), + have been added. + These functions are useful for converting between flake references encoded as attribute sets and URLs. diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 5aa44d6a1..9112becff 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -793,6 +793,101 @@ static RegisterPrimOp r2({ .experimentalFeature = Xp::Flakes, }); +static void prim_parseFlakeRef( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, + "while evaluating the argument passed to builtins.parseFlakeRef")); + auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs(); + auto binds = state.buildBindings(attrs.size()); + for (const auto & [key, value] : attrs) { + auto s = state.symbols.create(key); + auto & vv = binds.alloc(s); + std::visit(overloaded { + [&vv](const std::string & value) { vv.mkString(value); }, + [&vv](const uint64_t & value) { vv.mkInt(value); }, + [&vv](const Explicit & value) { vv.mkBool(value.t); } + }, value); + } + v.mkAttrs(binds); +} + +static RegisterPrimOp r3({ + .name = "__parseFlakeRef", + .args = {"flake-ref"}, + .doc = R"( + Parse a flake reference, and return its exploded form. + + For example: + ```nix + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + evaluates to: + ```nix + { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } + ``` + )", + .fun = prim_parseFlakeRef, + .experimentalFeature = Xp::Flakes, +}); + + +static void prim_flakeRefToString( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + state.forceAttrs(*args[0], noPos, + "while evaluating the argument passed to builtins.flakeRefToString"); + fetchers::Attrs attrs; + for (const auto & attr : *args[0]->attrs) { + auto t = attr.value->type(); + if (t == nInt) { + attrs.emplace(state.symbols[attr.name], + (uint64_t) attr.value->integer); + } else if (t == nBool) { + attrs.emplace(state.symbols[attr.name], + Explicit { attr.value->boolean }); + } else if (t == nString) { + attrs.emplace(state.symbols[attr.name], + std::string(attr.value->str())); + } else { + state.error( + "flake reference attribute sets may only contain integers, Booleans, " + "and strings, but attribute '%s' is %s", + state.symbols[attr.name], + showType(*attr.value)).debugThrow(); + } + } + auto flakeRef = FlakeRef::fromAttrs(attrs); + v.mkString(flakeRef.to_string()); +} + +static RegisterPrimOp r4({ + .name = "__flakeRefToString", + .args = {"attrs"}, + .doc = R"( + Convert a flake reference from attribute set format to URL format. + + For example: + ```nix + builtins.flakeRefToString { + dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; + } + ``` + evaluates to + ```nix + "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + )", + .fun = prim_flakeRefToString, + .experimentalFeature = Xp::Flakes, +}); + } Fingerprint LockedFlake::getFingerprint() const diff --git a/tests/lang/eval-okay-flake-ref-to-string.exp b/tests/lang/eval-okay-flake-ref-to-string.exp new file mode 100644 index 000000000..110f8442d --- /dev/null +++ b/tests/lang/eval-okay-flake-ref-to-string.exp @@ -0,0 +1 @@ +"github:NixOS/nixpkgs/23.05?dir=lib" diff --git a/tests/lang/eval-okay-flake-ref-to-string.nix b/tests/lang/eval-okay-flake-ref-to-string.nix new file mode 100644 index 000000000..dbb4e5b2a --- /dev/null +++ b/tests/lang/eval-okay-flake-ref-to-string.nix @@ -0,0 +1,7 @@ +builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + ref = "23.05"; + dir = "lib"; +} diff --git a/tests/lang/eval-okay-parse-flake-ref.exp b/tests/lang/eval-okay-parse-flake-ref.exp new file mode 100644 index 000000000..fc17ba085 --- /dev/null +++ b/tests/lang/eval-okay-parse-flake-ref.exp @@ -0,0 +1 @@ +{ dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } diff --git a/tests/lang/eval-okay-parse-flake-ref.nix b/tests/lang/eval-okay-parse-flake-ref.nix new file mode 100644 index 000000000..db4ed2742 --- /dev/null +++ b/tests/lang/eval-okay-parse-flake-ref.nix @@ -0,0 +1 @@ + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" From 33d58a90c26f9a62bb20863b767c0a505e6f997a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Jun 2023 01:29:11 +0200 Subject: [PATCH 61/78] toJSON: Add attribute path to trace --- doc/manual/src/release-notes/rl-next.md | 2 + src/libexpr/value-to-json.cc | 22 ++++++++-- tests/lang/eval-fail-toJSON.err.exp | 57 +++++++++++++++++++++++++ tests/lang/eval-fail-toJSON.nix | 10 +++++ 4 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 tests/lang/eval-fail-toJSON.err.exp create mode 100644 tests/lang/eval-fail-toJSON.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 065364453..df30ce83d 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -6,3 +6,5 @@ [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), have been added. These functions are useful for converting between flake references encoded as attribute sets and URLs. + +- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4996a5bde..ac3986c87 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -43,6 +43,7 @@ json printValueAsJSON(EvalState & state, bool strict, break; case nNull: + // already initialized as null break; case nAttrs: { @@ -59,7 +60,13 @@ json printValueAsJSON(EvalState & state, bool strict, names.emplace(state.symbols[j.name]); for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); - out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + try { + out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + } catch (Error & e) { + e.addTrace(state.positions[a.pos], + hintfmt("while evaluating attribute '%1%'", j)); + throw; + } } } else return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); @@ -68,8 +75,17 @@ json printValueAsJSON(EvalState & state, bool strict, case nList: { out = json::array(); - for (auto elem : v.listItems()) - out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + int i = 0; + for (auto elem : v.listItems()) { + try { + out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + } catch (Error & e) { + e.addTrace({}, + hintfmt("while evaluating list element at index %1%", i)); + throw; + } + i++; + } break; } diff --git a/tests/lang/eval-fail-toJSON.err.exp b/tests/lang/eval-fail-toJSON.err.exp new file mode 100644 index 000000000..4e618c203 --- /dev/null +++ b/tests/lang/eval-fail-toJSON.err.exp @@ -0,0 +1,57 @@ +error: + … while calling the 'toJSON' builtin + + at /pwd/lang/eval-fail-toJSON.nix:1:1: + + 1| builtins.toJSON { + | ^ + 2| a.b = [ + + … while evaluating attribute 'a' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating attribute 'b' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating list element at index 3 + + … while evaluating attribute 'c' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while evaluating attribute 'd' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while calling the 'throw' builtin + + at /pwd/lang/eval-fail-toJSON.nix:7:13: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + error: hah no diff --git a/tests/lang/eval-fail-toJSON.nix b/tests/lang/eval-fail-toJSON.nix new file mode 100644 index 000000000..8112e1c1f --- /dev/null +++ b/tests/lang/eval-fail-toJSON.nix @@ -0,0 +1,10 @@ +builtins.toJSON { + a.b = [ + true + false + "it's a bird" + { + c.d = throw "hah no"; + } + ]; +} From 1570e80219df92461ede2a672f8997a013364f5c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 31 Jul 2023 09:19:19 -0400 Subject: [PATCH 62/78] Move evaluator settings (type and global) to separate file/header --- src/libcmd/common-eval-args.cc | 1 + src/libcmd/installables.cc | 1 + src/libcmd/repl.cc | 1 + src/libexpr/eval-settings.cc | 95 ++++++++++++++++++++++++++ src/libexpr/eval-settings.hh | 98 +++++++++++++++++++++++++++ src/libexpr/eval.cc | 89 +----------------------- src/libexpr/eval.hh | 92 ------------------------- src/libexpr/flake/flake.cc | 1 + src/libexpr/parser.y | 1 + src/libexpr/primops.cc | 1 + src/libexpr/primops/fetchMercurial.cc | 1 + src/libexpr/primops/fetchTree.cc | 1 + src/nix/flake.cc | 1 + src/nix/main.cc | 1 + src/nix/repl.cc | 1 + src/nix/search.cc | 1 + src/nix/upgrade-nix.cc | 1 + 17 files changed, 207 insertions(+), 180 deletions(-) create mode 100644 src/libexpr/eval-settings.cc create mode 100644 src/libexpr/eval-settings.hh diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 3df2c71a5..e36bda52f 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,3 +1,4 @@ +#include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 10b077fb5..9d593a01f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -11,6 +11,7 @@ #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" +#include "eval-settings.hh" #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index f9e9c2bf8..d15162e76 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -26,6 +26,7 @@ extern "C" { #include "eval.hh" #include "eval-cache.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "attr-path.hh" #include "store-api.hh" #include "log-store.hh" diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc new file mode 100644 index 000000000..422aaf8d5 --- /dev/null +++ b/src/libexpr/eval-settings.cc @@ -0,0 +1,95 @@ +#include "globals.hh" +#include "profiles.hh" +#include "eval.hh" +#include "eval-settings.hh" + +namespace nix { + +/* Very hacky way to parse $NIX_PATH, which is colon-separated, but + can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ +static Strings parseNixPath(const std::string & s) +{ + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + auto prefix = std::string(start2, s.end()); + if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + + return res; +} + +EvalSettings::EvalSettings() +{ + auto var = getEnv("NIX_PATH"); + if (var) nixPath = parseNixPath(*var); +} + +Strings EvalSettings::getDefaultNixPath() +{ + Strings res; + auto add = [&](const Path & p, const std::string & s = std::string()) { + if (pathAccessible(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } + } + }; + + if (!evalSettings.restrictEval && !evalSettings.pureEval) { + add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); + } + + return res; +} + +bool EvalSettings::isPseudoUrl(std::string_view s) +{ + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == std::string::npos) return false; + std::string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +} + +std::string EvalSettings::resolvePseudoUrl(std::string_view url) +{ + if (hasPrefix(url, "channel:")) + return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + else + return std::string(url); +} + +EvalSettings evalSettings; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + +} diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh new file mode 100644 index 000000000..043af6cab --- /dev/null +++ b/src/libexpr/eval-settings.hh @@ -0,0 +1,98 @@ +#pragma once +#include "config.hh" + +namespace nix { + +struct EvalSettings : Config +{ + EvalSettings(); + + static Strings getDefaultNixPath(); + + static bool isPseudoUrl(std::string_view s); + + static std::string resolvePseudoUrl(std::string_view url); + + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting nixPath{ + this, getDefaultNixPath(), "nix-path", + R"( + List of directories to be searched for `<...>` file references + + In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). + )"}; + + Setting restrictEval{ + this, false, "restrict-eval", + R"( + If set to `true`, the Nix evaluator will not allow access to any + files outside of the Nix search path (as set via the `NIX_PATH` + environment variable or the `-I` option), or to URIs outside of + [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). + The default is `false`. + )"}; + + Setting pureEval{this, false, "pure-eval", + R"( + Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: + + - Restrict file system and network access to files specified by cryptographic hash + - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + )" + }; + + Setting enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + R"( + By default, Nix allows you to `import` from a derivation, allowing + building at evaluation time. With this option set to false, Nix will + throw an error when evaluating an expression that uses this feature, + allowing users to ensure their evaluation will not require any + builds to take place. + )"}; + + Setting allowedUris{this, {}, "allowed-uris", + R"( + A list of URI prefixes to which access is allowed in restricted + evaluation mode. For example, when set to + `https://github.com/NixOS`, builtin functions such as `fetchGit` are + allowed to access `https://github.com/NixOS/patchelf.git`. + )"}; + + Setting traceFunctionCalls{this, false, "trace-function-calls", + R"( + If set to `true`, the Nix evaluator will trace every function call. + Nix will print a log message at the "vomit" level for every function + entrance and function exit. + + function-trace entered undefined position at 1565795816999559622 + function-trace exited undefined position at 1565795816999581277 + function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 + function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + The `undefined position` means the function call is a builtin. + + Use the `contrib/stack-collapse.py` script distributed with the Nix + source code to convert the trace logs in to a format suitable for + `flamegraph.pl`. + )"}; + + Setting useEvalCache{this, true, "eval-cache", + "Whether to use the flake evaluation cache."}; + + Setting ignoreExceptionsDuringTry{this, false, "ignore-try", + R"( + If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in + debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. + )"}; + + Setting traceVerbose{this, false, "trace-verbose", + "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; +}; + +extern EvalSettings evalSettings; + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index be1bdb806..e57de6c1d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,4 +1,5 @@ #include "eval.hh" +#include "eval-settings.hh" #include "hash.hh" #include "types.hh" #include "util.hh" @@ -420,44 +421,6 @@ void initGC() } -/* Very hacky way to parse $NIX_PATH, which is colon-separated, but - can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) -{ - Strings res; - - auto p = s.begin(); - - while (p != s.end()) { - auto start = p; - auto start2 = p; - - while (p != s.end() && *p != ':') { - if (*p == '=') start2 = p + 1; - ++p; - } - - if (p == s.end()) { - if (p != start) res.push_back(std::string(start, p)); - break; - } - - if (*p == ':') { - auto prefix = std::string(start2, s.end()); - if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { - ++p; - while (p != s.end() && *p != ':') ++p; - } - res.push_back(std::string(start, p)); - if (p == s.end()) break; - } - - ++p; - } - - return res; -} - ErrorBuilder & ErrorBuilder::atPos(PosIdx pos) { info.errPos = state.positions[pos]; @@ -2626,54 +2589,4 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } -EvalSettings::EvalSettings() -{ - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); -} - -Strings EvalSettings::getDefaultNixPath() -{ - Strings res; - auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathAccessible(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } - } - }; - - if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); - add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); - add(rootChannelsDir()); - } - - return res; -} - -bool EvalSettings::isPseudoUrl(std::string_view s) -{ - if (s.compare(0, 8, "channel:") == 0) return true; - size_t pos = s.find("://"); - if (pos == std::string::npos) return false; - std::string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; -} - -std::string EvalSettings::resolvePseudoUrl(std::string_view url) -{ - if (hasPrefix(url, "channel:")) - return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; - else - return std::string(url); -} - -EvalSettings evalSettings; - -static GlobalConfig::Register rEvalSettings(&evalSettings); - - } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 46fa96d05..887b9cb97 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -787,98 +787,6 @@ struct InvalidPathError : EvalError #endif }; -struct EvalSettings : Config -{ - EvalSettings(); - - static Strings getDefaultNixPath(); - - static bool isPseudoUrl(std::string_view s); - - static std::string resolvePseudoUrl(std::string_view url); - - Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", - "Whether builtin functions that allow executing native code should be enabled."}; - - Setting nixPath{ - this, getDefaultNixPath(), "nix-path", - R"( - List of directories to be searched for `<...>` file references - - In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). - )"}; - - Setting restrictEval{ - this, false, "restrict-eval", - R"( - If set to `true`, the Nix evaluator will not allow access to any - files outside of the Nix search path (as set via the `NIX_PATH` - environment variable or the `-I` option), or to URIs outside of - [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). - The default is `false`. - )"}; - - Setting pureEval{this, false, "pure-eval", - R"( - Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: - - - Restrict file system and network access to files specified by cryptographic hash - - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) - )" - }; - - Setting enableImportFromDerivation{ - this, true, "allow-import-from-derivation", - R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. - )"}; - - Setting allowedUris{this, {}, "allowed-uris", - R"( - A list of URI prefixes to which access is allowed in restricted - evaluation mode. For example, when set to - `https://github.com/NixOS`, builtin functions such as `fetchGit` are - allowed to access `https://github.com/NixOS/patchelf.git`. - )"}; - - Setting traceFunctionCalls{this, false, "trace-function-calls", - R"( - If set to `true`, the Nix evaluator will trace every function call. - Nix will print a log message at the "vomit" level for every function - entrance and function exit. - - function-trace entered undefined position at 1565795816999559622 - function-trace exited undefined position at 1565795816999581277 - function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 - function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 - - The `undefined position` means the function call is a builtin. - - Use the `contrib/stack-collapse.py` script distributed with the Nix - source code to convert the trace logs in to a format suitable for - `flamegraph.pl`. - )"}; - - Setting useEvalCache{this, true, "eval-cache", - "Whether to use the flake evaluation cache."}; - - Setting ignoreExceptionsDuringTry{this, false, "ignore-try", - R"( - If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in - debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. - )"}; - - Setting traceVerbose{this, false, "trace-verbose", - "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; -}; - -extern EvalSettings evalSettings; - static const std::string corepkgsPrefix{"/__corepkgs__/"}; template diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 9112becff..6a27ea2e8 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -1,5 +1,6 @@ #include "flake.hh" #include "eval.hh" +#include "eval-settings.hh" #include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 217c17382..201370b90 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -22,6 +22,7 @@ #include "nixexpr.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" namespace nix { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7ff17b6ee..ddf529b9e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3,6 +3,7 @@ #include "downstream-placeholder.hh" #include "eval-inline.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "json-to-value.hh" #include "names.hh" diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 322692b52..b9ff01c16 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "url.hh" diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 5e668c629..f040a3510 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "filetransfer.hh" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b5f5d0cac..3ce1de44a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -4,6 +4,7 @@ #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "flake/flake.hh" #include "get-drvs.hh" #include "store-api.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 650c79d14..df66beb8c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -3,6 +3,7 @@ #include "command.hh" #include "common-args.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "legacy.hh" #include "shared.hh" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index bb14f3f99..9677c1b48 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,4 +1,5 @@ #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "command.hh" #include "installable-value.hh" diff --git a/src/nix/search.cc b/src/nix/search.cc index c92ed1663..ef0139e09 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "names.hh" #include "get-drvs.hh" #include "common-args.hh" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index d05c23fb7..d238456db 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "filetransfer.hh" #include "eval.hh" +#include "eval-settings.hh" #include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" From c9a87ce7ca50b0a14c4c77fd2b6ca41a74ba6cb9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Fri, 28 Jul 2023 10:11:45 +0100 Subject: [PATCH 63/78] Refactor verifyPath to take StorePath instead of Path. This way we avoid having to convert from Path to StorePath and vice versa in the body of verifyPath. --- src/libstore/local-store.cc | 21 ++++++++------------- src/libstore/local-store.hh | 4 ++-- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f8e0a0ca2..f78bd44ca 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1506,10 +1506,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking path existence..."); StorePathSet validPaths; - PathSet done; + StorePathSet done; for (auto & i : queryAllValidPaths()) - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, store, done, validPaths, repair, errors); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1595,19 +1595,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & pathS, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) +void LocalStore::verifyPath(const StorePath & path, const StringSet & store, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(pathS).second) return; - - if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); - return; - } - - auto path = parseStorePath(pathS); + if (!done.insert(path).second) return; if (!store.count(std::string(path.to_string()))) { /* Check any referrers first. If we can invalidate them @@ -1616,11 +1609,13 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, store, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } + auto pathS = printStorePath(path); + if (canInvalidate) { printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 4125cacf4..c9b570eaa 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -314,8 +314,8 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + void verifyPath(const StorePath & path, const StringSet & store, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); From 2a5f5fbb1776f5086646407e5296c372321863b9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 31 Jul 2023 11:13:38 -0400 Subject: [PATCH 64/78] `LocalStore::verifyPath`: Use `StorePathSet` for `store` local var We don't care about non-store-paths in there (things like `.links`, are, in fact, allowed). So let's just skip them up front and be more strongly typed. --- src/libstore/local-store.cc | 12 ++++++++---- src/libstore/local-store.hh | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f78bd44ca..9049f33aa 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1499,8 +1499,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StringSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); + StorePathSet store; + for (auto & i : readDirectory(realStoreDir)) { + try { + store.insert({i.name}); + } catch (BadStorePath &) { } + } /* Check whether all valid paths actually exist. */ printInfo("checking path existence..."); @@ -1595,14 +1599,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const StorePath & path, const StringSet & store, +void LocalStore::verifyPath(const StorePath & path, const StorePathSet & store, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); if (!done.insert(path).second) return; - if (!store.count(std::string(path.to_string()))) { + if (!store.count(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c9b570eaa..e97195f5b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -314,7 +314,7 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const StorePath & path, const StringSet & store, + void verifyPath(const StorePath & path, const StorePathSet & store, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); From 6525265f4640221efb0039ddbd6849a3b04babc9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 31 Jul 2023 12:22:06 -0400 Subject: [PATCH 65/78] `LocalStore::verifyPath`: Try to clarify data flow with more scopes It was initially unclear to me which of these are temporary state for the verify paths computation, and which of these are the results of that computation to be used in the rest of the function. Now, it is clear, and enforced. --- src/libstore/local-store.cc | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9049f33aa..17e2ebc38 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1499,21 +1499,24 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StorePathSet store; - for (auto & i : readDirectory(realStoreDir)) { - try { - store.insert({i.name}); - } catch (BadStorePath &) { } - } - - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); - StorePathSet validPaths; - StorePathSet done; - for (auto & i : queryAllValidPaths()) - verifyPath(i, store, done, validPaths, repair, errors); + { + StorePathSet store; + for (auto & i : readDirectory(realStoreDir)) { + try { + store.insert({i.name}); + } catch (BadStorePath &) { } + } + + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); + + StorePathSet done; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, store, done, validPaths, repair, errors); + } /* Optionally, check the content hashes (slow). */ if (checkContents) { From b9615419688353f4133520dc11456f7653e4cd08 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 31 Jul 2023 22:51:06 +0200 Subject: [PATCH 66/78] labeler: Stop removing labels > Whether or not to remove labels when matching files are reverted or no longer changed by the PR https://github.com/actions/labeler#inputs --- .github/workflows/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 5f949ddc5..d83cb4f18 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -21,4 +21,4 @@ jobs: - uses: actions/labeler@v4 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - sync-labels: true + sync-labels: false From d9e7758f4756c10592407b3fe99dc253ef56eac9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Aug 2023 16:07:20 +0200 Subject: [PATCH 67/78] Don't require .tar/.zip extension for tarball flakerefs Special-casing the file name is rather ugly, so we shouldn't do that. So now any {file,http,https} URL is handled by TarballInputScheme, except for non-flake inputs (i.e. inputs that have the attribute `flake = false`). --- src/libexpr/flake/flakeref.cc | 6 +++--- src/libfetchers/fetchers.cc | 8 ++++---- src/libfetchers/fetchers.hh | 6 +++--- src/libfetchers/git.cc | 2 +- src/libfetchers/github.cc | 2 +- src/libfetchers/indirect.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/path.cc | 2 +- src/libfetchers/tarball.cc | 18 +++++++++--------- tests/fetchTree-file.sh | 14 +++++++++++++- 10 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 08adbe0c9..d3fa1d557 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -105,7 +105,7 @@ std::pair parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), ""), + FlakeRef(Input::fromURL(parsedURL, isFlake), ""), percentDecode(match.str(6))); } @@ -176,7 +176,7 @@ std::pair parseFlakeRefWithFragment( parsedURL.query.insert_or_assign("shallow", "1"); return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")), fragment); } @@ -204,7 +204,7 @@ std::pair parseFlakeRefWithFragment( std::string fragment; std::swap(fragment, parsedURL.fragment); - auto input = Input::fromURL(parsedURL); + auto input = Input::fromURL(parsedURL, isFlake); input.parent = baseDir; return std::make_pair( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index f86c0604e..e683b9f80 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -13,9 +13,9 @@ void registerInputScheme(std::shared_ptr && inputScheme) inputSchemes->push_back(std::move(inputScheme)); } -Input Input::fromURL(const std::string & url) +Input Input::fromURL(const std::string & url, bool requireTree) { - return fromURL(parseURL(url)); + return fromURL(parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -31,10 +31,10 @@ static void fixupInput(Input & input) input.locked = true; } -Input Input::fromURL(const ParsedURL & url) +Input Input::fromURL(const ParsedURL & url, bool requireTree) { for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromURL(url); + auto res = inputScheme->inputFromURL(url, requireTree); if (res) { res->scheme = inputScheme; fixupInput(*res); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index d0738f619..6e10e9513 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -44,9 +44,9 @@ struct Input std::optional parent; public: - static Input fromURL(const std::string & url); + static Input fromURL(const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url); + static Input fromURL(const ParsedURL & url, bool requireTree = true); static Input fromAttrs(Attrs && attrs); @@ -129,7 +129,7 @@ struct InputScheme virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url) const = 0; + virtual std::optional inputFromURL(const ParsedURL & url, bool requireTree) const = 0; virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index be5842d53..f8d89ab2f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -256,7 +256,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 80598e7f8..291f457f0 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -30,7 +30,7 @@ struct GitArchiveInputScheme : InputScheme virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != type()) return {}; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b99504a16..4874a43ff 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -7,7 +7,7 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 86e8f81f4..51fd1ed42 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -43,7 +43,7 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 61541e69d..01f1be978 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -6,7 +6,7 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index a012234e0..107d38e92 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -194,11 +194,11 @@ struct CurlInputScheme : InputScheme || hasSuffix(path, ".tar.zst"); } - virtual bool isValidURL(const ParsedURL & url) const = 0; + virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0; - std::optional inputFromURL(const ParsedURL & _url) const override + std::optional inputFromURL(const ParsedURL & _url, bool requireTree) const override { - if (!isValidURL(_url)) + if (!isValidURL(_url, requireTree)) return std::nullopt; Input input; @@ -265,13 +265,13 @@ struct FileInputScheme : CurlInputScheme { const std::string inputType() const override { return "file"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : !hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == inputType() + : (!requireTree && !hasTarballExtension(url.path))); } std::pair fetch(ref store, const Input & input) override @@ -285,14 +285,14 @@ struct TarballInputScheme : CurlInputScheme { const std::string inputType() const override { return "tarball"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == inputType() + : (requireTree || hasTarballExtension(url.path))); } std::pair fetch(ref store, const Input & _input) override diff --git a/tests/fetchTree-file.sh b/tests/fetchTree-file.sh index fe569cfb8..6395c133d 100644 --- a/tests/fetchTree-file.sh +++ b/tests/fetchTree-file.sh @@ -27,6 +27,7 @@ test_file_flake_input () { mkdir inputs echo foo > inputs/test_input_file + echo '{ outputs = { self }: { }; }' > inputs/flake.nix tar cfa test_input.tar.gz inputs cp test_input.tar.gz test_input_no_ext input_tarball_hash="$(nix hash path test_input.tar.gz)" @@ -50,6 +51,9 @@ test_file_flake_input () { url = "file+file://$PWD/test_input.tar.gz"; flake = false; }; + inputs.flake_no_ext = { + url = "file://$PWD/test_input_no_ext"; + }; outputs = { ... }: {}; } EOF @@ -58,7 +62,7 @@ EOF nix eval --file - < Date: Wed, 2 Aug 2023 12:40:04 -0400 Subject: [PATCH 68/78] local-store verifying: Rename `store` to something more clear It is not a `Store` but a `StorePathSet`. --- src/libstore/local-store.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17e2ebc38..982a9059c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1502,10 +1502,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) StorePathSet validPaths; { - StorePathSet store; + StorePathSet storePathsInStoreDir; for (auto & i : readDirectory(realStoreDir)) { try { - store.insert({i.name}); + storePathsInStoreDir.insert({i.name}); } catch (BadStorePath &) { } } @@ -1515,7 +1515,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) StorePathSet done; for (auto & i : queryAllValidPaths()) - verifyPath(i, store, done, validPaths, repair, errors); + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); } /* Optionally, check the content hashes (slow). */ @@ -1602,21 +1602,21 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const StorePath & path, const StorePathSet & store, +void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePathsInStoreDir, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); if (!done.insert(path).second) return; - if (!store.count(path)) { + if (!storePathsInStoreDir.count(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(i, store, done, validPaths, repair, errors); + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } From 66550878dffe2a4bf55ea7ab694738aa14e6a01e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 2 Aug 2023 12:45:55 -0400 Subject: [PATCH 69/78] Add comment explaining the use of `readDirectory(realStoreDir)` --- src/libstore/local-store.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 982a9059c..40a3bc194 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1503,6 +1503,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { StorePathSet storePathsInStoreDir; + /* Why aren't we using `queryAllValidPaths`? Because that would + tell us about all the paths than the database knows about. Here we + want to know about all the store paths in the store directory, + regardless of what the database thinks. + + We will end up cross-referencing these two sources of truth (the + database and the filesystem) in the loop below, in order to catch + invalid states. + */ for (auto & i : readDirectory(realStoreDir)) { try { storePathsInStoreDir.insert({i.name}); From 9b908fa70a07219a110ddd63ec3593c2c2269918 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jul 2023 14:02:05 -0400 Subject: [PATCH 70/78] Factor out `nix-defexpr` path computation Avoid duplicated code, and also avoid "on the fly" path construction (which makes it harder to keep track of which paths we use). The factored out code doesn't create the Nix state dir anymore, but this is fine because other in nix-env and nix-channel does: - nix-channel: Line 158 in this commit - nix-env: Line 1407 in this commit --- src/libexpr/eval-settings.cc | 9 ++++++++- src/libexpr/eval-settings.hh | 5 +++++ src/nix-channel/nix-channel.cc | 3 ++- src/nix-env/nix-env.cc | 3 ++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 422aaf8d5..93b4a5289 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -63,7 +63,7 @@ Strings EvalSettings::getDefaultNixPath() }; if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); + add(getNixDefExpr() + "/channels"); add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); add(rootChannelsDir()); } @@ -92,4 +92,11 @@ EvalSettings evalSettings; static GlobalConfig::Register rEvalSettings(&evalSettings); +Path getNixDefExpr() +{ + return settings.useXDGBaseDirectories + ? getStateDir() + "/nix/defexpr" + : getHome() + "/.nix-defexpr"; +} + } diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 043af6cab..e6666061a 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -95,4 +95,9 @@ struct EvalSettings : Config extern EvalSettings evalSettings; +/** + * Conventionally part of the default nix path in impure mode. + */ +Path getNixDefExpr(); + } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index c1c8edd1d..95f401441 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "legacy.hh" #include "fetchers.hh" +#include "eval-settings.hh" // for defexpr #include "util.hh" #include @@ -165,7 +166,7 @@ static int main_nix_channel(int argc, char ** argv) // Figure out the name of the `.nix-channels' file to use auto home = getHome(); channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; - nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr"; + nixDefExpr = getNixDefExpr(); // Figure out the name of the channels profile. profile = profilesDir() + "/channels"; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 91b073b49..3dc3db0fc 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -15,6 +15,7 @@ #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" +#include "eval-settings.hh" // for defexpr #include #include @@ -1399,7 +1400,7 @@ static int main_nix_env(int argc, char * * argv) globals.instSource.type = srcUnknown; globals.instSource.systemFilter = "*"; - Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; + Path nixExprPath = getNixDefExpr(); if (!pathExists(nixExprPath)) { try { From 3b592c880ae76707cc832e5f227e261be29661bf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 11:58:47 -0400 Subject: [PATCH 71/78] Add infra for experimental store implemenations This is analogous to that for experimental settings and flags that we have also added as of late. --- doc/manual/generate-manpage.nix | 21 +++++++++++++++++++-- src/libstore/store-api.cc | 1 + src/libstore/store-api.hh | 15 +++++++++++++++ src/nix/main.cc | 6 ++++-- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index fb34898f3..65eec42d0 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -137,12 +137,29 @@ let storeDocs = let - showStore = name: { settings, doc }: - '' + showStore = name: { settings, doc, experimentalFeature }: + let + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To use this store, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ``` + ''; + in '' ## ${name} ${doc} + ${experimentalFeatureNote} + **Settings**: ${showSettings { useAnchors = false; } settings} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4ea16a1c0..28689e100 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1496,6 +1496,7 @@ ref openStore(const std::string & uri_, if (implem.uriSchemes.count(parsedUri.scheme)) { auto store = implem.create(parsedUri.scheme, baseURI, params); if (store) { + experimentalFeatureSettings.require(store->experimentalFeature()); store->init(); store->warnUnknownSettings(); return ref(store); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3758c730f..f9029ade1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -109,13 +109,28 @@ struct StoreConfig : public Config virtual ~StoreConfig() { } + /** + * The name of this type of store. + */ virtual const std::string name() = 0; + /** + * Documentation for this type of store. + */ virtual std::string doc() { return ""; } + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ + virtual std::optional experimentalFeature() const + { + return std::nullopt; + } + const PathSetting storeDir_{this, settings.nixStore, "store", R"( diff --git a/src/nix/main.cc b/src/nix/main.cc index df66beb8c..c5a9c8b33 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -180,8 +180,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs for (auto & implem : *Implementations::registered) { auto storeConfig = implem.getConfig(); auto storeName = storeConfig->name(); - stores[storeName]["doc"] = storeConfig->doc(); - stores[storeName]["settings"] = storeConfig->toJSON(); + auto & j = stores[storeName]; + j["doc"] = storeConfig->doc(); + j["settings"] = storeConfig->toJSON(); + j["experimentalFeature"] = storeConfig->experimentalFeature(); } res["stores"] = std::move(stores); From 7c09104a943978a165885e10697b418f8bab2795 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 4 Aug 2023 23:11:08 +0200 Subject: [PATCH 72/78] nix/why-depends: fix output of `--precise` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I haven't checked when this was exactly introduced, but on Nix 2.16 I realized that the additional lines inserted when using `--precise` are completely separated from the tree: nix why-depends /nix/store/ccgr4faaxys39s091qridxg1947lggh4-evcxr-0.14.2 /nix/store/b7hvml0m3qmqraz1022fwvyyg6fc1vdy-gcc-12.2.0 --precise --extra-experimental-features nix-command /nix/store/ccgr4faaxys39s091qridxg1947lggh4-evcxr-0.14.2 → /nix/store/lcf37pgp3rgww67v9x2990hbfwx96c1w-gcc-wrapper-12.2.0 → /nix/store/b7hvml0m3qmqraz1022fwvyyg6fc1vdy-gcc-12.2.0 └───bin/evcxr: …':'}.PATH=${PATH/':''/nix/store/lcf37pgp3rgww67v9x2990hbfwx96c1w-gcc-wrapper-12.2.0/bin'':'/':'}… └───bin/cpp: …k disable=SC2193.[[ "/nix/store/b7hvml0m3qmqraz1022fwvyyg6fc1vdy-gcc-12.2.0/bin/cpp" = *++ ]] &&… This is apparently because `std::cout` is buffered and flushed in the end whereas the rest of the output isn't. The fix is rather simple, just use `logger->cout` as it's already the case for the rest of the code. This way we also don't need to insert additional newlines in the `hits` map since that's something the logger takes care of. Also added a small test to make sure that the layout of this is somehow tested to reduce the risk of further regressions here. --- src/nix/why-depends.cc | 10 +++++----- tests/why-depends.sh | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index a3a9dc698..592de773c 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -239,7 +239,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (pos != std::string::npos) { size_t margin = 32; auto pos2 = pos >= margin ? pos - margin : 0; - hits[hash].emplace_back(fmt("%s: …%s…\n", + hits[hash].emplace_back(fmt("%s: …%s…", p2, hilite(filterPrintable( std::string(contents, pos2, pos - pos2 + hash.size() + margin)), @@ -255,7 +255,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & hash : hashes) { auto pos = target.find(hash); if (pos != std::string::npos) - hits[hash].emplace_back(fmt("%s -> %s\n", p2, + hits[hash].emplace_back(fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } @@ -272,9 +272,9 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & hit : hits[hash]) { bool first = hit == *hits[hash].begin(); - std::cout << tailPad - << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)) - << hit; + logger->cout("%s%s%s", tailPad, + (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)), + hit); if (!all) break; } diff --git a/tests/why-depends.sh b/tests/why-depends.sh index b35a0d1cf..9680bf80e 100644 --- a/tests/why-depends.sh +++ b/tests/why-depends.sh @@ -22,3 +22,8 @@ echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet input-2 # But only the “precise” one should refer to `reference-to-input-2` echo "$FAST_WHY_DEPENDS_OUTPUT" | grepQuietInverse reference-to-input-2 echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet reference-to-input-2 + +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '2p' | grepQuiet "└───reference-to-input-2 -> " +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '3p' | grep " →" | grepQuiet "dependencies-input-2" +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '4p' | grepQuiet " └───input0: …" # in input-2, file input0 +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '5p' | grep " →" | grepQuiet "dependencies-input-0" # is dependencies-input-0 referenced From 3fefc2b28494468c7a6078430eaaf8a6c8f17230 Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Fri, 28 Jul 2023 22:23:56 +0200 Subject: [PATCH 73/78] Fix derivation load assertion errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When loading a derivation from a JSON, malformed input would trigger cryptic "assertion failed" errors. Simply replacing calls to `operator []` with calls to `.at()` was not enough, as this would cause json.execptions to be printed verbatim. Display nice error messages instead and give some indication where the error happened. *Before:* ``` $ echo 4 | nix derivation add error: [json.exception.type_error.305] cannot use operator[] with a string argument with number $ nix derivation show nixpkgs#hello | nix derivation add Assertion failed: (it != m_value.object->end()), function operator[], file /nix/store/8h9pxgq1776ns6qi5arx08ifgnhmgl22-nlohmann_json-3.11.2/include/nlohmann/json.hpp, line 2135. $ nix derivation show nixpkgs#hello | jq '.[] | .name = 5' | nix derivation add error: [json.exception.type_error.302] type must be string, but is object $ nix derivation show nixpkgs#hello | jq '.[] | .outputs = { out: "/nix/store/8j3f8j-hello" }' | nix derivation add error: [json.exception.type_error.302] type must be object, but is string ``` *After:* ``` $ echo 4 | nix derivation add error: Expected JSON of derivation to be of type 'object', but it is of type 'number' $ nix derivation show nixpkgs#hello | nix derivation add error: Expected JSON object to contain key 'name' but it doesn't $ nix derivation show nixpkgs#hello | jq '.[] | .name = 5' | nix derivation add error: Expected JSON value to be of type 'string' but it is of type 'number' $ nix derivation show nixpkgs#hello | jq '.[] | .outputs = { out: "/nix/store/8j3f8j-hello" }' | nix derivation add error: … while reading key 'outputs' error: Expected JSON value to be of type 'object' but it is of type 'string' ``` --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libstore/derivations.cc | 40 +++++++++++++++++-------- src/libutil/json-utils.cc | 24 +++++++++++++++ src/libutil/json-utils.hh | 22 ++++++++++++++ 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index df30ce83d..526b64fde 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,3 +8,5 @@ These functions are useful for converting between flake references encoded as attribute sets and URLs. - [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. + +- Error messages regarding malformed input to [`derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b831b2fe5..8a84bb1c5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -993,6 +993,7 @@ DerivationOutput DerivationOutput::fromJSON( const ExperimentalFeatureSettings & xpSettings) { std::set keys; + ensureType(_json, nlohmann::detail::value_t::object); auto json = (std::map) _json; for (const auto & [key, _] : json) @@ -1097,36 +1098,51 @@ Derivation Derivation::fromJSON( const Store & store, const nlohmann::json & json) { + using nlohmann::detail::value_t; + Derivation res; - res.name = json["name"]; + ensureType(json, value_t::object); - { - auto & outputsObj = json["outputs"]; + res.name = ensureType(valueAt(json, "name"), value_t::string); + + try { + auto & outputsObj = ensureType(valueAt(json, "outputs"), value_t::object); for (auto & [outputName, output] : outputsObj.items()) { res.outputs.insert_or_assign( outputName, DerivationOutput::fromJSON(store, res.name, outputName, output)); } + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; } - { - auto & inputsList = json["inputSrcs"]; + try { + auto & inputsList = ensureType(valueAt(json, "inputSrcs"), value_t::array); for (auto & input : inputsList) res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputSrcs'"); + throw; } - { - auto & inputDrvsObj = json["inputDrvs"]; - for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + try { + auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object); + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) { + ensureType(inputOutputs, value_t::array); res.inputDrvs[store.parseStorePath(inputDrvPath)] = static_cast(inputOutputs); + } + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputDrvs'"); + throw; } - res.platform = json["system"]; - res.builder = json["builder"]; - res.args = json["args"]; - res.env = json["env"]; + res.platform = ensureType(valueAt(json, "system"), value_t::string); + res.builder = ensureType(valueAt(json, "builder"), value_t::string); + res.args = ensureType(valueAt(json, "args"), value_t::array); + res.env = ensureType(valueAt(json, "env"), value_t::object); return res; } diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index d7220e71d..61cef743d 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -1,4 +1,5 @@ #include "json-utils.hh" +#include "error.hh" namespace nix { @@ -16,4 +17,27 @@ nlohmann::json * get(nlohmann::json & map, const std::string & key) return &*i; } +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key) +{ + if (!map.contains(key)) + throw Error("Expected JSON object to contain key '%s' but it doesn't", key); + + return map[key]; +} + +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType + ) +{ + if (value.type() != expectedType) + throw Error( + "Expected JSON value to be of type '%s' but it is of type '%s'", + nlohmann::json(expectedType).type_name(), + value.type_name()); + + return value; +} } diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 5e63c1af4..77c63595c 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -10,6 +10,28 @@ const nlohmann::json * get(const nlohmann::json & map, const std::string & key); nlohmann::json * get(nlohmann::json & map, const std::string & key); +/** + * Get the value of a json object at a key safely, failing + * with a Nix Error if the key does not exist. + * + * Use instead of nlohmann::json::at() to avoid ugly exceptions. + * + * _Does not check whether `map` is an object_, use `ensureType` for that. + */ +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key); + +/** + * Ensure the type of a json object is what you expect, failing + * with a Nix Error if it isn't. + * + * Use before type conversions and element access to avoid ugly exceptions. + */ +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType); + /** * For `adl_serializer>` below, we need to track what * types are not already using `null`. Only for them can we use `null` From 31a6e10fe52fd4265501553ca479c51b510137e1 Mon Sep 17 00:00:00 2001 From: Simon Rainerson Date: Tue, 31 Jan 2023 11:36:13 +0100 Subject: [PATCH 74/78] Fix misread of source if path is already valid When receiving a stream of NARs through the ssh-ng protocol, an already existing path would cause the NAR archive to not be read in the stream, resulting in trying to parse the NAR as a ValidPathInfo. This results in the error message: error: not an absolute path: 'nix-archive-1' Fixes #6253 Usually this problem is avoided by running QueryValidPaths before AddMultipleToStore, but can arise when two parallel nix processes gets the same response from QueryValidPaths. This makes the problem more prominent when running builds in parallel. --- src/libstore/local-store.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 40a3bc194..17b4ecc73 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1196,6 +1196,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); + /* In case we are not interested in reading the NAR: discard it. */ + bool narRead = false; + Finally cleanup = [&]() { + if (!narRead) { + ParseSink sink; + parseDump(sink, source); + } + }; + addTempRoot(info.path); if (repair || !isValidPath(info.path)) { @@ -1220,6 +1229,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, TeeSource wrapperSource { source, hashSink }; + narRead = true; restorePath(realPath, wrapperSource); auto hashResult = hashSink.finish(); From ad410abbe0d66a72927bd715af355d88a4e939d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 12 May 2023 09:56:39 +0200 Subject: [PATCH 75/78] Stabilize `discard-references` It has been there for a few releases now (landed in 2.14.0), doesn't seem to cause any major issue and is wanted in a few places (https://github.com/NixOS/nix/pull/7087#issuecomment-1544471346). --- doc/manual/src/language/advanced-attributes.md | 10 ---------- doc/manual/src/release-notes/rl-next.md | 6 ++++++ src/libstore/build/local-derivation-goal.cc | 1 - src/libutil/experimental-features.cc | 11 +---------- src/libutil/experimental-features.hh | 1 - tests/check-refs.sh | 6 ++++-- 6 files changed, 11 insertions(+), 24 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 307971434..5e8aaeba0 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -320,16 +320,6 @@ Derivations can declare some infrequently used optional attributes. ``` - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ - > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). - > - > To use this attribute, you must enable the - > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature. - > For example, in [nix.conf](../command-ref/conf-file.md) you could add: - > - > ``` - > extra-experimental-features = discard-references - > ``` When using [structured attributes](#adv-attr-structuredAttrs), the attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 526b64fde..6516a2663 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -10,3 +10,9 @@ - [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. - Error messages regarding malformed input to [`derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. + +- The `discard-references` feature has been stabilized. + This means that the + [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + attribute is no longer guarded by an experimental flag and can be used + freely. diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b7a27490c..8b7fbdcda 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2307,7 +2307,6 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { - experimentalFeatureSettings.require(Xp::DiscardReferences); if (auto output = get(*udr, outputName)) { if (!output->is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 7c4112d32..782331283 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -182,15 +182,6 @@ constexpr std::array xpFeatureDetails = {{ the [`use-cgroups`](#conf-use-cgroups) setting for details. )", }, - { - .tag = Xp::DiscardReferences, - .name = "discard-references", - .description = R"( - Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations - that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for - runtime dependencies. - )", - }, { .tag = Xp::DaemonTrustOverride, .name = "daemon-trust-override", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index faf2e9398..add592ae6 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -27,7 +27,6 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, - DiscardReferences, DaemonTrustOverride, DynamicDerivations, ParseTomlTimestamps, diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 2778e491d..3b587d1e5 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -42,8 +42,10 @@ nix-build -o $RESULT check-refs.nix -A test7 nix-build -o $RESULT check-refs.nix -A test10 if isDaemonNewer 2.12pre20230103; then - enableFeatures discard-references - restartDaemon + if ! isDaemonNewer 2.16.0; then + enableFeatures discard-references + restartDaemon + fi # test11 should succeed. test11=$(nix-build -o $RESULT check-refs.nix -A test11) From afac001c39aadedcbe52ce45fbde8220834cf13f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 7 Aug 2023 11:23:19 +0200 Subject: [PATCH 76/78] Test the parallel copy over ssh-ng Regression test for https://github.com/NixOS/nix/issues/6253 --- tests/local.mk | 1 + tests/nix-copy-ssh-ng.sh | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 tests/nix-copy-ssh-ng.sh diff --git a/tests/local.mk b/tests/local.mk index 2afe91220..4edf31303 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -90,6 +90,7 @@ nix_tests = \ zstd.sh \ compression-levels.sh \ nix-copy-ssh.sh \ + nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ flakes/config.sh \ diff --git a/tests/nix-copy-ssh-ng.sh b/tests/nix-copy-ssh-ng.sh new file mode 100644 index 000000000..45e53c9c0 --- /dev/null +++ b/tests/nix-copy-ssh-ng.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore +clearCache + +remoteRoot=$TEST_ROOT/store2 +chmod -R u+w "$remoteRoot" || true +rm -rf "$remoteRoot" + +outPath=$(nix-build --no-out-link dependencies.nix) + +nix store ping --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" + +# Regression test for https://github.com/NixOS/nix/issues/6253 +nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & +nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs + +[ -f $remoteRoot$outPath/foobar ] From 4b1bd822ac7fd35b30f37c1b7705c523cc462761 Mon Sep 17 00:00:00 2001 From: Peter Waller Date: Fri, 30 Jun 2023 16:11:12 +0100 Subject: [PATCH 77/78] Try to realise CA derivations during queryMissing This enables nix to correctly report what will be fetched in the case that everything is a cache hit. Note however that if an intermediate build of something which is not cached could still cause products to end up being substituted if the intermediate build results in a CA path which is in the cache. Fixes #8615. Signed-off-by: Peter Waller --- src/libstore/misc.cc | 30 ++++++++++++++++++++ tests/ca/build-cache.sh | 51 ++++++++++++++++++++++++++++++++++ tests/ca/build.sh | 12 +------- tests/ca/content-addressed.nix | 18 ++++++++++++ tests/ca/local.mk | 1 + 5 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 tests/ca/build-cache.sh diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 14160dc8b..2fdf0a68d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -200,6 +200,36 @@ void Store::queryMissing(const std::vector & targets, auto drv = make_ref(derivationFromPath(bfd.drvPath)); ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + experimentalFeatureSettings.require(Xp::CaDerivations); + + // If there are unknown output paths, attempt to find if the + // paths are known to substituters through a realisation. + auto outputHashes = staticOutputHashes(*this, *drv); + knownOutputPaths = true; + + for (auto [outputName, hash] : outputHashes) { + if (!bfd.outputs.contains(outputName)) + continue; + + bool found = false; + for (auto &sub : getDefaultSubstituters()) { + auto realisation = sub->queryRealisation({hash, outputName}); + if (!realisation) + continue; + found = true; + if (!isValidPath(realisation->outPath)) + invalid.insert(realisation->outPath); + break; + } + if (!found) { + // Some paths did not have a realisation, this must be built. + knownOutputPaths = false; + break; + } + } + } + if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) diff --git a/tests/ca/build-cache.sh b/tests/ca/build-cache.sh new file mode 100644 index 000000000..6a4080fec --- /dev/null +++ b/tests/ca/build-cache.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +source common.sh + +# The substituters didn't work prior to this time. +requireDaemonNewerThan "2.18.0pre20230808" + +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out +nix derivation show "$drv" --arg seed 1 + +buildAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") + args+=("$@") + nix-build "${args[@]}" +} + +copyAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("-f" "./content-addressed.nix" "$derivationPath" --arg seed "$seedValue") + args+=("$@") + # Note: to copy CA derivations, we need to copy the realisations, which + # currently requires naming the installables, not just the derivation output + # path. + nix copy --to file://$cacheDir "${args[@]}" +} + +testRemoteCacheFor () { + local derivationPath=$1 + clearCache + copyAttr "$derivationPath" 1 + clearStore + # Check nothing gets built. + buildAttr "$derivationPath" 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse " will be built:" +} + +testRemoteCache () { + testRemoteCacheFor rootCA + testRemoteCacheFor dependentCA + testRemoteCacheFor dependentNonCA + testRemoteCacheFor dependentFixedOutput + testRemoteCacheFor dependentForBuildCA + testRemoteCacheFor dependentForBuildNonCA +} + +clearStore +testRemoteCache \ No newline at end of file diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 7754ad276..e1a8a7625 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -2,7 +2,7 @@ source common.sh -drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out nix derivation show "$drv" --arg seed 1 buildAttr () { @@ -14,14 +14,6 @@ buildAttr () { nix-build "${args[@]}" } -testRemoteCache () { - clearCache - local outPath=$(buildAttr dependentNonCA 1) - nix copy --to file://$cacheDir $outPath - clearStore - buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse "building dependent-non-ca" -} - testDeterministicCA () { [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]] } @@ -66,8 +58,6 @@ testNormalization () { test "$(stat -c %Y $outPath)" -eq 1 } -# Disabled until we have it properly working -# testRemoteCache clearStore testNormalization testDeterministicCA diff --git a/tests/ca/content-addressed.nix b/tests/ca/content-addressed.nix index 81bc4bf5c..2559c562f 100644 --- a/tests/ca/content-addressed.nix +++ b/tests/ca/content-addressed.nix @@ -61,6 +61,24 @@ rec { echo ${rootCA}/non-ca-hello > $out/dep ''; }; + dependentForBuildCA = mkCADerivation { + name = "dependent-for-build-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; + dependentForBuildNonCA = mkDerivation { + name = "dependent-for-build-non-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; dependentFixedOutput = mkDerivation { name = "dependent-fixed-output"; outputHashMode = "recursive"; diff --git a/tests/ca/local.mk b/tests/ca/local.mk index d15312708..0852e592e 100644 --- a/tests/ca/local.mk +++ b/tests/ca/local.mk @@ -1,6 +1,7 @@ ca-tests := \ $(d)/build-with-garbage-path.sh \ $(d)/build.sh \ + $(d)/build-cache.sh \ $(d)/concurrent-builds.sh \ $(d)/derivation-json.sh \ $(d)/duplicate-realisation-in-closure.sh \ From 60b7121d2c6d4322b7c2e8e7acfec7b701b2d3a1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 15 Jan 2023 17:39:04 -0500 Subject: [PATCH 78/78] Make the Derived Path family of types inductive for dynamic derivations We want to be able to write down `foo.drv^bar.drv^baz`: `foo.drv^bar.drv` is the dynamic derivation (since it is itself a derivation output, `bar.drv` from `foo.drv`). To that end, we create `Single{Derivation,BuiltPath}` types, that are very similar except instead of having multiple outputs (in a set or map), they have a single one. This is for everything to the left of the rightmost `^`. `NixStringContextElem` has an analogous change, and now can reuse `SingleDerivedPath` at the top level. In fact, if we ever get rid of `DrvDeep`, `NixStringContextElem` could be replaced with `SingleDerivedPath` entirely! Important note: some JSON formats have changed. We already can *produce* dynamic derivations, but we can't refer to them directly. Today, we can merely express building or example at the top imperatively over time by building `foo.drv^bar.drv`, and then with a second nix invocation doing `^baz`, but this is not declarative. The ethos of Nix of being able to write down the full plan everything you want to do, and then execute than plan with a single command, and for that we need the new inductive form of these types. Co-authored-by: Robert Hensing Co-authored-by: Valentin Gagarin --- doc/manual/src/release-notes/rl-next.md | 3 + src/build-remote/build-remote.cc | 7 +- src/libcmd/built-path.cc | 76 +++++- src/libcmd/built-path.hh | 50 +++- src/libcmd/installable-attr-path.cc | 2 +- src/libcmd/installable-derived-path.cc | 15 +- src/libcmd/installable-flake.cc | 2 +- src/libcmd/installable-value.cc | 3 +- src/libcmd/installables.cc | 36 ++- src/libcmd/repl.cc | 2 +- src/libexpr/eval-cache.cc | 2 +- src/libexpr/eval.cc | 59 +++-- src/libexpr/eval.hh | 10 +- src/libexpr/primops.cc | 15 +- src/libexpr/primops/context.cc | 7 +- src/libexpr/tests/derived-path.cc | 26 +-- src/libexpr/tests/value/context.cc | 70 ++++-- src/libexpr/value/context.cc | 97 +++++--- src/libexpr/value/context.hh | 21 +- src/libstore/build/derivation-goal.cc | 10 +- src/libstore/build/entry-points.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 15 +- src/libstore/build/worker.cc | 7 +- src/libstore/derived-path.cc | 247 ++++++++++++++++++-- src/libstore/derived-path.hh | 222 ++++++++++++++++-- src/libstore/downstream-placeholder.cc | 15 ++ src/libstore/downstream-placeholder.hh | 12 + src/libstore/legacy-ssh-store.cc | 3 + src/libstore/misc.cc | 85 ++++++- src/libstore/path-with-outputs.cc | 47 ++-- src/libstore/path-with-outputs.hh | 4 +- src/libstore/remote-store.cc | 21 +- src/libstore/store-api.hh | 1 + src/libstore/tests/derived-path.cc | 91 +++++++- src/libstore/tests/derived-path.hh | 14 +- src/libutil/comparator.hh | 16 ++ src/nix-build/nix-build.cc | 9 +- src/nix-env/nix-env.cc | 4 +- src/nix/app.cc | 7 +- src/nix/build.cc | 10 +- src/nix/bundle.cc | 2 +- src/nix/develop.cc | 2 +- src/nix/flake.cc | 6 +- src/nix/log.cc | 20 +- tests/build.sh | 2 +- tests/dyn-drv/build-built-drv.sh | 21 ++ tests/dyn-drv/local.mk | 3 +- tests/test-libstoreconsumer/main.cc | 2 +- 48 files changed, 1136 insertions(+), 267 deletions(-) create mode 100644 tests/dyn-drv/build-built-drv.sh diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 6516a2663..be922f95a 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -16,3 +16,6 @@ [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. + +- The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field. + This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 2fb17d06f..c1f03a8ef 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -322,7 +322,12 @@ connected: throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } }); + auto res = sshStore->buildPathsWithResults({ + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All {}, + } + }); // One path to build should produce exactly one build result assert(res.size() == 1); optResult = std::move(res[0]); diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index db9c440e3..c6cc7fa9c 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -8,13 +8,39 @@ namespace nix { -nlohmann::json BuiltPath::Built::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ } - return res; +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::pair +CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE std::map +CMP(SingleBuiltPath, BuiltPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +StorePath SingleBuiltPath::outPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) { return p.path; }, + [](const SingleBuiltPath::Built & b) { return b.output.second; }, + }, raw() + ); } StorePathSet BuiltPath::outPaths() const @@ -32,6 +58,40 @@ StorePathSet BuiltPath::outPaths() const ); } +nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + for (const auto & [outputName, outputPath] : outputs) { + res["outputs"][outputName] = store.printStorePath(outputPath); + } + return res; +} + +nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + auto & [outputName, outputPath] = output; + res["output"] = outputName; + res["outputPath"] = store.printStorePath(outputPath); + return res; +} + +nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json BuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; @@ -40,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); + staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto& [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { @@ -48,7 +108,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const if (!drvOutput) throw Error( "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); + store.printStorePath(p.drvPath->outPath()), outputName); auto thisRealisation = store.queryRealisation( DrvOutput{*drvOutput, outputName}); assert(thisRealisation); // We’ve built it, so we must diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 744e8090b..747bcc440 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -3,19 +3,60 @@ namespace nix { +struct SingleBuiltPath; + +struct SingleBuiltPathBuilt { + ref drvPath; + std::pair output; + + std::string to_string(const Store & store) const; + static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; + + DECLARE_CMP(SingleBuiltPathBuilt); +}; + +using _SingleBuiltPathRaw = std::variant< + DerivedPathOpaque, + SingleBuiltPathBuilt +>; + +struct SingleBuiltPath : _SingleBuiltPathRaw { + using Raw = _SingleBuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleBuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePath outPath() const; + + static SingleBuiltPath parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; +}; + +static inline ref staticDrv(StorePath drvPath) +{ + return make_ref(SingleBuiltPath::Opaque { drvPath }); +} + /** * A built derived path with hints in the form of optional concrete output paths. * * See 'BuiltPath' for more an explanation. */ struct BuiltPathBuilt { - StorePath drvPath; + ref drvPath; std::map outputs; - nlohmann::json toJSON(ref store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); + std::string to_string(const Store & store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; - GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -41,6 +82,7 @@ struct BuiltPath : _BuiltPathRaw { StorePathSet outPaths() const; RealisedPath::Set toRealisedPaths(Store & store) const; + nlohmann::json toJSON(const Store & store) const; }; typedef std::vector BuiltPaths; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index b35ca2910..2f89eee02 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() for (auto & [drvPath, outputs] : byDrvPath) res.push_back({ .path = DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = outputs, }, .info = make_ref(ExtraPathInfoValue::Value { diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 6ecf54b7c..b45641e8a 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -18,14 +18,7 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() std::optional InstallableDerivedPath::getStorePath() { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, derivedPath.raw()); + return derivedPath.getBaseStorePath(); } InstallableDerivedPath InstallableDerivedPath::parse( @@ -42,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( // Remove this prior to stabilizing the new CLI. if (storePath.isDerivation()) { auto oldDerivedPath = DerivedPath::Built { - .drvPath = storePath, + .drvPath = makeConstantStorePathRef(storePath), .outputs = OutputsSpec::All { }, }; warn( @@ -55,8 +48,10 @@ InstallableDerivedPath InstallableDerivedPath::parse( }, // If the user did use ^, we just do exactly what is written. [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + auto drv = make_ref(SingleDerivedPath::parse(*store, prefix)); + drvRequireExperiment(*drv); return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), + .drvPath = std::move(drv), .outputs = outputSpec, }; }, diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 4da9b131b..1b169c3bd 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() return {{ .path = DerivedPath::Built { - .drvPath = std::move(drvPath), + .drvPath = makeConstantStorePathRef(std::move(drvPath)), .outputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { std::set outputsToInstall; diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 1eff293cc..08ad35105 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -55,7 +55,8 @@ std::optional InstallableValue::trySinglePathToDerivedPaths else if (v.type() == nString) { return {{ - .path = state->coerceToDerivedPath(pos, v, errorCtx), + .path = DerivedPath::fromSingle( + state->coerceToSingleDerivedPath(pos, v, errorCtx)), .info = make_ref(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 9d593a01f..4c7d134ec 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -515,6 +515,30 @@ ref SourceExprCommand::parseInstallable( return installables.front(); } +static SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { + return SingleBuiltPath::Opaque { bo.path }; + }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd { + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built { + .drvPath = make_ref(std::move(drvPath)), + .output = { bfd.output, outputPath }, + }; + }, + }, + b.raw()); +} + std::vector Installable::build( ref evalStore, ref store, @@ -568,7 +592,10 @@ std::vector, BuiltPathWithResult>> Installable::build [&](const DerivedPath::Built & bfd) { auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { @@ -597,7 +624,10 @@ std::vector, BuiltPathWithResult>> Installable::build for (auto & [outputName, realisation] : buildResult.builtOutputs) outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info, .result = buildResult}}); }, @@ -691,7 +721,7 @@ StorePathSet Installable::toDerivations( : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { - drvPaths.insert(bfd.drvPath); + drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath)); }, }, b.path.raw()); diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index d15162e76..ea7d0e2ec 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -648,7 +648,7 @@ bool NixRepl::processLine(std::string line) if (command == ":b" || command == ":bl") { state->store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 9e734e654..6a60ba87b 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -599,7 +599,7 @@ string_t AttrCursor::getStringWithContext() return d.drvPath; }, [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath; + return b.drvPath->getBaseStorePath(); }, [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a8e6baea6..3e521b732 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1042,7 +1042,7 @@ void EvalState::mkOutputString( : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(), NixStringContext { NixStringContextElem::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .output = outputName, } }); @@ -2299,7 +2299,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon } -std::pair EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) +std::pair EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) { NixStringContext context; auto s = forceString(v, context, pos, errorCtx); @@ -2310,21 +2310,16 @@ std::pair EvalState::coerceToDerivedPathUnchecked s, csize) .withTrace(pos, errorCtx).debugThrow(); auto derivedPath = std::visit(overloaded { - [&](NixStringContextElem::Opaque && o) -> DerivedPath { - return DerivedPath::Opaque { - .path = std::move(o.path), - }; + [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { + return std::move(o); }, - [&](NixStringContextElem::DrvDeep &&) -> DerivedPath { + [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { error( "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", s).withTrace(pos, errorCtx).debugThrow(); }, - [&](NixStringContextElem::Built && b) -> DerivedPath { - return DerivedPath::Built { - .drvPath = std::move(b.drvPath), - .outputs = OutputsSpec::Names { std::move(b.output) }, - }; + [&](NixStringContextElem::Built && b) -> SingleDerivedPath { + return std::move(b); }, }, ((NixStringContextElem &&) *context.begin()).raw()); return { @@ -2334,12 +2329,12 @@ std::pair EvalState::coerceToDerivedPathUnchecked } -DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) { - auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx); + auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); auto s = s_; std::visit(overloaded { - [&](const DerivedPath::Opaque & o) { + [&](const SingleDerivedPath::Opaque & o) { auto sExpected = store->printStorePath(o.path); if (s != sExpected) error( @@ -2347,25 +2342,27 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str s, sExpected) .withTrace(pos, errorCtx).debugThrow(); }, - [&](const DerivedPath::Built & b) { - // TODO need derived path with single output to make this - // total. Will add as part of RFC 92 work and then this is - // cleaned up. - auto output = *std::get(b.outputs).begin(); - - auto drv = store->readDerivation(b.drvPath); - auto i = drv.outputs.find(output); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output); - auto optOutputPath = i->second.path(*store, drv.name, output); - // This is testing for the case of CA derivations - auto sExpected = optOutputPath - ? store->printStorePath(*optOutputPath) - : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); + [&](const SingleDerivedPath::Built & b) { + auto sExpected = std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + auto drv = store->readDerivation(o.path); + auto i = drv.outputs.find(b.output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); + auto optOutputPath = i->second.path(*store, drv.name, b.output); + // This is testing for the case of CA derivations + return optOutputPath + ? store->printStorePath(*optOutputPath) + : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + [&](const SingleDerivedPath::Built & o) { + return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + }, b.drvPath->raw()); if (s != sExpected) error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", - s, output, store->printStorePath(b.drvPath), sExpected) + s, b.output, b.drvPath->to_string(*store), sExpected) .withTrace(pos, errorCtx).debugThrow(); } }, derivedPath.raw()); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 29d0f05a1..0268a2a12 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -22,7 +22,7 @@ namespace nix { class Store; class EvalState; class StorePath; -struct DerivedPath; +struct SingleDerivedPath; enum RepairFlag : bool; @@ -532,12 +532,12 @@ public: StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** - * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only. + * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only. */ - std::pair coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); + std::pair coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); /** - * Coerce to `DerivedPath`. + * Coerce to `SingleDerivedPath`. * * Must be a string which is either a literal store path or a * "placeholder (see `DownstreamPlaceholder`). @@ -551,7 +551,7 @@ public: * source of truth, and ultimately tells us what we want, and then * we ensure the string corresponds to it. */ - DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); + SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); public: diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 430607214..54943b481 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -56,7 +56,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) .drvPath = b.drvPath, .outputs = OutputsSpec::Names { b.output }, }); - ensureValid(b.drvPath); + ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { auto ctxS = store->printStorePath(o.path); @@ -77,7 +77,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) if (!evalSettings.enableImportFromDerivation) debugThrowLastTrace(Error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - store->printStorePath(drvs.begin()->drvPath))); + drvs.begin()->to_string(*store))); /* Build/substitute the context. */ std::vector buildReqs; @@ -95,7 +95,11 @@ StringMap EvalState::realiseContext(const NixStringContext & context) /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { res.insert_or_assign( - DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = drv.drvPath, + .output = outputName, + }).render(), store->printStorePath(outputPath) ); } @@ -1251,7 +1255,10 @@ drvName, Bindings * attrs, Value & v) } }, [&](const NixStringContextElem::Built & b) { - drv.inputDrvs[b.drvPath].insert(b.output); + if (auto * p = std::get_if(&*b.drvPath)) + drv.inputDrvs[p->path].insert(b.output); + else + throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported"); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8b3468009..bfc731744 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -106,7 +106,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, contextInfos[std::move(d.drvPath)].allOutputs = true; }, [&](NixStringContextElem::Built && b) { - contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); + // FIXME should eventually show string context as is, no + // resolving here. + auto drvPath = resolveDerivedPath(*state.store, *b.drvPath); + contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); }, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; @@ -222,7 +225,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); context.emplace(NixStringContextElem::Built { - .drvPath = namePath, + .drvPath = makeConstantStorePathRef(namePath), .output = std::string { outputName }, }); } diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index c713fe28a..2a5ca64f6 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -21,12 +21,12 @@ TEST_F(DerivedPathExpressionTest, force_init) RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, prop_opaque_path_round_trip, - (const DerivedPath::Opaque & o)) + (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); state.mkStorePathString(o.path, *v); - auto d = state.coerceToDerivedPath(noPos, *v, ""); - RC_ASSERT(DerivedPath { o } == d); + auto d = state.coerceToSingleDerivedPath(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { o } == d); } // TODO use DerivedPath::Built for parameter once it supports a single output @@ -46,12 +46,12 @@ RC_GTEST_FIXTURE_PROP( auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } RC_GTEST_FIXTURE_PROP( @@ -61,12 +61,12 @@ RC_GTEST_FIXTURE_PROP( { auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, outPath); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } } /* namespace nix */ diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 0d9381577..c56b50b59 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,6 +8,8 @@ namespace nix { +// Test a few cases of invalid string context elements. + TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( NixStringContextElem::parse(""), @@ -38,6 +40,10 @@ TEST(NixStringContextElemTest, slash_invalid) { BadStorePath); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::Opaque`. + */ TEST(NixStringContextElemTest, opaque) { std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; auto elem = NixStringContextElem::parse(opaque); @@ -47,6 +53,10 @@ TEST(NixStringContextElemTest, opaque) { ASSERT_EQ(elem.to_string(), opaque); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::DrvDeep`. + */ TEST(NixStringContextElemTest, drvDeep) { std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(drvDeep); @@ -56,28 +66,62 @@ TEST(NixStringContextElemTest, drvDeep) { ASSERT_EQ(elem.to_string(), drvDeep); } -TEST(NixStringContextElemTest, built) { +/** + * Round trip (string <-> data structure) test for a simpler + * `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_opaque) { std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(built); auto * p = std::get_if(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(5) }, + })); ASSERT_EQ(elem.to_string(), built); } +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "bar"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(9) }, + })); + ASSERT_EQ(elem.to_string(), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive string context element. + */ +TEST(NixStringContextElemTest, built_built_xp) { + ASSERT_THROW( + NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); +} + } namespace rc { using namespace nix; -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Opaque { - .path = *gen::arbitrary(), - }); -} - Gen Arbitrary::arbitrary() { return gen::just(NixStringContextElem::DrvDeep { @@ -85,14 +129,6 @@ Gen Arbitrary::arb }); } -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Built { - .drvPath = *gen::arbitrary(), - .output = (*gen::arbitrary()).name, - }); -} - Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index f76fc76e4..d8116011e 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -4,29 +4,52 @@ namespace nix { -NixStringContextElem NixStringContextElem::parse(std::string_view s0) +NixStringContextElem NixStringContextElem::parse( + std::string_view s0, + const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; + std::function parseRest; + parseRest = [&]() -> SingleDerivedPath { + // Case on whether there is a '!' + size_t index = s.find("!"); + if (index == std::string_view::npos) { + return SingleDerivedPath::Opaque { + .path = StorePath { s }, + }; + } else { + std::string output { s.substr(0, index) }; + // Advance string to parse after the '!' + s = s.substr(index + 1); + auto drv = make_ref(parseRest()); + drvRequireExperiment(*drv, xpSettings); + return SingleDerivedPath::Built { + .drvPath = std::move(drv), + .output = std::move(output), + }; + } + }; + if (s.size() == 0) { throw BadNixStringContextElem(s0, "String context element should never be an empty string"); } + switch (s.at(0)) { case '!': { - s = s.substr(1); // advance string to parse after first ! - size_t index = s.find("!"); - // This makes index + 1 safe. Index can be the length (one after index - // of last character), so given any valid character index --- a - // successful find --- we can add one. - if (index == std::string_view::npos) { + // Advance string to parse after the '!' + s = s.substr(1); + + // Find *second* '!' + if (s.find("!") == std::string_view::npos) { throw BadNixStringContextElem(s0, "String content element beginning with '!' should have a second '!'"); } - return NixStringContextElem::Built { - .drvPath = StorePath { s.substr(index + 1) }, - .output = std::string(s.substr(0, index)), - }; + + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } case '=': { return NixStringContextElem::DrvDeep { @@ -34,33 +57,51 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0) }; } default: { - return NixStringContextElem::Opaque { - .path = StorePath { s }, - }; + // Ensure no '!' + if (s.find("!") != std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element not beginning with '!' should not have a second '!'"); + } + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } } } -std::string NixStringContextElem::to_string() const { - return std::visit(overloaded { +std::string NixStringContextElem::to_string() const +{ + std::string res; + + std::function toStringRest; + toStringRest = [&](auto & p) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + res += o.path.to_string(); + }, + [&](const SingleDerivedPath::Built & o) { + res += o.output; + res += '!'; + toStringRest(*o.drvPath); + }, + }, p.raw()); + }; + + std::visit(overloaded { [&](const NixStringContextElem::Built & b) { - std::string res; res += '!'; - res += b.output; - res += '!'; - res += b.drvPath.to_string(); - return res; - }, - [&](const NixStringContextElem::DrvDeep & d) { - std::string res; - res += '='; - res += d.drvPath.to_string(); - return res; + toStringRest(b); }, [&](const NixStringContextElem::Opaque & o) { - return std::string { o.path.to_string() }; + toStringRest(o); + }, + [&](const NixStringContextElem::DrvDeep & d) { + res += '='; + res += d.drvPath.to_string(); }, }, raw()); + + return res; } } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 287ae08a9..a1b71695b 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "comparator.hh" -#include "path.hh" +#include "derived-path.hh" #include @@ -31,11 +31,7 @@ public: * * Encoded as just the path: ‘’. */ -struct NixStringContextElem_Opaque { - StorePath path; - - GENERATE_CMP(NixStringContextElem_Opaque, me->path); -}; +typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque; /** * Path to a derivation and its entire build closure. @@ -57,12 +53,7 @@ struct NixStringContextElem_DrvDeep { * * Encoded in the form ‘!!’. */ -struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; - - GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); -}; +typedef SingleDerivedPath::Built NixStringContextElem_Built; using _NixStringContextElem_Raw = std::variant< NixStringContextElem_Opaque, @@ -93,8 +84,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘’ * - ‘=’ * - ‘!!’ + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static NixStringContextElem parse(std::string_view s); + static NixStringContextElem parse( + std::string_view s, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); std::string to_string() const; }; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5e37f7ecb..8bdf2f367 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -65,7 +65,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -84,7 +84,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -95,7 +95,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -1490,7 +1490,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) for (auto & outputName : outputs->second) { auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = dg->drvPath, + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 4aa4d6dca..e941b4e65 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -77,7 +77,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); return goal->getBuildResult(DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); } catch (Error & e) { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8b7fbdcda..920097680 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1172,6 +1172,19 @@ void LocalDerivationGoal::writeStructuredAttrs() } +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { @@ -1179,7 +1192,7 @@ static StorePath pathPartOfReq(const DerivedPath & req) return bo.path; }, [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; + return pathPartOfReq(*bfd.drvPath); }, }, req.raw()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index a9ca9cbbc..6779dbcf3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -111,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -265,7 +268,7 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built{makeConstantStorePathRef(goal->drvPath), goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 52d073f81..3594b7570 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -7,52 +7,133 @@ namespace nix { -nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ + } +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::string +CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE OutputsSpec +CMP(SingleDerivedPath, DerivedPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +{ + return store.printStorePath(path); +} + +nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["path"] = store->printStorePath(path); + res["drvPath"] = drvPath->toJSON(store); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); + res["output"] = output; + auto outputPathIter = outputMap.find(output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; return res; } -nlohmann::json DerivedPath::Built::toJSON(ref store) const { +nlohmann::json DerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); + res["drvPath"] = drvPath->toJSON(store); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); for (const auto & [output, outputPathOpt] : outputMap) { if (!outputs.contains(output)) continue; if (outputPathOpt) - res["outputs"][output] = store->printStorePath(*outputPathOpt); + res["outputs"][output] = store.printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } return res; } +nlohmann::json SingleDerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json DerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); } +std::string SingleDerivedPath::Built::to_string(const Store & store) const +{ + return drvPath->to_string(store) + "^" + output; +} + +std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +{ + return drvPath->to_string(store) + "!" + output; +} + std::string DerivedPath::Built::to_string(const Store & store) const { - return store.printStorePath(drvPath) + return drvPath->to_string(store) + '^' + outputs.to_string(); } std::string DerivedPath::Built::to_string_legacy(const Store & store) const { - return store.printStorePath(drvPath) - + '!' + return drvPath->to_string_legacy(store) + + "!" + outputs.to_string(); } +std::string SingleDerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + std::string DerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + +std::string SingleDerivedPath::to_string_legacy(const Store & store) const { return std::visit(overloaded { - [&](const DerivedPath::Built & req) { return req.to_string(store); }, - [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); }, }, this->raw()); } @@ -70,30 +151,156 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque &) { + // plain drv path; no experimental features required. + }, + [&](const SingleDerivedPath::Built &) { + xpSettings.require(Xp::DynamicDerivations); + }, + }, drv.raw()); +} + +SingleDerivedPath::Built SingleDerivedPath::Built::parse( + const Store & store, ref drv, + std::string_view output, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); return { - .drvPath = store.parseStorePath(drvS), + .drvPath = drv, + .output = std::string { output }, + }; +} + +DerivedPath::Built DerivedPath::Built::parse( + const Store & store, ref drv, + std::string_view outputsS, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); + return { + .drvPath = drv, .outputs = OutputsSpec::parse(outputsS), }; } -static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) +static SingleDerivedPath parseWithSingle( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) { - size_t n = s.find(separator); + size_t n = s.rfind(separator); + return n == s.npos + ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) + : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "^", xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "!", xpSettings); +} + +static DerivedPath parseWith( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); + : (DerivedPath) DerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +DerivedPath DerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) { - return parseWith(store, s, "^"); + return parseWith(store, s, "^", xpSettings); } -DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +DerivedPath DerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) { - return parseWith(store, s, "!"); + return parseWith(store, s, "!", xpSettings); +} + +DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) -> DerivedPath { + return o; + }, + [&](const SingleDerivedPath::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + }, req.raw()); +} + +const StorePath & SingleDerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +const StorePath & DerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +template +static inline const StorePath & getBaseStorePath_(const DP & derivedPath) +{ + return std::visit(overloaded { + [&](const typename DP::Built & bfd) -> auto & { + return bfd.drvPath->getBaseStorePath(); + }, + [&](const typename DP::Opaque & bo) -> auto & { + return bo.path; + }, + }, derivedPath.raw()); +} + +const StorePath & SingleDerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); +} + +const StorePath & DerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); } } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 7a4261ce0..ec30dac61 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -24,28 +24,37 @@ class Store; struct DerivedPathOpaque { StorePath path; - nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; +struct SingleDerivedPath; + /** - * A derived path that is built from a derivation + * A single derived path that is built from a derivation * - * Built derived paths are pair of a derivation and some output names. - * They are evaluated by building the derivation, and then replacing the - * output names with the resulting outputs. - * - * Note that does mean a derived store paths evaluates to multiple - * opaque paths, which is sort of icky as expressions are supposed to - * evaluate to single values. Perhaps this should have just a single - * output name. + * Built derived paths are pair of a derivation and an output name. They are + * evaluated by building the derivation, and then taking the resulting output + * path of the given output name. */ -struct DerivedPathBuilt { - StorePath drvPath; - OutputsSpec outputs; +struct SingleDerivedPathBuilt { + ref drvPath; + std::string output; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; /** * Uses `^` as the separator @@ -57,11 +66,139 @@ struct DerivedPathBuilt { std::string to_string_legacy(const Store & store) const; /** * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); - nlohmann::json toJSON(ref store) const; + static SingleDerivedPathBuilt parse( + const Store & store, ref drvPath, + std::string_view outputs, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; - GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(SingleDerivedPathBuilt); +}; + +using _SingleDerivedPathRaw = std::variant< + DerivedPathOpaque, + SingleDerivedPathBuilt +>; + +/** + * A "derived path" is a very simple sort of expression (not a Nix + * language expression! But an expression in a the general sense) that + * evaluates to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ +struct SingleDerivedPath : _SingleDerivedPathRaw { + using Raw = _SingleDerivedPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleDerivedPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; +}; + +static inline ref makeConstantStorePathRef(StorePath drvPath) +{ + return make_ref(SingleDerivedPath::Opaque { drvPath }); +} + +/** + * A set of derived paths that are built from a derivation + * + * Built derived paths are pair of a derivation and some output names. + * They are evaluated by building the derivation, and then replacing the + * output names with the resulting outputs. + * + * Note that does mean a derived store paths evaluates to multiple + * opaque paths, which is sort of icky as expressions are supposed to + * evaluate to single values. Perhaps this should have just a single + * output name. + */ +struct DerivedPathBuilt { + ref drvPath; + OutputsSpec outputs; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivedPathBuilt parse( + const Store & store, ref, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; + + DECLARE_CMP(DerivedPathBuilt); }; using _DerivedPathRaw = std::variant< @@ -71,13 +208,13 @@ using _DerivedPathRaw = std::variant< /** * A "derived path" is a very simple sort of expression that evaluates - * to (concrete) store path. It is either: + * to one or more (concrete) store paths. It is either: * - * - opaque, in which case it is just a concrete store path with + * - opaque, in which case it is just a single concrete store path with * possibly no known derivation * - * - built, in which case it is a pair of a derivation path and an - * output name. + * - built, in which case it is a pair of a derivation path and some + * output names. */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; @@ -90,6 +227,18 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + /** * Uses `^` as the separator */ @@ -100,14 +249,43 @@ struct DerivedPath : _DerivedPathRaw { std::string to_string_legacy(const Store & store) const; /** * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parse(const Store & store, std::string_view); + static DerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parseLegacy(const Store & store, std::string_view); + static DerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convert a `SingleDerivedPath` to a `DerivedPath`. + */ + static DerivedPath fromSingle(const SingleDerivedPath &); + + nlohmann::json toJSON(Store & store) const; }; typedef std::vector DerivedPaths; +/** + * Used by various parser functions to require experimental features as + * needed. + * + * Somewhat unfortunate this cannot just be an implementation detail for + * this module. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index d623c05e2..885b3604d 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -38,4 +38,19 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( }; } +DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & b) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return DownstreamPlaceholder::unknownCaOutput(o.path, b.output); + }, + [&](const SingleDerivedPath::Built & b2) { + return DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2), + b.output); + }, + }, b.drvPath->raw()); +} + } diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index 97f77e6b8..9372dcd58 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -3,6 +3,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" namespace nix { @@ -73,6 +74,17 @@ public: const DownstreamPlaceholder & drvPlaceholder, std::string_view outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convenience constructor that handles both cases (unknown + * content-addressed output and unknown derivation), delegating as + * needed to `unknownCaOutput` and `unknownDerivation`. + * + * Recursively builds up a placeholder from a + * `SingleDerivedPath::Built.drvPath` chain. + */ + static DownstreamPlaceholder fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & built); }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fa17d606d..78b05031a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -358,6 +358,9 @@ public: [&](const StorePath & drvPath) { throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, }, sOrDrvPath); } conn->to << ss; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 14160dc8b..1ece7a4c0 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -132,7 +132,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second })); }; auto checkOutput = [&]( @@ -176,10 +176,18 @@ void Store::queryMissing(const std::vector & targets, std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - if (!isValidPath(bfd.drvPath)) { + auto drvPathP = std::get_if(&*bfd.drvPath); + if (!drvPathP) { + // TODO make work in this case. + warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this)); + return; + } + auto & drvPath = drvPathP->path; + + if (!isValidPath(drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(bfd.drvPath); + state->unknown.insert(drvPath); return; } @@ -187,7 +195,7 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; @@ -197,15 +205,15 @@ void Store::queryMissing(const std::vector & targets, } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(bfd.drvPath)); - ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + auto drv = make_ref(derivationFromPath(drvPath)); + ParsedDerivation parsedDrv(StorePath(drvPath), *drv); if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); } else - mustBuildDrv(bfd.drvPath, *drv); + mustBuildDrv(drvPath, *drv); }, [&](const DerivedPath::Opaque & bo) { @@ -310,7 +318,9 @@ std::map drvOutputReferences( OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { - auto outputsOpt_ = store.queryPartialDerivationOutputMap(bfd.drvPath, evalStore_); + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + + auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_); auto outputsOpt = std::visit(overloaded { [&](const OutputsSpec::All &) { @@ -325,7 +335,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, if (!pOutputPathOpt) throw Error( "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); + bfd.drvPath->to_string(store), output); outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); } return outputsOpt; @@ -335,11 +345,64 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(store.printStorePath(bfd.drvPath), outputName); + throw MissingRealisation(bfd.drvPath->to_string(store), outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } return outputs; } + +StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_); + if (outputPaths.count(bfd.output) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store.printStorePath(drvPath), bfd.output); + auto & optPath = outputPaths.at(bfd.output); + if (!optPath) + throw Error("'%s' does not yet map to a known concrete store path", + bfd.to_string(store)); + return *optPath; + }, + }, req.raw()); +} + + +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath); + auto outputMap = store.queryDerivationOutputMap(drvPath); + auto outputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast(names); + }, + }, bfd.outputs.raw()); + for (auto iter = outputMap.begin(); iter != outputMap.end();) { + auto & outputName = iter->first; + if (bfd.outputs.contains(outputName)) { + outputsLeft.erase(outputName); + ++iter; + } else { + iter = outputMap.erase(iter); + } + } + if (!outputsLeft.empty()) + throw Error("derivation '%s' does not have an outputs %s", + store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(std::get(bfd.outputs)))); + return outputMap; +} + } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 869b490ad..81f360f3a 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -16,10 +16,16 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty()) { - return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::Names { outputs }, + }; } else if (path.isDerivation()) { assert(outputs.empty()); - return DerivedPath::Built { path, OutputsSpec::All { } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::All { }, + }; } else { return DerivedPath::Opaque { path }; } @@ -34,29 +40,36 @@ std::vector toDerivedPaths(const std::vector } -std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) +StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) -> std::variant { + [&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](const DerivedPath::Built & bfd) -> std::variant { - return StorePathWithOutputs { - .path = bfd.drvPath, - // Use legacy encoding of wildcard as empty set - .outputs = std::visit(overloaded { - [&](const OutputsSpec::All &) -> StringSet { - return {}; - }, - [&](const OutputsSpec::Names & outputs) { - return static_cast(outputs); - }, - }, bfd.outputs.raw()), - }; + [&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { + return StorePathWithOutputs { + .path = bo.path, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast(outputs); + }, + }, bfd.outputs.raw()), + }; + }, + [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult { + return std::monostate {}; + }, + }, bfd.drvPath->raw()); }, }, p.raw()); } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index d75850868..57e03252d 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -23,7 +23,9 @@ struct StorePathWithOutputs DerivedPath toDerivedPath() const; - static std::variant tryFromDerivedPath(const DerivedPath &); + typedef std::variant ParseResult; + + static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &); }; std::vector toDerivedPaths(const std::vector); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 21258daec..58f72beb9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -656,6 +656,9 @@ static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & con GET_PROTOCOL_MAJOR(conn.daemonVersion), GET_PROTOCOL_MINOR(conn.daemonVersion)); }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); + }, }, sOrDrvPath); } conn.to << ss; @@ -670,9 +673,16 @@ void RemoteStore::copyDrvsFromEvalStore( /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : paths) - if (auto p = std::get_if(&i)) - drvPaths2.insert(p->drvPath); + for (const auto & i : paths) { + std::visit(overloaded { + [&](const DerivedPath::Opaque & bp) { + // Do nothing, path is hopefully there already + }, + [&](const DerivedPath::Built & bp) { + drvPaths2.insert(bp.drvPath->getBaseStorePath()); + }, + }, i.raw()); + } copyClosure(*evalStore, *this, drvPaths2); } } @@ -742,7 +752,8 @@ std::vector RemoteStore::buildPathsWithResults( }; OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); + auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { @@ -750,7 +761,7 @@ std::vector RemoteStore::buildPathsWithResults( if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - printStorePath(bfd.drvPath), output); + printStorePath(drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f9029ade1..da1a3eefb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -945,6 +945,7 @@ void removeTempRoots(); * Resolve the derived path completely, failing if any derivation output * is unknown. */ +StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index 160443ec1..d6549f66f 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -17,14 +17,34 @@ Gen Arbitrary::arbitrary() }); } +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + Gen Arbitrary::arbitrary() { return gen::just(DerivedPath::Built { - .drvPath = *gen::arbitrary(), + .drvPath = make_ref(*gen::arbitrary()), .outputs = *gen::arbitrary(), }); } +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { @@ -45,12 +65,69 @@ class DerivedPathTest : public LibStoreTest { }; -// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is -// no a real fixture. -// -// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args -TEST_F(DerivedPathTest, force_init) -{ +/** + * Round trip (string <-> data structure) test for + * `DerivedPath::Opaque`. + */ +TEST_F(DerivedPathTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = DerivedPath::parse(*store, opaque); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store->parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(*store), opaque); +} + +/** + * Round trip (string <-> data structure) test for a simpler + * `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_opaque) { + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo"; + auto elem = DerivedPath::parse(*store, built); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" })); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"; + auto elem = DerivedPath::parse(*store, built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" })); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "foo"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive derived path. + */ +TEST_F(DerivedPathTest, built_built_xp) { + ASSERT_THROW( + DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"), + MissingExperimentalFeature); } RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 506f3ccb1..98d61f228 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -12,8 +12,18 @@ namespace rc { using namespace nix; template<> -struct Arbitrary { - static Gen arbitrary(); +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); }; template<> diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 9f661c5c3..7982fdc5e 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,22 @@ #pragma once ///@file +/** + * Declare comparison methods without defining them. + */ +#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \ + bool operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(my_type) \ + DECLARE_ONE_CMP(==, my_type) +#define DECLARE_LEQ(my_type) \ + DECLARE_ONE_CMP(<, my_type) +#define DECLARE_NEQ(my_type) \ + DECLARE_ONE_CMP(!=, my_type) +#define DECLARE_CMP(my_type) \ + DECLARE_EQUAL(my_type) \ + DECLARE_LEQ(my_type) \ + DECLARE_NEQ(my_type) + /** * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6510df8f0..66f319c3e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -393,7 +393,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { - .drvPath = bashDrv, + .drvPath = makeConstantStorePathRef(bashDrv), .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); @@ -417,7 +417,7 @@ static void main_nix_build(int argc, char * * argv) })) { pathsToBuild.push_back(DerivedPath::Built { - .drvPath = inputDrv, + .drvPath = makeConstantStorePathRef(inputDrv), .outputs = OutputsSpec::Names { inputOutputs }, }); pathsToCopy.insert(inputDrv); @@ -590,7 +590,10 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); + pathsToBuild.push_back(DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::Names{outputName}, + }); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 91b073b49..8dd071aa6 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -481,7 +481,7 @@ static void printMissing(EvalState & state, DrvInfos & elems) for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) targets.push_back(DerivedPath::Built{ - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else @@ -759,7 +759,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) std::vector paths { drvPath ? (DerivedPath) (DerivedPath::Built { - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }) : (DerivedPath) (DerivedPath::Opaque { diff --git a/src/nix/app.cc b/src/nix/app.cc index e0f68b4fc..16a921194 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -25,7 +25,8 @@ StringPairs resolveRewrites( if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) for (auto & [ outputName, outputPath ] : drvDep->outputs) res.emplace( - DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), + DownstreamPlaceholder::unknownCaOutput( + drvDep->drvPath->outPath(), outputName).render(), store.printStorePath(outputPath) ); return res; @@ -65,7 +66,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { /* We want all outputs of the drv */ return DerivedPath::Built { - .drvPath = d.drvPath, + .drvPath = makeConstantStorePathRef(d.drvPath), .outputs = OutputsSpec::All {}, }; }, @@ -106,7 +107,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { .context = { DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::Names { outputName }, } }, .program = program, diff --git a/src/nix/build.cc b/src/nix/build.cc index ad1842a4e..479100186 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -9,18 +9,18 @@ using namespace nix; -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) +static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - std::visit([&res, store](const auto & t) { + std::visit([&](const auto & t) { res.push_back(t.toJSON(store)); }, t.raw()); } return res; } -nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, ref store) +static nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, const Store & store) { auto res = nlohmann::json::array(); for (auto & b : buildables) { @@ -125,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile printMissing(store, pathsToBuild, lvlError); if (json) - logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + logger->cout("%s", derivedPathsToJSON(pathsToBuild, *store).dump()); return; } @@ -136,7 +136,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile installables, repair ? bmRepair : buildMode); - if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index bcc00d490..5a80f0308 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -109,7 +109,7 @@ struct CmdBundle : InstallableValueCommand store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 195eeaa21..c033804e4 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -235,7 +235,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore /* Build the derivation. */ store->buildPaths( { DerivedPath::Built { - .drvPath = shellDrvPath, + .drvPath = makeConstantStorePathRef(shellDrvPath), .outputs = OutputsSpec::All { }, }}, bmNormal, evalStore); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3ce1de44a..83b74c8ca 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -544,9 +544,9 @@ struct CmdFlakeCheck : FlakeCommand *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { drvPaths.push_back(DerivedPath::Built { - .drvPath = *drvPath, - .outputs = OutputsSpec::All { }, - }); + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All { }, + }); } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index aaf829764..9a9bd30f9 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -33,6 +33,17 @@ struct CmdLog : InstallableCommand auto b = installable->toDerivedPath(); + // For compat with CLI today, TODO revisit + auto oneUp = std::visit(overloaded { + [&](const DerivedPath::Opaque & bo) { + return make_ref(bo); + }, + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + }, b.path.raw()); + auto path = resolveDerivedPath(*store, *oneUp); + RunPager pager; for (auto & sub : subs) { auto * logSubP = dynamic_cast(&*sub); @@ -42,14 +53,7 @@ struct CmdLog : InstallableCommand } auto & logSub = *logSubP; - auto log = std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) { - return logSub.getBuildLog(bo.path); - }, - [&](const DerivedPath::Built & bfd) { - return logSub.getBuildLog(bfd.drvPath); - }, - }, b.path.raw()); + auto log = logSub.getBuildLog(path); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/tests/build.sh b/tests/build.sh index 8ae20f0df..7fbdb0f07 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -78,7 +78,7 @@ expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e | grepQuiet "has 2 entries in its context. It should only have exactly one entry" nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status ' - (.[0] | .path | match(".*multiple-outputs-e.drv")) + (.[0] | match(".*multiple-outputs-e.drv")) ' # Test building from raw store path to drv not expression. diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/dyn-drv/build-built-drv.sh new file mode 100644 index 000000000..647be9457 --- /dev/null +++ b/tests/dyn-drv/build-built-drv.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source common.sh + +# In the corresponding nix file, we have two derivations: the first, named `hello`, +# is a normal recursive derivation, while the second, named dependent, has the +# new outputHashMode "text". Note that in "dependent", we don't refer to the +# build output of `hello`, but only to the path of the drv file. For this reason, +# we only need to: +# +# - instantiate `hello` +# - build `producingDrv` +# - check that the path of the output coincides with that of the original derivation + +out1=$(nix build -f ./text-hashed-output.nix hello --no-link) + +clearStore + +drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) + +expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk index f065a5627..b087ecd1c 100644 --- a/tests/dyn-drv/local.mk +++ b/tests/dyn-drv/local.mk @@ -1,6 +1,7 @@ dyn-drv-tests := \ $(d)/text-hashed-output.sh \ - $(d)/recursive-mod-json.sh + $(d)/recursive-mod-json.sh \ + $(d)/build-built-drv.sh install-tests-groups += dyn-drv diff --git a/tests/test-libstoreconsumer/main.cc b/tests/test-libstoreconsumer/main.cc index 31b6d8ef1..c61489af6 100644 --- a/tests/test-libstoreconsumer/main.cc +++ b/tests/test-libstoreconsumer/main.cc @@ -23,7 +23,7 @@ int main (int argc, char **argv) std::vector paths { DerivedPath::Built { - .drvPath = store->parseStorePath(drvPath), + .drvPath = makeConstantStorePathRef(store->parseStorePath(drvPath)), .outputs = OutputsSpec::Names{"out"} } };