From 3ae1489847e3bde531833571d28a6e0915b4b766 Mon Sep 17 00:00:00 2001 From: Walter Franzini <5097668+wfranzini@users.noreply.github.com> Date: Tue, 31 Jan 2023 23:33:05 +0100 Subject: [PATCH 001/100] nix flakes metadata: Show lastModified timestamp for each input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this commit, the output of `nix flake metadata` in nix repo looked like this: ... Last modified: 2023-07-09 16:00:16 Inputs: ├───flake-compat: github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9 ├───lowdown-src: github:kristapsdz/lowdown/d2c2b44ff6c27b936ec27358a2653caaef8f73b8 ├───nixpkgs: github:NixOS/nixpkgs/04a75b2eecc0acf6239acf9dd04485ff8d14f425 └───nixpkgs-regression: github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2 This commit changes that to: ... Last modified: 2023-07-09 16:00:16 Inputs: ├───flake-compat: github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9 (2023-01-17 11:47:33) ├───lowdown-src: github:kristapsdz/lowdown/d2c2b44ff6c27b936ec27358a2653caaef8f73b8 (2021-10-06 10:00:07) ├───nixpkgs: github:NixOS/nixpkgs/04a75b2eecc0acf6239acf9dd04485ff8d14f425 (2022-12-08 01:04:00) └───nixpkgs-regression: github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2 (2022-01-24 19:20:45) --- src/nix/flake.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 87dd4da1b..b62c45d7e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -233,9 +233,13 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON bool last = i + 1 == node.inputs.size(); if (auto lockedNode = std::get_if<0>(&input.second)) { - logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + std::string lastModifiedStr = ""; + if (auto lastModified = (*lockedNode)->lockedRef.input.getLastModified()) + lastModifiedStr = fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%F %T")); + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s%s", prefix + (last ? treeLast : treeConn), input.first, - (*lockedNode)->lockedRef); + (*lockedNode)->lockedRef, + lastModifiedStr); bool firstVisit = visited.insert(*lockedNode).second; From b4b1a07f9771d4fd106323672d833a1cb17c7364 Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Fri, 13 Oct 2023 06:48:35 +0530 Subject: [PATCH 002/100] store info alias created --- src/nix/ping-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index ec450e8e0..5f8a10e22 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -47,3 +47,4 @@ struct CmdPingStore : StoreCommand, MixJSON }; static auto rCmdPingStore = registerCommand2({"store", "ping"}); +static auto rCmdInfoStore = registerCommand2({"store", "info"}); From 5c65379b228cee2c37408e1810c6654da5c91b90 Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Fri, 13 Oct 2023 07:16:05 +0530 Subject: [PATCH 003/100] info store alias added to store-ping --- src/nix/ping-store.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 5f8a10e22..247d9c6bb 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -46,5 +46,15 @@ struct CmdPingStore : StoreCommand, MixJSON } }; +struct CmdInfoStore : CmdPingStore +{ + void run(nix::ref store) override + { + warn("'nix store info' is a deprecated alias for 'nix store ping'"); + CmdPingStore::run(store); + } +}; + + static auto rCmdPingStore = registerCommand2({"store", "ping"}); -static auto rCmdInfoStore = registerCommand2({"store", "info"}); +static auto rCmdInfoStore = registerCommand2({"store", "info"}); From 4112dd1fc93c9ff03a5a4e8be773c45ebefbbd1f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Sep 2023 17:08:04 +0200 Subject: [PATCH 004/100] Mark fetchTree as stable --- doc/manual/src/release-notes/rl-next.md | 4 +++- src/libexpr/primops/fetchTree.cc | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 418c1187c..c905f445f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -6,4 +6,6 @@ - The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. -- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. \ No newline at end of file +- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. + +- `builtins.fetchTree` is now marked as stable. diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 3d7a85047..0335e5f36 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -213,7 +213,6 @@ static RegisterPrimOp primop_fetchTree({ ``` )", .fun = prim_fetchTree, - .experimentalFeature = Xp::Flakes, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, From a23cc147cb93fe9138b9a2f8283ad3ebbee43088 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 13 Oct 2023 11:00:55 -0400 Subject: [PATCH 005/100] Factor out Perl bindings Nix package Progress breaking up `flake.nix` by introducing separate `default.nix` files which make sense on their own. (This one is a regular `callPackage`-able package.) --- flake.nix | 43 ++++------------------------------------ perl/default.nix | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 39 deletions(-) create mode 100644 perl/default.nix diff --git a/flake.nix b/flake.nix index 301e65545..2263ea572 100644 --- a/flake.nix +++ b/flake.nix @@ -474,45 +474,10 @@ hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { - name = "nix-perl-${version}"; - - src = fileset.toSource { - root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - ./perl - ./.version - ./m4 - ./mk - ]); - }; - - nativeBuildInputs = - [ buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - pkgs.perl - boost - ] - ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium - ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; - }); + passthru.perl-bindings = final.callPackage ./perl { + inherit fileset; + stdenv = currentStdenv; + }; meta.platforms = lib.platforms.unix; }); diff --git a/perl/default.nix b/perl/default.nix new file mode 100644 index 000000000..4687976a1 --- /dev/null +++ b/perl/default.nix @@ -0,0 +1,51 @@ +{ lib, fileset +, stdenv +, perl, perlPackages +, autoconf-archive, autoreconfHook, pkg-config +, nix, curl, bzip2, xz, boost, libsodium, darwin +}: + +perl.pkgs.toPerlModule (stdenv.mkDerivation { + name = "nix-perl-${nix.version}"; + + src = fileset.toSource { + root = ../.; + fileset = fileset.unions [ + ../.version + ../m4 + ../mk + ./MANIFEST + ./Makefile + ./Makefile.config.in + ./configure.ac + ./lib + ./local.mk + ]; + }; + + nativeBuildInputs = + [ autoconf-archive + autoreconfHook + pkg-config + ]; + + buildInputs = + [ nix + curl + bzip2 + xz + perl + boost + ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + + configureFlags = [ + "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" + "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" + ]; + + enableParallelBuilding = true; + + postUnpack = "sourceRoot=$sourceRoot/perl"; +}) From e5ce53f3db93ddbc9cca6d51d5fc0e2fd7335517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=A9clairevoyant?= <848000+eclairevoyant@users.noreply.github.com> Date: Sun, 15 Oct 2023 15:08:07 -0400 Subject: [PATCH 006/100] explicitly set meta.mainProgram --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 301e65545..05bb8fad4 100644 --- a/flake.nix +++ b/flake.nix @@ -515,6 +515,7 @@ }); meta.platforms = lib.platforms.unix; + meta.mainProgram = "nix"; }); lowdown-nix = with final; currentStdenv.mkDerivation rec { From c27d2f8da9643a0a87e36ae992b29ea00a50e0d9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 23:37:55 -0400 Subject: [PATCH 007/100] Add two more completions tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks @ncfavier for catching these regressions in my PR. Co-Authored-By: Naïm Favier --- tests/functional/completions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh index 19dc61098..7c1e4b287 100644 --- a/tests/functional/completions.sh +++ b/tests/functional/completions.sh @@ -48,6 +48,8 @@ EOF [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]] ## Out of order [[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] From 483d99c622414504bb1b6a565f30ca6f21284315 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 10:42:15 -0400 Subject: [PATCH 008/100] Add API docs to some args-related functionality --- src/libcmd/command.hh | 78 ++++++++++++++++++++++++++++++++++++------- src/libutil/args.hh | 59 ++++++++++++++++++++++++++++++-- 2 files changed, 123 insertions(+), 14 deletions(-) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 96236b987..5c4569001 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -34,21 +34,28 @@ struct NixMultiCommand : virtual MultiCommand, virtual Command // For the overloaded run methods #pragma GCC diagnostic ignored "-Woverloaded-virtual" -/* A command that requires a Nix store. */ +/** + * A command that requires a \ref Store "Nix store". + */ struct StoreCommand : virtual Command { StoreCommand(); void run() override; ref getStore(); virtual ref createStore(); + /** + * Main entry point, with a `Store` provided + */ virtual void run(ref) = 0; private: std::shared_ptr _store; }; -/* A command that copies something between `--from` and `--to` - stores. */ +/** + * A command that copies something between `--from` and `--to` \ref + * Store stores. + */ struct CopyCommand : virtual StoreCommand { std::string srcUri, dstUri; @@ -60,6 +67,9 @@ struct CopyCommand : virtual StoreCommand ref getDstStore(); }; +/** + * A command that needs to evaluate Nix language expressions. + */ struct EvalCommand : virtual StoreCommand, MixEvalArgs { bool startReplOnEvalErrors = false; @@ -79,6 +89,10 @@ private: std::shared_ptr evalState; }; +/** + * A mixin class for commands that process flakes, adding a few standard + * flake-related options/flags. + */ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; @@ -87,6 +101,14 @@ struct MixFlakeOptions : virtual Args, EvalCommand MixFlakeOptions(); + /** + * The completion for some of these flags depends on the flake(s) in + * question. + * + * This method should be implemented to gather all flakerefs the + * command is operating with (presumably specified via some other + * arguments) so that the completions for these flags can use them. + */ virtual std::vector getFlakesForCompletion() { return {}; } @@ -112,15 +134,29 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions virtual Strings getDefaultFlakeAttrPathPrefixes(); + /** + * Complete an installable from the given prefix. + */ void completeInstallable(std::string_view prefix); }; +/** + * A mixin class for commands that need a read-only flag. + * + * What exactly is "read-only" is unspecified, but it will usually be + * the \ref Store "Nix store". + */ struct MixReadOnlyOption : virtual Args { MixReadOnlyOption(); }; -/* Like InstallablesCommand but the installables are not loaded */ +/** + * Like InstallablesCommand but the installables are not loaded. + * + * This is needed by `CmdRepl` which wants to load (and reload) the + * installables itself. + */ struct RawInstallablesCommand : virtual Args, SourceExprCommand { RawInstallablesCommand(); @@ -129,7 +165,7 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand void run(ref store) override; - // FIXME make const after CmdRepl's override is fixed up + // FIXME make const after `CmdRepl`'s override is fixed up virtual void applyDefaultInstallables(std::vector & rawInstallables); bool readFromStdIn = false; @@ -140,8 +176,11 @@ private: std::vector rawInstallables; }; -/* A command that operates on a list of "installables", which can be - store paths, attribute paths, Nix expressions, etc. */ + +/** + * A command that operates on a list of "installables", which can be + * store paths, attribute paths, Nix expressions, etc. + */ struct InstallablesCommand : RawInstallablesCommand { virtual void run(ref store, Installables && installables) = 0; @@ -149,7 +188,9 @@ struct InstallablesCommand : RawInstallablesCommand void run(ref store, std::vector && rawInstallables) override; }; -/* A command that operates on exactly one "installable" */ +/** + * A command that operates on exactly one "installable". + */ struct InstallableCommand : virtual Args, SourceExprCommand { InstallableCommand(); @@ -175,7 +216,12 @@ struct MixOperateOnOptions : virtual Args MixOperateOnOptions(); }; -/* A command that operates on zero or more store paths. */ +/** + * A command that operates on zero or more extant store paths. + * + * If the argument the user passes is a some sort of recipe for a path + * not yet built, it must be built first. + */ struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: @@ -207,7 +253,9 @@ struct StorePathsCommand : public BuiltPathsCommand void run(ref store, BuiltPaths && paths) override; }; -/* A command that operates on exactly one store path. */ +/** + * A command that operates on exactly one store path. + */ struct StorePathCommand : public StorePathsCommand { virtual void run(ref store, const StorePath & storePath) = 0; @@ -215,7 +263,9 @@ struct StorePathCommand : public StorePathsCommand void run(ref store, StorePaths && storePaths) override; }; -/* A helper class for registering commands globally. */ +/** + * A helper class for registering \ref Command commands globally. + */ struct RegisterCommand { typedef std::map, std::function()>> Commands; @@ -271,7 +321,11 @@ struct MixEnvironment : virtual Args { MixEnvironment(); - /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + /*** + * Modify global environ based on `ignoreEnvironment`, `keep`, and + * `unset`. It's expected that exec will be called before this class + * goes out of scope, otherwise `environ` will become invalid. + */ void setEnviron(); }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 1e5bac650..6457cceed 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -39,8 +39,21 @@ public: protected: + /** + * The largest `size_t` is used to indicate the "any" arity, for + * handlers/flags/arguments that accept an arbitrary number of + * arguments. + */ static const size_t ArityAny = std::numeric_limits::max(); + /** + * Arguments (flags/options and positional) have a "handler" which is + * caused when the argument is parsed. The handler has an arbitrary side + * effect, including possible affect further command-line parsing. + * + * There are many constructors in order to support many shorthand + * initializations, and this is used a lot. + */ struct Handler { std::function)> fun; @@ -110,7 +123,12 @@ protected: { } }; - /* Options. */ + /** + * Description of flags / options + * + * These are arguments like `-s` or `--long` that can (mostly) + * appear in any order. + */ struct Flag { typedef std::shared_ptr ptr; @@ -130,12 +148,30 @@ protected: static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); }; + /** + * Index of all registered "long" flag descriptions (flags like + * `--long`). + */ std::map longFlags; + + /** + * Index of all registered "short" flag descriptions (flags like + * `-s`). + */ std::map shortFlags; + /** + * Process a single flag and its arguments, pulling from an iterator + * of raw CLI args as needed. + */ virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); - /* Positional arguments. */ + /** + * Description of positional arguments + * + * These are arguments that do not start with a `-`, and for which + * the order does matter. + */ struct ExpectedArg { std::string label; @@ -144,8 +180,24 @@ protected: std::function completer; }; + /** + * Queue of expected positional argument forms. + * + * Positional arugment descriptions are inserted on the back. + * + * As positional arguments are passed, these are popped from the + * front, until there are hopefully none left as all args that were + * expected in fact were passed. + */ std::list expectedArgs; + /** + * Process some positional arugments + * + * @param finish: We have parsed everything else, and these are the only + * arguments left. Used because we accumulate some "pending args" we might + * have left over. + */ virtual bool processArgs(const Strings & args, bool finish); virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) @@ -204,6 +256,9 @@ public: friend class MultiCommand; + /** + * The parent command, used if this is a subcommand. + */ MultiCommand * parent = nullptr; private: From f7a36f981276a535962c1f5b0d02b00b9cd05298 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 12:52:59 -0400 Subject: [PATCH 009/100] Fix language tests a bit - Remove some stray saved error messages that didn't correspond to any test, because they were renamed in d11faa01b5b16773ba97bcb852b4123992829afc. - Need `--eval` in test failure test in order to get in "read-only" mode where we don't try to write to the store. (The other tests already do this.) - Need `--strict` so top-level attribute sets are still forced, like they are without `--eval`. --- tests/functional/lang.sh | 2 +- .../lang/eval-fail-antiquoted-path.err.exp | 1 - .../lang/eval-fail-bad-antiquote-1.err.exp | 10 ---------- .../lang/eval-fail-bad-antiquote-2.err.exp | 1 - .../lang/eval-fail-bad-antiquote-3.err.exp | 10 ---------- .../eval-fail-bad-string-interpolation-2.err.exp | 2 +- .../lang/eval-fail-dup-dynamic-attrs.err.exp | 12 +++++++++++- .../functional/lang/eval-fail-nonexist-path.err.exp | 2 +- tests/functional/lang/parse-fail-dup-attrs-6.err.exp | 1 - 9 files changed, 14 insertions(+), 27 deletions(-) delete mode 100644 tests/functional/lang/eval-fail-antiquoted-path.err.exp delete mode 100644 tests/functional/lang/eval-fail-bad-antiquote-1.err.exp delete mode 100644 tests/functional/lang/eval-fail-bad-antiquote-2.err.exp delete mode 100644 tests/functional/lang/eval-fail-bad-antiquote-3.err.exp delete mode 100644 tests/functional/lang/parse-fail-dup-attrs-6.err.exp diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index f4760eced..c3acef5ee 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -68,7 +68,7 @@ for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; i=$(basename "$i" .nix) if - expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \ + expectStderr 1 nix-instantiate --eval --strict --show-trace "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then diffAndAccept "$i" err err.exp diff --git a/tests/functional/lang/eval-fail-antiquoted-path.err.exp b/tests/functional/lang/eval-fail-antiquoted-path.err.exp deleted file mode 100644 index 425deba42..000000000 --- a/tests/functional/lang/eval-fail-antiquoted-path.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: getting attributes of path ‘PWD/lang/fnord’: No such file or directory diff --git a/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp deleted file mode 100644 index cf94f53bc..000000000 --- a/tests/functional/lang/eval-fail-bad-antiquote-1.err.exp +++ /dev/null @@ -1,10 +0,0 @@ -error: - … while evaluating a path segment - - at /pwd/lang/eval-fail-bad-antiquote-1.nix:1:2: - - 1| "${x: x}" - | ^ - 2| - - error: cannot coerce a function to a string diff --git a/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp deleted file mode 100644 index c8fe39d12..000000000 --- a/tests/functional/lang/eval-fail-bad-antiquote-2.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp b/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp deleted file mode 100644 index fbefbc826..000000000 --- a/tests/functional/lang/eval-fail-bad-antiquote-3.err.exp +++ /dev/null @@ -1,10 +0,0 @@ -error: - … while evaluating a path segment - - at /pwd/lang/eval-fail-bad-antiquote-3.nix:1:3: - - 1| ''${x: x}'' - | ^ - 2| - - error: cannot coerce a function to a string diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp index c8fe39d12..dea119ae8 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -1 +1 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' +error: getting status of '/pwd/lang/fnord': No such file or directory diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp index e01f8e6d0..c5fa67523 100644 --- a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -1,4 +1,14 @@ -error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 +error: + … while evaluating the attribute 'set' + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:3: + + 1| { + 2| set = { "${"" + "b"}" = 1; }; + | ^ + 3| set = { "${"b" + ""}" = 2; }; + + error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp index c8fe39d12..dea119ae8 100644 --- a/tests/functional/lang/eval-fail-nonexist-path.err.exp +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -1 +1 @@ -error: operation 'addToStoreFromDump' is not supported by store 'dummy' +error: getting status of '/pwd/lang/fnord': No such file or directory diff --git a/tests/functional/lang/parse-fail-dup-attrs-6.err.exp b/tests/functional/lang/parse-fail-dup-attrs-6.err.exp deleted file mode 100644 index 74823fc25..000000000 --- a/tests/functional/lang/parse-fail-dup-attrs-6.err.exp +++ /dev/null @@ -1 +0,0 @@ -error: attribute ‘services.ssh’ at (string):3:3 already defined at (string):2:3 From b3fd7db63f78ec736238016745338277703ad1d5 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Mon, 16 Oct 2023 13:00:49 -0400 Subject: [PATCH 010/100] Detect cycles in flake follows. This change results in an error thrown as opposed to segfaulting due to stack overflow. Fixes #9144 --- src/libexpr/flake/lockfile.cc | 22 ++++++++++++++--- tests/functional/flakes/follow-paths.sh | 32 ++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 3c202967a..68443211b 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -45,16 +45,26 @@ StorePath LockedNode::computeStorePath(Store & store) const return lockedRef.input.computeStorePath(store); } -std::shared_ptr LockFile::findInput(const InputPath & path) -{ + +static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { auto pos = root; + auto pathS = printInputPath(path); + auto found = std::find(visited.cbegin(), visited.cend(), pathS); + + if(found != visited.end()) { + std::vector cycle(found, visited.cend()); + cycle.push_back(pathS); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + } + visited.push_back(pathS); + for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { if (auto node = std::get_if<0>(&*i)) pos = *node; else if (auto follows = std::get_if<1>(&*i)) { - if (auto p = findInput(*follows)) + if (auto p = doFind(root, *follows, visited)) pos = ref(p); else return {}; @@ -66,6 +76,12 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return pos; } +std::shared_ptr LockFile::findInput(const InputPath & path) +{ + std::vector visited; + return doFind(root, path, visited); +} + LockFile::LockFile(const nlohmann::json & json, const Path & path) { auto version = json.value("version", 0); diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index dc97027ac..8573b5511 100644 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -167,7 +167,7 @@ nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override # # The message was # error: input 'B/D' follows a non-existent input 'B/C/D' -# +# # Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. flakeFollowsOverloadA="$TEST_ROOT/follows/overload/flakeA" flakeFollowsOverloadB="$TEST_ROOT/follows/overload/flakeA/flakeB" @@ -230,3 +230,33 @@ git -C "$flakeFollowsOverloadA" add flake.nix flakeB/flake.nix \ nix flake metadata "$flakeFollowsOverloadA" nix flake update "$flakeFollowsOverloadA" nix flake lock "$flakeFollowsOverloadA" + +# Now test follow cycle detection +# We construct the following follows graph: +# +# foo +# / ^ +# / \ +# v \ +# bar -> baz +# The message was +# error: follow cycle detected: [baz -> foo -> bar -> baz] +flakeFollowCycle="$TEST_ROOT/follows/followCycle" + +# Test following path flakerefs. +mkdir -p "$flakeFollowCycle" + +cat > $flakeFollowCycle/flake.nix <&1 && fail "nix flake lock should have failed." || true) +echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" From add066cc7bdd1357ccf471b6ab6e68c470e6f604 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 16 Oct 2023 19:32:47 +0100 Subject: [PATCH 011/100] Fix broken move --- src/libutil/config.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 38d406e8a..8672edaa8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -68,6 +68,7 @@ void AbstractConfig::warnUnknownSettings() void AbstractConfig::reapplyUnknownSettings() { auto unknownSettings2 = std::move(unknownSettings); + unknownSettings = {}; for (auto & s : unknownSettings2) set(s.first, s.second); } From d6066c90f87d9ba8ed1ddd8e2a3bd199f8498d6f Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Mon, 16 Oct 2023 15:47:28 -0400 Subject: [PATCH 012/100] Don't convert InputPaths to strings prematurely. --- src/libexpr/flake/lockfile.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 68443211b..f3ea9063f 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -2,8 +2,10 @@ #include "store-api.hh" #include "url-parts.hh" +#include #include +#include #include namespace nix::flake { @@ -46,18 +48,18 @@ StorePath LockedNode::computeStorePath(Store & store) const } -static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { +static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { auto pos = root; - auto pathS = printInputPath(path); - auto found = std::find(visited.cbegin(), visited.cend(), pathS); + auto found = std::find(visited.cbegin(), visited.cend(), path); if(found != visited.end()) { - std::vector cycle(found, visited.cend()); - cycle.push_back(pathS); + std::vector cycle; + std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath); + cycle.push_back(printInputPath(path)); throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); } - visited.push_back(pathS); + visited.push_back(path); for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { @@ -78,7 +80,7 @@ static std::shared_ptr doFind(const ref& root, const InputPath & pat std::shared_ptr LockFile::findInput(const InputPath & path) { - std::vector visited; + std::vector visited; return doFind(root, path, visited); } From abf7df2b376a1cdbc00527bc662344fd9232ce05 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 16 Oct 2023 21:47:33 +0100 Subject: [PATCH 013/100] Fix moves that accidentally copy anyway --- src/libexpr/eval.cc | 4 ++-- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/outputs-spec.cc | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b18eb5266..5280614a1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -539,7 +539,7 @@ EvalState::EvalState( auto r = resolveSearchPathPath(i.path); if (!r) continue; - auto path = *std::move(r); + auto path = std::move(*r); if (store->isInStore(path)) { try { @@ -1035,7 +1035,7 @@ std::string EvalState::mkOutputStringRaw( /* In practice, this is testing for the case of CA derivations, or dynamic derivations. */ return optStaticOutputPath - ? store->printStorePath(*std::move(optStaticOutputPath)) + ? store->printStorePath(std::move(*optStaticOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render(); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index dc4d91079..360c6b70b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -561,7 +561,7 @@ void DerivationGoal::inputsRealised() attempt = fullDrv.tryResolve(worker.store); } assert(attempt); - Derivation drvResolved { *std::move(attempt) }; + Derivation drvResolved { std::move(*attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 817b20b2f..6a02f5ad4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2521,7 +2521,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() ValidPathInfo newInfo0 { worker.store, outputPathName(drv->name, outputName), - *std::move(optCA), + std::move(*optCA), Hash::dummy, }; if (*scratchPath != newInfo0.path) { diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index d943bc111..21c069223 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -63,7 +63,7 @@ std::optional> ExtendedOutputsS auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); if (!specOpt) return std::nullopt; - return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } }; + return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { std::move(*specOpt) } }; } From 54b350d517932ef11de4b3d638d67ffa5cec1634 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Mon, 16 Oct 2023 21:48:02 +0100 Subject: [PATCH 014/100] Drop some moves that would happen anyway but forbid NRVO where appicable --- src/libexpr/eval.cc | 2 +- src/libexpr/paths.cc | 2 +- src/libstore/content-address.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5280614a1..511a5f434 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2290,7 +2290,7 @@ BackedStringView EvalState::coerceToString( && (!v2->isList() || v2->listSize() != 0)) result += " "; } - return std::move(result); + return result; } } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 1d690b722..b6a696f47 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -4,7 +4,7 @@ namespace nix { SourcePath EvalState::rootPath(CanonPath path) { - return std::move(path); + return path; } } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e290a8d38..ae91b859b 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -83,7 +83,7 @@ static std::pair parseContentAddressMethodPrefix if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); + return hashType; }; // Switch on prefix From 564922939404db110bcdfa71319470533df54cb4 Mon Sep 17 00:00:00 2001 From: Artturin Date: Mon, 11 Sep 2023 00:19:21 +0300 Subject: [PATCH 015/100] Bindmount files instead of hardlinking or copying to chroot https://github.com/nixos/nix/commit/16591eb3cccf86da8cd3f20c56e2dd847576ff5e#diff-19f999107b609d37cfb22c58e7f0bc1cf76edf1180e238dd6389e03cc279b604 (2013) added support for files to doBind This is work towards allowing users to change the location of chrootRootDir, to, for example, a tmpfs. inspired by trofi on matrix > It looks like build sandbox created by nix-daemon runs on the same filesystem, as /nix/store including things like /tmp which makes all small temporary files hit the disk. Is it intentional? If it is is there an easy way to redirect chroot's root to be tmpfs? dirsInChroot -> pathsInChroot --- src/libstore/build/local-derivation-goal.cc | 64 +++++++-------------- src/libstore/build/local-derivation-goal.hh | 4 +- 2 files changed, 23 insertions(+), 45 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64b55ca6a..a3ebfc681 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -581,7 +581,7 @@ void LocalDerivationGoal::startBuilder() /* Allow a user-configurable set of directories from the host file system. */ - dirsInChroot.clear(); + pathsInChroot.clear(); for (auto i : settings.sandboxPaths.get()) { if (i.empty()) continue; @@ -592,19 +592,19 @@ void LocalDerivationGoal::startBuilder() } size_t p = i.find('='); if (p == std::string::npos) - dirsInChroot[i] = {i, optional}; + pathsInChroot[i] = {i, optional}; else - dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) { throw Error("`sandbox-build-dir` must not contain the storeDir"); } - dirsInChroot[tmpDirInSandbox] = tmpDir; + pathsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ StorePathSet closure; - for (auto & i : dirsInChroot) + for (auto & i : pathsInChroot) try { if (worker.store.isInStore(i.second.source)) worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); @@ -615,7 +615,7 @@ void LocalDerivationGoal::startBuilder() } for (auto & i : closure) { auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); + pathsInChroot.insert_or_assign(p, p); } PathSet allowedPaths = settings.allowedImpureHostPrefixes; @@ -643,7 +643,7 @@ void LocalDerivationGoal::startBuilder() /* Allow files in __impureHostDeps to be missing; e.g. macOS 11+ has no /usr/lib/libSystem*.dylib */ - dirsInChroot[i] = {i, true}; + pathsInChroot[i] = {i, true}; } #if __linux__ @@ -711,15 +711,12 @@ void LocalDerivationGoal::startBuilder() for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); + pathsInChroot.insert_or_assign(p, r); } /* If we're repairing, checking or rebuilding part of a multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot + rebuilding a path that is in settings.sandbox-paths (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputsAndOptPaths(worker.store)) { @@ -729,7 +726,7 @@ void LocalDerivationGoal::startBuilder() is already in the sandbox, so we don't need to worry about removing it. */ if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + pathsInChroot.erase(worker.store.printStorePath(*i.second.second)); } if (cgroup) { @@ -787,9 +784,9 @@ void LocalDerivationGoal::startBuilder() } else { auto p = line.find('='); if (p == std::string::npos) - dirsInChroot[line] = line; + pathsInChroot[line] = line; else - dirsInChroot[line.substr(0, p)] = line.substr(p + 1); + pathsInChroot[line.substr(0, p)] = line.substr(p + 1); } } } @@ -1779,7 +1776,7 @@ void LocalDerivationGoal::runChild() /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + if (pathsInChroot.find("/dev") == pathsInChroot.end()) { createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); @@ -1814,34 +1811,15 @@ void LocalDerivationGoal::runChild() ss.push_back(path); if (settings.caFile != "") - dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); } - for (auto & i : ss) dirsInChroot.emplace(i, i); + for (auto & i : ss) pathsInChroot.emplace(i, i); /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility #if HAVE_EMBEDDED_SANDBOX_SHELL @@ -1882,7 +1860,7 @@ void LocalDerivationGoal::runChild() if /dev/ptx/ptmx exists). */ if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) + && !pathsInChroot.count("/dev/pts")) { if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { @@ -2017,7 +1995,7 @@ void LocalDerivationGoal::runChild() /* We build the ancestry before adding all inputPaths to the store because we know they'll all have the same parents (the store), and there might be lots of inputs. This isn't particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { Path cur = i.first; while (cur.compare("/") != 0) { cur = dirOf(cur); @@ -2025,7 +2003,7 @@ void LocalDerivationGoal::runChild() } } - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost path component this time, since it's typically /nix/store and we care about that. */ Path cur = worker.store.storeDir; while (cur.compare("/") != 0) { @@ -2036,7 +2014,7 @@ void LocalDerivationGoal::runChild() /* Add all our input paths to the chroot */ for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; + pathsInChroot[p] = p; } /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ @@ -2067,7 +2045,7 @@ void LocalDerivationGoal::runChild() without file-write* allowed, access() incorrectly returns EPERM */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.first != i.second.source) throw Error( "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 0a05081c7..05c2c3a56 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -86,8 +86,8 @@ struct LocalDerivationGoal : public DerivationGoal : source(source), optional(optional) { } }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; + typedef map PathsInChroot; // maps target path to source path + PathsInChroot pathsInChroot; typedef map Environment; Environment env; From 630c2545d13cd8f6de4d678f19e89dfca375f62e Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 14 Sep 2023 04:18:18 +0300 Subject: [PATCH 016/100] remove linkOrCopy and use bindmounts for files in addDependency --- src/libstore/build/local-derivation-goal.cc | 62 +++++++-------------- 1 file changed, 21 insertions(+), 41 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a3ebfc681..204acab11 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -387,26 +387,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() } -#if __linux__ -static void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - or fail with EXDEV in OpenAFS - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM && errno != EXDEV) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} -#endif - - void LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) @@ -1559,34 +1539,34 @@ void LocalDerivationGoal::addDependency(const StorePath & path) auto st = lstat(source); - if (S_ISDIR(st.st_mode)) { + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); - if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) - throw SysError("entering sandbox user namespace"); - - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); + if (S_ISDIR(st.st_mode)) createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError("bind mount from '%s' to '%s' failed", source, target); - _exit(0); - })); + _exit(0); + })); - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", From 11e47e7dfbca2f330b693665a4ee38302d1f6ad2 Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 14 Sep 2023 04:36:25 +0300 Subject: [PATCH 017/100] factor out doBind from runChild --- src/libstore/build/local-derivation-goal.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 204acab11..9592df43a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -386,6 +386,26 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() cleanupPostOutputsRegisteredModeCheck(); } +#if __linux__ +static void doBind(const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); +}; +#endif void LocalDerivationGoal::startBuilder() { From b8dfa3d53bda6793fe2c2566ae6e63e7c83718d8 Mon Sep 17 00:00:00 2001 From: Artturin Date: Thu, 14 Sep 2023 04:36:40 +0300 Subject: [PATCH 018/100] use doBind in addDependency --- src/libstore/build/local-derivation-goal.cc | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9592df43a..fd12b66b9 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1552,13 +1552,12 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); if (pathExists(target)) + // There is a similar debug message in doBind, so only run it in this block to not have double messages. + debug("bind-mounting %s -> %s", target, source); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); - auto st = lstat(source); - /* Bind-mount the path into the sandbox. This requires entering its mount namespace, which is not possible in multithreaded programs. So we do this in a @@ -1571,15 +1570,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) if (setns(sandboxMountNamespace.get(), 0) == -1) throw SysError("entering sandbox mount namespace"); - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); + doBind(source, target); _exit(0); })); From dcc5f801f4fd88e08182b13304cdfa1b3aed8ece Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Tue, 17 Oct 2023 09:39:59 +0530 Subject: [PATCH 019/100] Store info command help updates --- src/nix/{ping-store.cc => store-info.cc} | 8 ++++---- src/nix/{ping-store.md => store-info.md} | 6 +++--- tests/functional/{store-ping.sh => store-info.sh} | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) rename src/nix/{ping-store.cc => store-info.cc} (91%) rename src/nix/{ping-store.md => store-info.md} (82%) rename tests/functional/{store-ping.sh => store-info.sh} (69%) diff --git a/src/nix/ping-store.cc b/src/nix/store-info.cc similarity index 91% rename from src/nix/ping-store.cc rename to src/nix/store-info.cc index 247d9c6bb..a7c595761 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/store-info.cc @@ -17,7 +17,7 @@ struct CmdPingStore : StoreCommand, MixJSON std::string doc() override { return - #include "ping-store.md" + #include "store-info.md" ; } @@ -50,11 +50,11 @@ struct CmdInfoStore : CmdPingStore { void run(nix::ref store) override { - warn("'nix store info' is a deprecated alias for 'nix store ping'"); + warn("'nix store ping' is a deprecated alias for 'nix store info'"); CmdPingStore::run(store); } }; -static auto rCmdPingStore = registerCommand2({"store", "ping"}); -static auto rCmdInfoStore = registerCommand2({"store", "info"}); +static auto rCmdPingStore = registerCommand2({"store", "info"}); +static auto rCmdInfoStore = registerCommand2({"store", "ping"}); diff --git a/src/nix/ping-store.md b/src/nix/store-info.md similarity index 82% rename from src/nix/ping-store.md rename to src/nix/store-info.md index 8c846791b..f86efd722 100644 --- a/src/nix/ping-store.md +++ b/src/nix/store-info.md @@ -5,19 +5,19 @@ R""( * Test whether connecting to a remote Nix store via SSH works: ```console - # nix store ping --store ssh://mac1 + # nix store info --store ssh://mac1 ``` * Test whether a URL is a valid binary cache: ```console - # nix store ping --store https://cache.nixos.org + # nix store info --store https://cache.nixos.org ``` * Test whether the Nix daemon is up and running: ```console - # nix store ping --store daemon + # nix store info --store daemon ``` # Description diff --git a/tests/functional/store-ping.sh b/tests/functional/store-info.sh similarity index 69% rename from tests/functional/store-ping.sh rename to tests/functional/store-info.sh index 9846c7d3d..c002e50be 100644 --- a/tests/functional/store-ping.sh +++ b/tests/functional/store-info.sh @@ -1,7 +1,7 @@ source common.sh -STORE_INFO=$(nix store ping 2>&1) -STORE_INFO_JSON=$(nix store ping --json) +STORE_INFO=$(nix store info 2>&1) +STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" @@ -11,7 +11,7 @@ if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi -expect 127 NIX_REMOTE=unix:$PWD/store nix store ping || \ - fail "nix store ping on a non-existent store should fail" +expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ + fail "nix store info on a non-existent store should fail" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] From f62b5500ff96cc46e610da13dcf72c134935642d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Oct 2023 14:52:34 +0200 Subject: [PATCH 020/100] fetchTree: Require the flakes experimental feature for the URL syntax --- src/libexpr/primops/fetchTree.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 0335e5f36..3431ff013 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -148,6 +148,11 @@ static void fetchTree( attrs.emplace("url", fixGitURL(url)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"), + .errPos = state.positions[pos] + })); input = fetchers::Input::fromURL(url); } } @@ -180,6 +185,10 @@ static RegisterPrimOp primop_fetchTree({ *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. + > **Note** + > + > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + Here are some examples of how to use `fetchTree`: - Fetch a GitHub repository using the attribute set representation: From 3470cd68c46334d9d2c55e890f7fe50fa9ebe35c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Oct 2023 14:57:29 +0200 Subject: [PATCH 021/100] Mark some fetchers as experimental --- src/libfetchers/github.cc | 5 +++++ src/libfetchers/path.cc | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 291f457f0..6c5c792ec 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -223,6 +223,11 @@ struct GitArchiveInputScheme : InputScheme return {result.tree.storePath, input}; } + + std::optional experimentalFeature() override + { + return Xp::Flakes; + } }; struct GitHubInputScheme : GitArchiveInputScheme diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 01f1be978..092975f5d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -125,6 +125,11 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } + + std::optional experimentalFeature() override + { + return Xp::Flakes; + } }; static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); From ff684260950cd31efffbe9804494d42bc226df59 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 17 Oct 2023 11:15:36 -0400 Subject: [PATCH 022/100] Name the protocol version types This makes the code clearer, and will help us replace them with proper structs and get rid of the macros later. --- src/libstore/daemon.cc | 10 +++++----- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/remote-store-connection.hh | 2 +- src/libstore/serve-protocol.hh | 7 +++++++ src/libstore/worker-protocol.hh | 7 +++++++ src/nix-store/nix-store.cc | 2 +- 6 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8cbf6f044..fbff64e89 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -45,9 +45,9 @@ struct TunnelLogger : public Logger Sync state_; - unsigned int clientVersion; + WorkerProto::Version clientVersion; - TunnelLogger(FdSink & to, unsigned int clientVersion) + TunnelLogger(FdSink & to, WorkerProto::Version clientVersion) : to(to), clientVersion(clientVersion) { } void enqueueMsg(const std::string & s) @@ -261,7 +261,7 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn) +static std::vector readDerivedPaths(Store & store, WorkerProto::Version clientVersion, WorkerProto::ReadConn conn) { std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { @@ -274,7 +274,7 @@ static std::vector readDerivedPaths(Store & store, unsigned int cli } static void performOp(TunnelLogger * logger, ref store, - TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, + TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) { WorkerProto::ReadConn rconn { .from = from }; @@ -1017,7 +1017,7 @@ void processConnection( if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); to << WORKER_MAGIC_2 << PROTOCOL_VERSION; to.flush(); - unsigned int clientVersion = readInt(from); + WorkerProto::Version clientVersion = readInt(from); if (clientVersion < 0x10a) throw Error("the Nix client version is too old"); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 703ded0b2..efced20d0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -45,7 +45,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor std::unique_ptr sshConn; FdSink to; FdSource from; - int remoteVersion; + ServeProto::Version remoteVersion; bool good = true; /** diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index ce4740a9c..8738e4d95 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -30,7 +30,7 @@ struct RemoteStore::Connection * sides support. (If the maximum doesn't exist, we would fail to * establish a connection and produce a value of this type.) */ - unsigned int daemonVersion; + WorkerProto::Version daemonVersion; /** * Whether the remote side trusts us or not. diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index e2345d450..dddb7c54d 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -30,6 +30,13 @@ struct ServeProto */ enum struct Command : uint64_t; + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c84060103..e9fc8d957 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -47,6 +47,13 @@ struct WorkerProto */ enum struct Op : uint64_t; + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9b6c80a75..ea53e49b0 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -818,7 +818,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; out.flush(); - unsigned int clientVersion = readInt(in); + ServeProto::Version clientVersion = readInt(in); ServeProto::ReadConn rconn { .from = in }; ServeProto::WriteConn wconn { .to = out }; From e36c9175f49c0fac4f0511d4cae5ab991fd9cb7d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 04:40:49 +0000 Subject: [PATCH 023/100] Add protocol versions to `{Worker,Serve}Proto::*Conn` This will allow us to factor out logic, which is currently scattered inline, into several reusable instances The tests are also updated to support versioning. Currently all Worker and Serve protocol tests are using the minimum version, since no version-specific serialisers have been created yet. But in subsequent commits when that changes, we will test individual versions to ensure complete coverage. --- src/libstore/daemon.cc | 15 ++++- src/libstore/legacy-ssh-store.cc | 2 + src/libstore/path-info.cc | 10 +++- src/libstore/remote-store-connection.hh | 2 + src/libstore/serve-protocol.hh | 8 +-- src/libstore/tests/common-protocol.cc | 73 +++++++++++++++++++++---- src/libstore/tests/protocol.hh | 45 ++++++++------- src/libstore/tests/serve-protocol.cc | 38 +++++++++---- src/libstore/tests/worker-protocol.cc | 49 ++++++++++++----- src/libstore/worker-protocol.hh | 8 +-- src/nix-store/nix-store.cc | 10 +++- 11 files changed, 185 insertions(+), 75 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index fbff64e89..d61e97a64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -277,8 +277,14 @@ static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) { - WorkerProto::ReadConn rconn { .from = from }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::ReadConn rconn { + .from = from, + .version = clientVersion, + }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; switch (op) { @@ -1052,7 +1058,10 @@ void processConnection( auto temp = trusted ? store->isTrustedClient() : std::optional { NotTrusted }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; WorkerProto::write(*store, wconn, temp); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index efced20d0..9143be72e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -60,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { return ServeProto::ReadConn { .from = from, + .version = remoteVersion, }; } @@ -75,6 +76,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { return ServeProto::WriteConn { .to = to, + .version = remoteVersion, }; } }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ccb57104f..a9231ce8d 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -141,7 +141,10 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = source }); + WorkerProto::ReadConn { + .from = source, + .version = format, + }); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -163,7 +166,10 @@ void ValidPathInfo::write( sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); WorkerProto::write(store, - WorkerProto::WriteConn { .to = sink }, + WorkerProto::WriteConn { + .to = sink, + .version = format, + }, references); sink << registrationTime << narSize; if (format >= 16) { diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index 8738e4d95..e4a9cacb9 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -70,6 +70,7 @@ struct RemoteStore::Connection { return WorkerProto::ReadConn { .from = from, + .version = daemonVersion, }; } @@ -85,6 +86,7 @@ struct RemoteStore::Connection { return WorkerProto::WriteConn { .to = to, + .version = daemonVersion, }; } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index dddb7c54d..a627c6ad6 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -40,23 +40,19 @@ struct ServeProto /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. - * - * This currently is just a `Source &`, but more fields will be added - * later. */ struct ReadConn { Source & from; + Version version; }; /** * A unidirectional write connection, to be used by the write half of the * canonical serializers below. - * - * This currently is just a `Sink &`, but more fields will be added - * later. */ struct WriteConn { Sink & to; + Version version; }; /** diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc index de57211f0..61c2cb70c 100644 --- a/src/libstore/tests/common-protocol.cc +++ b/src/libstore/tests/common-protocol.cc @@ -13,10 +13,71 @@ namespace nix { const char commonProtoDir[] = "common-protocol"; -using CommonProtoTest = ProtoTest; +class CommonProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readTest(PathView testStem, T value) + { + if (testAccept()) + { + GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + } + else + { + auto expected = readFile(goldenMaster(testStem)); + + T got = ({ + StringSource from { expected }; + CommonProto::Serialise::read( + *store, + CommonProto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, value); + } + } + + /** + * Golden test for `T` write + */ + template + void writeTest(PathView testStem, const T & value) + { + auto file = goldenMaster(testStem); + + StringSink to; + CommonProto::write( + *store, + CommonProto::WriteConn { .to = to }, + value); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile(file, to.s); + GTEST_SKIP() << "Updating golden master"; + } + else + { + auto expected = readFile(file); + ASSERT_EQ(to.s, expected); + } + } +}; + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME ## _read) { \ + readTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME ## _write) { \ + writeTest(STEM, VALUE); \ + } CHARACTERIZATION_TEST( - CommonProtoTest, string, "string", (std::tuple { @@ -28,7 +89,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, storePath, "store-path", (std::tuple { @@ -37,7 +97,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, contentAddress, "content-address", (std::tuple { @@ -56,7 +115,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, drvOutput, "drv-output", (std::tuple { @@ -71,7 +129,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, realisation, "realisation", (std::tuple { @@ -103,7 +160,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, vector, "vector", (std::tuple, std::vector, std::vector, std::vector>> { @@ -114,7 +170,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, set, "set", (std::tuple, std::set, std::set, std::set>> { @@ -125,7 +180,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, optionalStorePath, "optional-store-path", (std::tuple, std::optional> { @@ -136,7 +190,6 @@ CHARACTERIZATION_TEST( })) CHARACTERIZATION_TEST( - CommonProtoTest, optionalContentAddress, "optional-content-address", (std::tuple, std::optional> { diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 86a900757..496915745 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -9,26 +9,23 @@ namespace nix { template class ProtoTest : public LibStoreTest { - /** - * Read this as simply `using S = Inner::Serialise;`. - * - * See `LengthPrefixedProtoHelper::S` for the same trick, and its - * rationale. - */ - template using S = typename Proto::template Serialise; - -public: +protected: Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; Path goldenMaster(std::string_view testStem) { return unitTestData + "/" + testStem + ".bin"; } +}; +template +class VersionedProtoTest : public ProtoTest +{ +public: /** * Golden test for `T` reading */ template - void readTest(PathView testStem, T value) + void readTest(PathView testStem, typename Proto::Version version, T value) { if (testAccept()) { @@ -36,13 +33,16 @@ public: } else { - auto expected = readFile(goldenMaster(testStem)); + auto expected = readFile(ProtoTest::goldenMaster(testStem)); T got = ({ StringSource from { expected }; - S::read( - *store, - typename Proto::ReadConn { .from = from }); + Proto::template Serialise::read( + *LibStoreTest::store, + typename Proto::ReadConn { + .from = from, + .version = version, + }); }); ASSERT_EQ(got, value); @@ -53,14 +53,17 @@ public: * Golden test for `T` write */ template - void writeTest(PathView testStem, const T & value) + void writeTest(PathView testStem, typename Proto::Version version, const T & value) { - auto file = goldenMaster(testStem); + auto file = ProtoTest::goldenMaster(testStem); StringSink to; Proto::write( - *store, - typename Proto::WriteConn { .to = to }, + *LibStoreTest::store, + typename Proto::WriteConn { + .to = to, + .version = version, + }, value); if (testAccept()) @@ -77,12 +80,12 @@ public: } }; -#define CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VALUE) \ +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ TEST_F(FIXTURE, NAME ## _read) { \ - readTest(STEM, VALUE); \ + readTest(STEM, VERSION, VALUE); \ } \ TEST_F(FIXTURE, NAME ## _write) { \ - writeTest(STEM, VALUE); \ + writeTest(STEM, VERSION, VALUE); \ } } diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc index 5d7df56cf..ba798ab1c 100644 --- a/src/libstore/tests/serve-protocol.cc +++ b/src/libstore/tests/serve-protocol.cc @@ -11,14 +11,22 @@ namespace nix { -const char commonProtoDir[] = "serve-protocol"; +const char serveProtoDir[] = "serve-protocol"; -using ServeProtoTest = ProtoTest; +struct ServeProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + ServeProto::Version defaultVersion = 1 << 8 | 0; +}; -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, string, "string", + defaultVersion, (std::tuple { "", "hi", @@ -27,19 +35,21 @@ CHARACTERIZATION_TEST( "oh no \0\0\0 what was that!", })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, storePath, "store-path", + defaultVersion, (std::tuple { StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, contentAddress, "content-address", + defaultVersion, (std::tuple { ContentAddress { .method = TextIngestionMethod {}, @@ -55,10 +65,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, drvOutput, "drv-output", + defaultVersion, (std::tuple { { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), @@ -70,10 +81,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, realisation, "realisation", + defaultVersion, (std::tuple { Realisation { .id = DrvOutput { @@ -102,10 +114,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, "vector", + defaultVersion, (std::tuple, std::vector, std::vector, std::vector>> { { }, { "" }, @@ -113,10 +126,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, set, "set", + defaultVersion, (std::tuple, std::set, std::set, std::set>> { { }, { "" }, @@ -124,10 +138,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, optionalStorePath, "optional-store-path", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { @@ -135,10 +150,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, optionalContentAddress, "optional-content-address", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 59d7ff96d..63fecce96 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -14,12 +14,21 @@ namespace nix { const char workerProtoDir[] = "worker-protocol"; -using WorkerProtoTest = ProtoTest; +struct WorkerProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + WorkerProto::Version defaultVersion = 1 << 8 | 0; +}; -CHARACTERIZATION_TEST( + +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, string, "string", + defaultVersion, (std::tuple { "", "hi", @@ -28,19 +37,21 @@ CHARACTERIZATION_TEST( "oh no \0\0\0 what was that!", })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, storePath, "store-path", + defaultVersion, (std::tuple { StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, contentAddress, "content-address", + defaultVersion, (std::tuple { ContentAddress { .method = TextIngestionMethod {}, @@ -56,10 +67,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, derivedPath, "derived-path", + defaultVersion, (std::tuple { DerivedPath::Opaque { .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, @@ -72,10 +84,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, drvOutput, "drv-output", + defaultVersion, (std::tuple { { .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), @@ -87,10 +100,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, realisation, "realisation", + defaultVersion, (std::tuple { Realisation { .id = DrvOutput { @@ -119,10 +133,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult, "build-result", + defaultVersion, ({ using namespace std::literals::chrono_literals; std::tuple t { @@ -177,10 +192,11 @@ CHARACTERIZATION_TEST( t; })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, keyedBuildResult, "keyed-build-result", + defaultVersion, ({ using namespace std::literals::chrono_literals; std::tuple t { @@ -213,20 +229,22 @@ CHARACTERIZATION_TEST( t; })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalTrustedFlag, "optional-trusted-flag", + defaultVersion, (std::tuple, std::optional, std::optional> { std::nullopt, std::optional { Trusted }, std::optional { NotTrusted }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, vector, "vector", + defaultVersion, (std::tuple, std::vector, std::vector, std::vector>> { { }, { "" }, @@ -234,10 +252,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, set, "set", + defaultVersion, (std::tuple, std::set, std::set, std::set>> { { }, { "" }, @@ -245,10 +264,11 @@ CHARACTERIZATION_TEST( { {}, { "" }, { "", "1", "2" } }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalStorePath, "optional-store-path", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { @@ -256,10 +276,11 @@ CHARACTERIZATION_TEST( }, })) -CHARACTERIZATION_TEST( +VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalContentAddress, "optional-content-address", + defaultVersion, (std::tuple, std::optional> { std::nullopt, std::optional { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index e9fc8d957..2d55d926a 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -57,23 +57,19 @@ struct WorkerProto /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. - * - * This currently is just a `Source &`, but more fields will be added - * later. */ struct ReadConn { Source & from; + Version version; }; /** * A unidirectional write connection, to be used by the write half of the * canonical serializers below. - * - * This currently is just a `Sink &`, but more fields will be added - * later. */ struct WriteConn { Sink & to; + Version version; }; /** diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index ea53e49b0..11d2e86e3 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -820,8 +820,14 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); ServeProto::Version clientVersion = readInt(in); - ServeProto::ReadConn rconn { .from = in }; - ServeProto::WriteConn wconn { .to = out }; + ServeProto::ReadConn rconn { + .from = in, + .version = clientVersion, + }; + ServeProto::WriteConn wconn { + .to = out, + .version = clientVersion, + }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're From a0f071f1d3b05140fa1c97de5a7612d1b938ffbf Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Wed, 18 Oct 2023 00:12:10 +0530 Subject: [PATCH 024/100] store info sh renamed --- tests/functional/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 643351163..3679349f8 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -113,7 +113,7 @@ nix_tests = \ pass-as-file.sh \ nix-profile.sh \ suggestions.sh \ - store-ping.sh \ + store-info.sh \ fetchClosure.sh \ completions.sh \ flakes/show.sh \ From 891dfb435943f68f165a0c5921f0af8e9e7362e8 Mon Sep 17 00:00:00 2001 From: vicky1999 Date: Wed, 18 Oct 2023 00:14:11 +0530 Subject: [PATCH 025/100] updated store ping to store info in files --- doc/manual/src/advanced-topics/distributed-builds.md | 4 ++-- doc/manual/src/release-notes/rl-2.15.md | 2 +- doc/manual/src/release-notes/rl-2.7.md | 2 +- tests/functional/legacy-ssh-store.sh | 4 ++-- tests/functional/local-store.sh | 4 ++-- tests/functional/nix-copy-ssh-ng.sh | 2 +- tests/functional/remote-store.sh | 8 ++++---- tests/installer/default.nix | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 73a113d35..507c5ecb7 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can test whether connecting to the remote Nix instance works, e.g. ```console -$ nix store ping --store ssh://mac +$ nix store info --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store info --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..4faf0b143 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -44,7 +44,7 @@ (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. - `nix store ping` and `nix doctor` now display this information. + `nix store info` and `nix doctor` now display this information. * The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/src/release-notes/rl-2.7.md index 2f3879422..dd649e166 100644 --- a/doc/manual/src/release-notes/rl-2.7.md +++ b/doc/manual/src/release-notes/rl-2.7.md @@ -24,7 +24,7 @@ [repository](https://github.com/NixOS/bundlers) has various bundlers implemented. -* `nix store ping` now reports the version of the remote Nix daemon. +* `nix store info` now reports the version of the remote Nix daemon. * `nix flake {init,new}` now display information about which files have been created. diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh index 71b716b84..894efccd4 100644 --- a/tests/functional/legacy-ssh-store.sh +++ b/tests/functional/legacy-ssh-store.sh @@ -1,4 +1,4 @@ source common.sh -# Check that store ping trusted doesn't yet work with ssh:// -nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' +# Check that store info trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e 'has("trusted") | not' diff --git a/tests/functional/local-store.sh b/tests/functional/local-store.sh index 89502f864..f7c8eb3f1 100644 --- a/tests/functional/local-store.sh +++ b/tests/functional/local-store.sh @@ -18,5 +18,5 @@ PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] -# Ensure store ping trusted works with local store -nix --store ./x store ping --json | jq -e '.trusted' +# Ensure store info trusted works with local store +nix --store ./x store info --json | jq -e '.trusted' diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 45e53c9c0..463b5e0c4 100644 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -9,7 +9,7 @@ rm -rf "$remoteRoot" outPath=$(nix-build --no-out-link dependencies.nix) -nix store ping --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" +nix store info --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" # Regression test for https://github.com/NixOS/nix/issues/6253 nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index 50e6f24b9..5c7bfde46 100644 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -5,17 +5,17 @@ clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor -# Ensure that store ping trusted works with ssh-ng:// -nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' +# Ensure that store info trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e '.trusted' startDaemon if isDaemonNewer "2.15pre0"; then # Ensure that ping works trusted with new daemon - nix store ping --json | jq -e '.trusted' + nix store info --json | jq -e '.trusted' else # And the the field is absent with the old daemon - nix store ping --json | jq -e 'has("trusted") | not' + nix store info --json | jq -e 'has("trusted") | not' fi # Test import-from-derivation through the daemon. diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 49cfd2bcc..238c6ac8e 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -213,7 +213,7 @@ let source /etc/bashrc || true nix-env --version - nix --extra-experimental-features nix-command store ping + nix --extra-experimental-features nix-command store info out=\$(nix-build --no-substitute -E 'derivation { name = "foo"; system = "x86_64-linux"; builder = "/bin/sh"; args = ["-c" "echo foobar > \$out"]; }') [[ \$(cat \$out) = foobar ]] From ea38605d116e1c8db55a8a8cb7400724b931458d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Oct 2023 15:32:31 +0200 Subject: [PATCH 026/100] Introduce FSInputAccessor and use it Backported from the lazy-trees branch. Note that this doesn't yet use the access control features of FSInputAccessor. --- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval.cc | 35 +++-- src/libexpr/eval.hh | 3 + src/libexpr/flake/flake.cc | 6 +- src/libexpr/nixexpr.hh | 7 +- src/libexpr/parser.y | 9 +- src/libexpr/paths.cc | 3 +- src/libexpr/primops.cc | 18 +-- src/libexpr/tests/json.cc | 2 +- src/libexpr/tests/libexpr.hh | 15 ++- src/libexpr/value.hh | 17 ++- src/libfetchers/fs-input-accessor.cc | 141 ++++++++++++++++++++ src/libfetchers/fs-input-accessor.hh | 33 +++++ src/libfetchers/input-accessor.cc | 189 ++++++++++++++++++++------- src/libfetchers/input-accessor.hh | 105 ++++++++++++--- src/libstore/store-api.cc | 14 +- src/libstore/store-api.hh | 15 ++- src/nix/main.cc | 14 +- 18 files changed, 502 insertions(+), 126 deletions(-) create mode 100644 src/libfetchers/fs-input-accessor.cc create mode 100644 src/libfetchers/fs-input-accessor.hh diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index ab654c1b0..d12345710 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -132,7 +132,7 @@ std::pair findPackageFilename(EvalState & state, Value & v if (colon == std::string::npos) fail(); std::string filename(fn, 0, colon); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); - return {CanonPath(fn.substr(0, colon)), lineno}; + return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); abort(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 511a5f434..87791fa52 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -12,6 +12,7 @@ #include "function-trace.hh" #include "profiles.hh" #include "print.hh" +#include "fs-input-accessor.hh" #include #include @@ -503,6 +504,18 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) + , rootFS( + makeFSInputAccessor( + CanonPath::root, + evalSettings.restrictEval || evalSettings.pureEval + ? std::optional>(std::set()) + : std::nullopt, + [](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = evalSettings.pureEval + ? "in pure evaluation mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); + })) , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) , store(store) , buildStore(buildStore ? buildStore : store) @@ -518,6 +531,8 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { + rootFS->allowPath(CanonPath::root); // FIXME + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); @@ -599,7 +614,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) */ Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath); + if (hasPrefix(abspath, corepkgsPrefix)) return rootPath(CanonPath(abspath)); for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { @@ -617,7 +632,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) /* Resolve symlinks. */ debug("checking access to '%s'", abspath); - SourcePath path = CanonPath(canonPath(abspath, true)); + SourcePath path = rootPath(CanonPath(canonPath(abspath, true))); for (auto & i : *allowedPaths) { if (isDirOrInDir(path.path.abs(), i)) { @@ -649,12 +664,12 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(CanonPath(uri)); + checkSourcePath(rootPath(CanonPath(uri))); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(CanonPath(std::string(uri, 7))); + checkSourcePath(rootPath(CanonPath(std::string(uri, 7)))); return; } @@ -950,7 +965,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context) void Value::mkPath(const SourcePath & path) { - mkPath(makeImmutableString(path.path.abs())); + mkPath(&*path.accessor, makeImmutableString(path.path.abs())); } @@ -2037,7 +2052,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) else if (firstType == nPath) { if (!context.empty()) state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); - v.mkPath(CanonPath(canonPath(str()))); + v.mkPath(state.rootPath(CanonPath(canonPath(str())))); } else v.mkStringMove(c_str(), context); } @@ -2236,7 +2251,7 @@ BackedStringView EvalState::coerceToString( !canonicalizePath && !copyToStore ? // FIXME: hack to preserve path literals that end in a // slash, as in /foo/${x}. - v._path + v._path.path : copyToStore ? store->printStorePath(copyPathToStore(context, v.path())) : std::string(v.path().path.abs()); @@ -2329,7 +2344,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return CanonPath(path); + return rootPath(CanonPath(path)); } @@ -2429,7 +2444,9 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.string_view().compare(v2.string_view()) == 0; case nPath: - return strcmp(v1._path, v2._path) == 0; + return + v1._path.accessor == v2._path.accessor + && strcmp(v1._path.path, v2._path.path) == 0; case nNull: return true; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index c95558952..4bb0a6a0b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -24,6 +24,7 @@ class EvalState; class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; +struct FSInputAccessor; /** @@ -211,6 +212,8 @@ public: Bindings emptyBindings; + const ref rootFS; + const SourcePath derivationInternal; /** diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index a6212c12f..2f91ad433 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -223,9 +223,9 @@ static Flake getFlake( throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); Value vInfo; - state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack + state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack - expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); + expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1)); if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); @@ -741,7 +741,7 @@ void callFlake(EvalState & state, state.vCallFlake = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "call-flake.nix.gen.hh" - , CanonPath::root), **state.vCallFlake); + , state.rootPath(CanonPath::root)), **state.vCallFlake); } state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e766996c8..10099d49e 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -20,7 +20,6 @@ MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); -MakeError(RestrictedPathError, Error); /** * Position objects. @@ -200,9 +199,13 @@ struct ExprString : Expr struct ExprPath : Expr { + ref accessor; std::string s; Value v; - ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; + ExprPath(ref accessor, std::string s) : accessor(accessor), s(std::move(s)) + { + v.mkPath(&*accessor, this->s.c_str()); + } Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 70228e1e2..78436c6a2 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,6 +64,7 @@ struct StringToken { #include "parser-tab.hh" #include "lexer-tab.hh" +#include "fs-input-accessor.hh" YY_DECL; @@ -520,7 +521,7 @@ path_start /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(std::move(path)); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -530,7 +531,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(std::move(path)); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } ; @@ -756,11 +757,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ auto r = *rOpt; Path res = suffix == "" ? r : concatStrings(r, "/", suffix); - if (pathExists(res)) return CanonPath(canonPath(res)); + if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); } if (hasPrefix(path, "nix/")) - return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); + return rootPath(CanonPath(concatStrings(corepkgsPrefix, path.substr(4)))); debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index b6a696f47..099607638 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,10 +1,11 @@ #include "eval.hh" +#include "fs-input-accessor.hh" namespace nix { SourcePath EvalState::rootPath(CanonPath path) { - return path; + return {rootFS, std::move(path)}; } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3bed7c5aa..dd3fa12ee 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -202,7 +202,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , CanonPath::root), **state.vImportedDrvToDerivation); + , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation); } state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); @@ -213,7 +213,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v else if (path2 == corepkgsPrefix + "fetchurl.nix") { state.eval(state.parseExprFromString( #include "fetchurl.nix.gen.hh" - , CanonPath::root), v); + , state.rootPath(CanonPath::root)), v); } else { @@ -599,7 +599,8 @@ struct CompareValues case nString: return v1->string_view().compare(v2->string_view()) < 0; case nPath: - return strcmp(v1->_path, v2->_path) < 0; + // FIXME: handle accessor? + return strcmp(v1->_path.path, v2->_path.path) < 0; case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -2203,7 +2204,7 @@ static void addPath( path = evalSettings.pureEval && expectedHash ? path - : state.checkSourcePath(CanonPath(path)).path.abs(); + : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs(); PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); @@ -2236,9 +2237,10 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - StorePath dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs); + // FIXME + if (method != FileIngestionMethod::Recursive) + throw Error("'recursive = false' is not implemented"); + auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, &filter, state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); @@ -4447,7 +4449,7 @@ void EvalState::createBaseEnv() // the parser needs two NUL bytes as terminators; one of them // is implied by being a C string. "\0"; - eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation); + eval(parse(code, sizeof(code), derivationInternal, rootPath(CanonPath::root), staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index 7586bdd9b..f4cc118d6 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -62,7 +62,7 @@ namespace nix { // not supported by store 'dummy'" thrown in the test body. TEST_F(JSONValueTest, DISABLED_Path) { Value v; - v.mkPath("test"); + v.mkPath(state.rootPath(CanonPath("/test"))); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); } } /* namespace nix */ diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index 97b05ac46..968431446 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -103,14 +103,17 @@ namespace nix { } MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) { - if (arg.type() != nPath) { - *result_listener << "Expected a path got " << arg.type(); - return false; - } else if (std::string_view(arg._path) != p) { - *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str(); + if (arg.type() != nPath) { + *result_listener << "Expected a path got " << arg.type(); + return false; + } else { + auto path = arg.path(); + if (path.path != CanonPath(p)) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path; return false; } - return true; + } + return true; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 7a1165cac..622e613ea 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -189,7 +189,12 @@ public: const char * c_str; const char * * context; // must be in sorted order } string; - const char * _path; + + struct { + InputAccessor * accessor; + const char * path; + } _path; + Bindings * attrs; struct { size_t size; @@ -286,11 +291,12 @@ public: void mkPath(const SourcePath & path); - inline void mkPath(const char * path) + inline void mkPath(InputAccessor * accessor, const char * path) { clearValue(); internalType = tPath; - _path = path; + _path.accessor = accessor; + _path.path = path; } inline void mkNull() @@ -437,7 +443,10 @@ public: SourcePath path() const { assert(internalType == tPath); - return SourcePath{CanonPath(_path)}; + return SourcePath { + .accessor = ref(_path.accessor->shared_from_this()), + .path = CanonPath(CanonPath::unchecked_t(), _path.path) + }; } std::string_view string_view() const diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc new file mode 100644 index 000000000..a35955465 --- /dev/null +++ b/src/libfetchers/fs-input-accessor.cc @@ -0,0 +1,141 @@ +#include "fs-input-accessor.hh" +#include "store-api.hh" + +namespace nix { + +struct FSInputAccessorImpl : FSInputAccessor +{ + CanonPath root; + std::optional> allowedPaths; + MakeNotAllowedError makeNotAllowedError; + + FSInputAccessorImpl( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) + : root(root) + , allowedPaths(std::move(allowedPaths)) + , makeNotAllowedError(std::move(makeNotAllowedError)) + { + } + + std::string readFile(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return nix::readFile(absPath.abs()); + } + + bool pathExists(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + return isAllowed(absPath) && nix::pathExists(absPath.abs()); + } + + Stat lstat(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + auto st = nix::lstat(absPath.abs()); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; + } + + DirEntries readDirectory(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + DirEntries res; + for (auto & entry : nix::readDirectory(absPath.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + if (isAllowed(absPath + entry.name)) + res.emplace(entry.name, type); + } + return res; + } + + std::string readLink(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return nix::readLink(absPath.abs()); + } + + CanonPath makeAbsPath(const CanonPath & path) + { + return root + path; + } + + void checkAllowed(const CanonPath & absPath) override + { + if (!isAllowed(absPath)) + throw makeNotAllowedError + ? makeNotAllowedError(absPath) + : RestrictedPathError("access to path '%s' is forbidden", absPath); + } + + bool isAllowed(const CanonPath & absPath) + { + if (!absPath.isWithin(root)) + return false; + + if (allowedPaths) { + auto p = absPath.removePrefix(root); + if (!p.isAllowed(*allowedPaths)) + return false; + } + + return true; + } + + void allowPath(CanonPath path) override + { + if (allowedPaths) + allowedPaths->insert(std::move(path)); + } + + bool hasAccessControl() override + { + return (bool) allowedPaths; + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return makeAbsPath(path); + } +}; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) +{ + return make_ref(root, std::move(allowedPaths), std::move(makeNotAllowedError)); +} + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError) +{ + return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError)); +} + +SourcePath getUnfilteredRootPath(CanonPath path) +{ + static auto rootFS = makeFSInputAccessor(CanonPath::root); + return {rootFS, path}; +} + +} diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh new file mode 100644 index 000000000..19a5211c8 --- /dev/null +++ b/src/libfetchers/fs-input-accessor.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +class StorePath; +class Store; + +struct FSInputAccessor : InputAccessor +{ + virtual void checkAllowed(const CanonPath & absPath) = 0; + + virtual void allowPath(CanonPath path) = 0; + + virtual bool hasAccessControl() = 0; +}; + +typedef std::function MakeNotAllowedError; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths = {}, + MakeNotAllowedError && makeNotAllowedError = {}); + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError = {}); + +SourcePath getUnfilteredRootPath(CanonPath path); + +} diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index f37a8058b..fec16e99f 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -3,12 +3,149 @@ namespace nix { +static std::atomic nextNumber{0}; + +InputAccessor::InputAccessor() + : number(++nextNumber) +{ +} + +// FIXME: merge with archive.cc. +void InputAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) +{ + auto dumpContents = [&](const CanonPath & path) + { + // FIXME: pipe + auto s = readFile(path); + sink << "contents" << s.size(); + sink(s); + writePadding(s.size(), sink); + }; + + std::function dump; + + dump = [&](const CanonPath & path) { + checkInterrupt(); + + auto st = lstat(path); + + sink << "("; + + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } + + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (/* archiveSettings.useCaseHack */ false) { // FIXME + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); + + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; + } + } + + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error("file '%s' has an unsupported type", path); + + sink << ")"; + }; + + sink << narVersionMagic1; + dump(path); +} + +Hash InputAccessor::hashPath( + const CanonPath & path, + PathFilter & filter, + HashType ht) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish().first; +} + +StorePath InputAccessor::fetchToStore( + ref store, + const CanonPath & path, + std::string_view name, + PathFilter * filter, + RepairFlag repair) +{ + Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); + + auto source = sinkToSource([&](Sink & sink) { + dumpPath(path, sink, filter ? *filter : defaultPathFilter); + }); + + auto storePath = + settings.readOnlyMode + ? store->computeStorePathFromDump(*source, name).first + : store->addToStoreFromDump(*source, name, FileIngestionMethod::Recursive, htSHA256, repair); + + return storePath; +} + +std::optional InputAccessor::maybeLstat(const CanonPath & path) +{ + // FIXME: merge these into one operation. + if (!pathExists(path)) + return {}; + return lstat(path); +} + +std::string InputAccessor::showPath(const CanonPath & path) +{ + return path.abs(); +} + +SourcePath InputAccessor::root() +{ + return {ref(shared_from_this()), CanonPath::root}; +} + std::ostream & operator << (std::ostream & str, const SourcePath & path) { str << path.to_string(); return str; } +StorePath SourcePath::fetchToStore( + ref store, + std::string_view name, + PathFilter * filter, + RepairFlag repair) const +{ + return accessor->fetchToStore(store, path, name, filter, repair); +} + std::string_view SourcePath::baseName() const { return path.baseName().value_or("source"); @@ -18,60 +155,12 @@ SourcePath SourcePath::parent() const { auto p = path.parent(); assert(p); - return std::move(*p); -} - -InputAccessor::Stat SourcePath::lstat() const -{ - auto st = nix::lstat(path.abs()); - return InputAccessor::Stat { - .type = - S_ISREG(st.st_mode) ? InputAccessor::tRegular : - S_ISDIR(st.st_mode) ? InputAccessor::tDirectory : - S_ISLNK(st.st_mode) ? InputAccessor::tSymlink : - InputAccessor::tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; -} - -std::optional SourcePath::maybeLstat() const -{ - // FIXME: merge these into one operation. - if (!pathExists()) - return {}; - return lstat(); -} - -InputAccessor::DirEntries SourcePath::readDirectory() const -{ - InputAccessor::DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = InputAccessor::Type::tRegular; break; - case DT_LNK: type = InputAccessor::Type::tSymlink; break; - case DT_DIR: type = InputAccessor::Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; -} - -StorePath SourcePath::fetchToStore( - ref store, - std::string_view name, - PathFilter * filter, - RepairFlag repair) const -{ - return - settings.readOnlyMode - ? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first - : store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair); + return {accessor, std::move(*p)}; } SourcePath SourcePath::resolveSymlinks() const { - SourcePath res(CanonPath::root); + auto res = accessor->root(); int linksAllowed = 1024; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5a2f17f62..a0f08e295 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -5,14 +5,29 @@ #include "archive.hh" #include "canon-path.hh" #include "repair-flag.hh" +#include "hash.hh" namespace nix { +MakeError(RestrictedPathError, Error); + +struct SourcePath; class StorePath; class Store; -struct InputAccessor +struct InputAccessor : public std::enable_shared_from_this { + const size_t number; + + InputAccessor(); + + virtual ~InputAccessor() + { } + + virtual std::string readFile(const CanonPath & path) = 0; + + virtual bool pathExists(const CanonPath & path) = 0; + enum Type { tRegular, tSymlink, tDirectory, /** @@ -32,9 +47,63 @@ struct InputAccessor bool isExecutable = false; // regular files only }; + virtual Stat lstat(const CanonPath & path) = 0; + + std::optional maybeLstat(const CanonPath & path); + typedef std::optional DirEntry; typedef std::map DirEntries; + + virtual DirEntries readDirectory(const CanonPath & path) = 0; + + virtual std::string readLink(const CanonPath & path) = 0; + + virtual void dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter = defaultPathFilter); + + Hash hashPath( + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashType ht = htSHA256); + + StorePath fetchToStore( + ref store, + const CanonPath & path, + std::string_view name = "source", + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + + /* Return a corresponding path in the root filesystem, if + possible. This is only possible for inputs that are + materialized in the root filesystem. */ + virtual std::optional getPhysicalPath(const CanonPath & path) + { return std::nullopt; } + + bool operator == (const InputAccessor & x) const + { + return number == x.number; + } + + bool operator < (const InputAccessor & x) const + { + return number < x.number; + } + + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + + virtual std::string showPath(const CanonPath & path); + + SourcePath root(); + + /* Return the maximum last-modified time of the files in this + tree, if available. */ + virtual std::optional getLastModified() + { + return std::nullopt; + } }; /** @@ -45,12 +114,9 @@ struct InputAccessor */ struct SourcePath { + ref accessor; CanonPath path; - SourcePath(CanonPath path) - : path(std::move(path)) - { } - std::string_view baseName() const; /** @@ -64,39 +130,42 @@ struct SourcePath * return its contents; otherwise throw an error. */ std::string readFile() const - { return nix::readFile(path.abs()); } + { return accessor->readFile(path); } /** * Return whether this `SourcePath` denotes a file (of any type) * that exists */ bool pathExists() const - { return nix::pathExists(path.abs()); } + { return accessor->pathExists(path); } /** * Return stats about this `SourcePath`, or throw an exception if * it doesn't exist. */ - InputAccessor::Stat lstat() const; + InputAccessor::Stat lstat() const + { return accessor->lstat(path); } /** * Return stats about this `SourcePath`, or std::nullopt if it * doesn't exist. */ - std::optional maybeLstat() const; + std::optional maybeLstat() const + { return accessor->maybeLstat(path); } /** * If this `SourcePath` denotes a directory (not a symlink), * return its directory entries; otherwise throw an error. */ - InputAccessor::DirEntries readDirectory() const; + InputAccessor::DirEntries readDirectory() const + { return accessor->readDirectory(path); } /** * If this `SourcePath` denotes a symlink, return its target; * otherwise throw an error. */ std::string readLink() const - { return nix::readLink(path.abs()); } + { return accessor->readLink(path); } /** * Dump this `SourcePath` to `sink` as a NAR archive. @@ -104,7 +173,7 @@ struct SourcePath void dumpPath( Sink & sink, PathFilter & filter = defaultPathFilter) const - { return nix::dumpPath(path.abs(), sink, filter); } + { return accessor->dumpPath(path, sink, filter); } /** * Copy this `SourcePath` to the Nix store. @@ -120,7 +189,7 @@ struct SourcePath * it has a physical location. */ std::optional getPhysicalPath() const - { return path; } + { return accessor->getPhysicalPath(path); } std::string to_string() const { return path.abs(); } @@ -129,7 +198,7 @@ struct SourcePath * Append a `CanonPath` to this path. */ SourcePath operator + (const CanonPath & x) const - { return {path + x}; } + { return {accessor, path + x}; } /** * Append a single component `c` to this path. `c` must not @@ -137,21 +206,21 @@ struct SourcePath * and `c`. */ SourcePath operator + (std::string_view c) const - { return {path + c}; } + { return {accessor, path + c}; } bool operator == (const SourcePath & x) const { - return path == x.path; + return std::tie(accessor, path) == std::tie(x.accessor, x.path); } bool operator != (const SourcePath & x) const { - return path != x.path; + return std::tie(accessor, path) != std::tie(x.accessor, x.path); } bool operator < (const SourcePath & x) const { - return path < x.path; + return std::tie(accessor, path) < std::tie(x.accessor, x.path); } /** diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28689e100..e761c0e96 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -225,12 +225,16 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const +std::pair Store::computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashType hashAlgo, + const StorePathSet & references) const { - Hash h = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath, filter).first - : hashFile(hashAlgo, srcPath); + HashSink sink(hashAlgo); + dump.drainInto(sink); + auto h = sink.finish().first; FixedOutputInfo caInfo { .method = method, .hash = h, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4e233bac3..bd69999f8 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -292,14 +292,15 @@ public: StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** - * Preparatory part of addToStore(). - * - * @return the store path to which srcPath is to be copied - * and the cryptographic hash of the contents of srcPath. + * Read-only variant of addToStoreFromDump(). It returns the store + * path to which a NAR or flat file would be written. */ - std::pair computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; + std::pair computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = {}) const; /** * Preparatory part of addTextToStore(). diff --git a/src/nix/main.cc b/src/nix/main.cc index 879baa5f5..6cd7ace51 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -204,30 +204,30 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( #include "generate-manpage.nix.gen.hh" - , CanonPath::root), *vGenerateManpage); + , state.rootPath(CanonPath::root)), *vGenerateManpage); auto vUtils = state.allocValue(); state.cacheFile( - CanonPath("/utils.nix"), CanonPath("/utils.nix"), + state.rootPath(CanonPath("/utils.nix")), state.rootPath(CanonPath("/utils.nix")), state.parseExprFromString( #include "utils.nix.gen.hh" - , CanonPath::root), + , state.rootPath(CanonPath::root)), *vUtils); auto vSettingsInfo = state.allocValue(); state.cacheFile( - CanonPath("/generate-settings.nix"), CanonPath("/generate-settings.nix"), + state.rootPath(CanonPath("/generate-settings.nix")), state.rootPath(CanonPath("/generate-settings.nix")), state.parseExprFromString( #include "generate-settings.nix.gen.hh" - , CanonPath::root), + , state.rootPath(CanonPath::root)), *vSettingsInfo); auto vStoreInfo = state.allocValue(); state.cacheFile( - CanonPath("/generate-store-info.nix"), CanonPath("/generate-store-info.nix"), + state.rootPath(CanonPath("/generate-store-info.nix")), state.rootPath(CanonPath("/generate-store-info.nix")), state.parseExprFromString( #include "generate-store-info.nix.gen.hh" - , CanonPath::root), + , state.rootPath(CanonPath::root)), *vStoreInfo); auto vDump = state.allocValue(); From df73c6eb8cb14fe11f2d5b27e2e28135c7c5dbaf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Oct 2023 17:34:58 +0200 Subject: [PATCH 027/100] Introduce MemoryInputAccessor and use it for corepkgs MemoryInputAccessor is an in-memory virtual filesystem that returns files like . This removes the need for special hacks to handle those files. --- doc/manual/generate-manpage.nix | 2 +- doc/manual/local.mk | 2 +- src/libexpr/eval.cc | 77 ++++++++++++++++-------- src/libexpr/eval.hh | 32 +++++----- src/libexpr/flake/flake.cc | 10 +-- src/libexpr/parser.y | 5 +- src/libexpr/primops.cc | 29 +++------ src/libexpr/tests/error_traces.cc | 14 ++--- src/libfetchers/memory-input-accessor.cc | 54 +++++++++++++++++ src/libfetchers/memory-input-accessor.hh | 15 +++++ src/nix/main.cc | 34 ++++------- 11 files changed, 176 insertions(+), 98 deletions(-) create mode 100644 src/libfetchers/memory-input-accessor.cc create mode 100644 src/libfetchers/memory-input-accessor.hh diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index d81eac182..14136016d 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -2,7 +2,7 @@ let inherit (builtins) attrNames attrValues fromJSON listToAttrs mapAttrs groupBy concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique; + inherit (import ) attrsToList concatStrings optionalString filterAttrs trim squash unique; showStoreDocs = import ./generate-store-info.nix; in diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 702fb1ff8..2e9f08678 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -32,7 +32,7 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 87791fa52..a28bb2101 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -13,6 +13,7 @@ #include "profiles.hh" #include "print.hh" #include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" #include #include @@ -516,7 +517,16 @@ EvalState::EvalState( : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); })) - , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) + , corepkgsFS(makeMemoryInputAccessor()) + , internalFS(makeMemoryInputAccessor()) + , derivationInternal{corepkgsFS->addFile( + CanonPath("derivation-internal.nix"), + #include "primops/derivation.nix.gen.hh" + )} + , callFlakeInternal{internalFS->addFile( + CanonPath("call-flake.nix"), + #include "flake/call-flake.nix.gen.hh" + )} , store(store) , buildStore(buildStore ? buildStore : store) , debugRepl(nullptr) @@ -531,7 +541,8 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { - rootFS->allowPath(CanonPath::root); // FIXME + // For now, don't rely on FSInputAccessor for access control. + rootFS->allowPath(CanonPath::root); countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; @@ -570,6 +581,11 @@ EvalState::EvalState( } } + corepkgsFS->addFile( + CanonPath("fetchurl.nix"), + #include "fetchurl.nix.gen.hh" + ); + createBaseEnv(); } @@ -600,6 +616,7 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & SourcePath EvalState::checkSourcePath(const SourcePath & path_) { + if (path_.accessor != rootFS) return path_; if (!allowedPaths) return path_; auto i = resolvedPaths.find(path_.path.abs()); @@ -614,8 +631,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) */ Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return rootPath(CanonPath(abspath)); - for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { found = true; @@ -1180,24 +1195,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial if (!e) e = parseExprFromFile(checkSourcePath(resolvedPath)); - cacheFile(path, resolvedPath, e, v, mustBeTrivial); -} - - -void EvalState::resetFileCache() -{ - fileEvalCache.clear(); - fileParseCache.clear(); -} - - -void EvalState::cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial) -{ fileParseCache[resolvedPath] = e; try { @@ -1226,6 +1223,13 @@ void EvalState::cacheFile( } +void EvalState::resetFileCache() +{ + fileEvalCache.clear(); + fileParseCache.clear(); +} + + void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); @@ -2341,10 +2345,31 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { - auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); - if (path == "" || path[0] != '/') - error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return rootPath(CanonPath(path)); + try { + forceValue(v, pos); + + if (v.type() == nString) { + copyContext(v, context); + auto s = v.string_view(); + if (!hasPrefix(s, "/")) + error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow(); + return rootPath(CanonPath(s)); + } + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } + + if (v.type() == nPath) + return v.path(); + + if (v.type() == nAttrs) { + auto i = v.attrs->find(sOutPath); + if (i != v.attrs->end()) + return coerceToPath(pos, *i->value, context, errorCtx); + } + + error("cannot coerce %1% to a path", showType(v)).withTrace(pos, errorCtx).debugThrow(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 4bb0a6a0b..048dff42b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -25,6 +25,7 @@ class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; struct FSInputAccessor; +struct MemoryInputAccessor; /** @@ -212,10 +213,26 @@ public: Bindings emptyBindings; + /** + * The accessor for the root filesystem. + */ const ref rootFS; + /** + * The in-memory filesystem for paths. + */ + const ref corepkgsFS; + + /** + * In-memory filesystem for internal, non-user-callable Nix + * expressions like call-flake.nix. + */ + const ref internalFS; + const SourcePath derivationInternal; + const SourcePath callFlakeInternal; + /** * Store used to materialise .drv files. */ @@ -226,7 +243,6 @@ public: */ const ref buildStore; - RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; /** @@ -408,16 +424,6 @@ public: */ void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); - /** - * Like `evalFile`, but with an already parsed expression. - */ - void cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial = false); - void resetFileCache(); /** @@ -427,7 +433,7 @@ public: SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** - * Try to resolve a search path value (not the optinal key part) + * Try to resolve a search path value (not the optional key part) * * If the specified search path element is a URI, download it. * @@ -832,8 +838,6 @@ struct InvalidPathError : EvalError #endif }; -static const std::string corepkgsPrefix{"/__corepkgs__/"}; - template void ErrorBuilder::debugThrow() { diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2f91ad433..acc84ea4f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -737,14 +737,10 @@ void callFlake(EvalState & state, vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); - if (!state.vCallFlake) { - state.vCallFlake = allocRootValue(state.allocValue()); - state.eval(state.parseExprFromString( - #include "call-flake.nix.gen.hh" - , state.rootPath(CanonPath::root)), **state.vCallFlake); - } + auto vCallFlake = state.allocValue(); + state.evalFile(state.callFlakeInternal, *vCallFlake); - state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 78436c6a2..19812cc49 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,7 +64,6 @@ struct StringToken { #include "parser-tab.hh" #include "lexer-tab.hh" -#include "fs-input-accessor.hh" YY_DECL; @@ -650,6 +649,8 @@ formal #include "fetchers.hh" #include "store-api.hh" #include "flake/flake.hh" +#include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" namespace nix { @@ -761,7 +762,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ } if (hasPrefix(path, "nix/")) - return rootPath(CanonPath(concatStrings(corepkgsPrefix, path.substr(4)))); + return {corepkgsFS, CanonPath(path.substr(3))}; debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index dd3fa12ee..f1c24a75c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { - StringMap rewrites = state.realiseContext(context); - - auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); + if (!context.empty()) { + auto rewrites = state.realiseContext(context); + auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); + return {path.accessor, CanonPath(realPath)}; + } return flags.checkForPureEval - ? state.checkSourcePath(realPath) - : realPath; + ? state.checkSourcePath(path) + : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } - else if (path2 == corepkgsPrefix + "fetchurl.nix") { - state.eval(state.parseExprFromString( - #include "fetchurl.nix.gen.hh" - , state.rootPath(CanonPath::root)), v); - } - else { if (!vScope) state.evalFile(path, v); @@ -1486,7 +1482,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); NixStringContext context; - auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; + auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ @@ -2257,7 +2253,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg { NixStringContext context; auto path = state.coerceToPath(pos, *args[1], context, - "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); + "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } @@ -4444,12 +4440,7 @@ void EvalState::createBaseEnv() /* Note: we have to initialize the 'derivation' constant *after* building baseEnv/staticBaseEnv because it uses 'builtins'. */ - char code[] = - #include "primops/derivation.nix.gen.hh" - // the parser needs two NUL bytes as terminators; one of them - // is implied by being a C string. - "\0"; - eval(parse(code, sizeof(code), derivationInternal, rootPath(CanonPath::root), staticBaseEnv), *vDerivation); + evalFile(derivationInternal, *vDerivation); } diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 285651256..6da8a5954 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a path", "a list"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,8 +309,8 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a string", "a Boolean"), - hintfmt("while evaluating the first argument passed to builtins.storePath")); + hintfmt("cannot coerce %s to a path", "a Boolean"), + hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a path", "a list"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -377,13 +377,13 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", EvalError, hintfmt("string '%s' doesn't represent an absolute path", "foo"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] ./.", TypeError, diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc new file mode 100644 index 000000000..817d063ba --- /dev/null +++ b/src/libfetchers/memory-input-accessor.cc @@ -0,0 +1,54 @@ +#include "memory-input-accessor.hh" + +namespace nix { + +struct MemoryInputAccessorImpl : MemoryInputAccessor +{ + std::map files; + + std::string readFile(const CanonPath & path) override + { + auto i = files.find(path); + if (i == files.end()) + throw Error("file '%s' does not exist", path); + return i->second; + } + + bool pathExists(const CanonPath & path) override + { + auto i = files.find(path); + return i != files.end(); + } + + Stat lstat(const CanonPath & path) override + { + auto i = files.find(path); + if (i != files.end()) + return Stat { .type = tRegular, .isExecutable = false }; + throw Error("file '%s' does not exist", path); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return {}; + } + + std::string readLink(const CanonPath & path) override + { + throw UnimplementedError("MemoryInputAccessor::readLink"); + } + + SourcePath addFile(CanonPath path, std::string && contents) override + { + files.emplace(path, std::move(contents)); + + return {ref(shared_from_this()), std::move(path)}; + } +}; + +ref makeMemoryInputAccessor() +{ + return make_ref(); +} + +} diff --git a/src/libfetchers/memory-input-accessor.hh b/src/libfetchers/memory-input-accessor.hh new file mode 100644 index 000000000..b75b02bfd --- /dev/null +++ b/src/libfetchers/memory-input-accessor.hh @@ -0,0 +1,15 @@ +#include "input-accessor.hh" + +namespace nix { + +/** + * An input accessor for an in-memory file system. + */ +struct MemoryInputAccessor : InputAccessor +{ + virtual SourcePath addFile(CanonPath path, std::string && contents) = 0; +}; + +ref makeMemoryInputAccessor(); + +} diff --git a/src/nix/main.cc b/src/nix/main.cc index 6cd7ace51..ee3878cc7 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -12,6 +12,7 @@ #include "finally.hh" #include "loggers.hh" #include "markdown.hh" +#include "memory-input-accessor.hh" #include #include @@ -206,29 +207,20 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) #include "generate-manpage.nix.gen.hh" , state.rootPath(CanonPath::root)), *vGenerateManpage); - auto vUtils = state.allocValue(); - state.cacheFile( - state.rootPath(CanonPath("/utils.nix")), state.rootPath(CanonPath("/utils.nix")), - state.parseExprFromString( - #include "utils.nix.gen.hh" - , state.rootPath(CanonPath::root)), - *vUtils); + state.corepkgsFS->addFile( + CanonPath("utils.nix"), + #include "utils.nix.gen.hh" + ); - auto vSettingsInfo = state.allocValue(); - state.cacheFile( - state.rootPath(CanonPath("/generate-settings.nix")), state.rootPath(CanonPath("/generate-settings.nix")), - state.parseExprFromString( - #include "generate-settings.nix.gen.hh" - , state.rootPath(CanonPath::root)), - *vSettingsInfo); + state.corepkgsFS->addFile( + CanonPath("/generate-settings.nix"), + #include "generate-settings.nix.gen.hh" + ); - auto vStoreInfo = state.allocValue(); - state.cacheFile( - state.rootPath(CanonPath("/generate-store-info.nix")), state.rootPath(CanonPath("/generate-store-info.nix")), - state.parseExprFromString( - #include "generate-store-info.nix.gen.hh" - , state.rootPath(CanonPath::root)), - *vStoreInfo); + state.corepkgsFS->addFile( + CanonPath("/generate-store-info.nix"), + #include "generate-store-info.nix.gen.hh" + ); auto vDump = state.allocValue(); vDump->mkString(toplevel.dumpCli()); From e9ddf0b4008262542b422c76f60c3fb80982cd74 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 9 Oct 2023 10:53:47 +0800 Subject: [PATCH 028/100] Simplify parseHashTypeOpt Remove redundant "else" after "return". Use std::nullopt to increase readability. --- src/libutil/hash.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c36d9d94..6d56c1989 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -389,10 +389,10 @@ Hash compressHash(const Hash & hash, unsigned int newSize) std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; - else if (s == "sha1") return htSHA1; - else if (s == "sha256") return htSHA256; - else if (s == "sha512") return htSHA512; - else return std::optional {}; + if (s == "sha1") return htSHA1; + if (s == "sha256") return htSHA256; + if (s == "sha512") return htSHA512; + return std::nullopt; } HashType parseHashType(std::string_view s) From aff177d8609c392b4edf59819a086440cca101a7 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 9 Oct 2023 11:00:14 +0800 Subject: [PATCH 029/100] Elaborate the "unknown hash algorithm" error List the allowed hash formats --- src/libexpr/tests/error_traces.cc | 2 +- src/libutil/hash.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 285651256..1af601ec8 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -1084,7 +1084,7 @@ namespace nix { ASSERT_TRACE1("hashString \"foo\" \"content\"", UsageError, - hintfmt("unknown hash algorithm '%s'", "foo")); + hintfmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); ASSERT_TRACE2("hashString \"sha256\" {}", TypeError, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 6d56c1989..251440363 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -401,7 +401,7 @@ HashType parseHashType(std::string_view s) if (opt_h) return *opt_h; else - throw UsageError("unknown hash algorithm '%1%'", s); + throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s); } std::string_view printHashType(HashType ht) From 838c70f62116328ce01cb41a01886e4f1b9a727f Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Tue, 10 Oct 2023 18:53:01 +0800 Subject: [PATCH 030/100] treewide: Rename hashBase to hashFormat hashBase is ambiguous, since it's not about the digital bases, but about the format of hashes. Base16, Base32 and Base64 are all character maps for binary encoding. Rename the enum Base to HashFormat. Rename variables of type HashFormat from [hash]Base to hashFormat, including CmdHashBase::hashFormat and CmdToBase::hashFormat. --- src/libstore/store-api.cc | 6 ++--- src/libstore/store-api.hh | 2 +- src/libutil/hash.cc | 8 +++---- src/libutil/hash.hh | 4 ++-- src/libutil/tests/hash.cc | 16 +++++++------- src/nix/hash.cc | 46 +++++++++++++++++++-------------------- 6 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28689e100..fa23f976e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -938,7 +938,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor json Store::pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - Base hashBase, + HashFormat hashFormat, AllowInvalidFlag allowInvalid) { json::array_t jsonList = json::array(); @@ -951,7 +951,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, jsonPath["path"] = printStorePath(info->path); jsonPath["valid"] = true; - jsonPath["narHash"] = info->narHash.to_string(hashBase, true); + jsonPath["narHash"] = info->narHash.to_string(hashFormat, true); jsonPath["narSize"] = info->narSize; { @@ -993,7 +993,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, if (!narInfo->url.empty()) jsonPath["url"] = narInfo->url; if (narInfo->fileHash) - jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); + jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true); if (narInfo->fileSize) jsonPath["downloadSize"] = narInfo->fileSize; if (showClosureSize) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4e233bac3..f30ccc950 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -676,7 +676,7 @@ public: */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - Base hashBase = Base32, + HashFormat hashFormat = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 251440363..401a65ec3 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -115,14 +115,14 @@ std::string printHash16or32(const Hash & hash) } -std::string Hash::to_string(Base base, bool includeType) const +std::string Hash::to_string(HashFormat hashFormat, bool includeType) const { std::string s; - if (base == SRI || includeType) { + if (hashFormat == SRI || includeType) { s += printHashType(type); - s += base == SRI ? '-' : ':'; + s += hashFormat == SRI ? '-' : ':'; } - switch (base) { + switch (hashFormat) { case Base16: s += printHash16(*this); break; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index c3aa5cd81..73c2183ef 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -23,7 +23,7 @@ extern std::set hashTypes; extern const std::string base32Chars; -enum Base : int { Base64, Base32, Base16, SRI }; +enum HashFormat : int { Base64, Base32, Base16, SRI }; struct Hash @@ -114,7 +114,7 @@ public: * or base-64. By default, this is prefixed by the hash type * (e.g. "sha256:"). */ - std::string to_string(Base base, bool includeType) const; + std::string to_string(HashFormat hashFormat, bool includeType) const; std::string gitRev() const { diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index e4e928b3b..d14e5b26d 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -18,28 +18,28 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; auto hash = hashString(HashType::htMD5, s1); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; auto hash = hashString(HashType::htMD5, s2); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } TEST(hashString, testKnownSHA256Hashes1) { @@ -47,7 +47,7 @@ namespace nix { auto s = "abc"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -55,7 +55,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -63,7 +63,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); @@ -74,7 +74,7 @@ namespace nix { auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9feca9345..6af6bd19f 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -11,7 +11,7 @@ using namespace nix; struct CmdHashBase : Command { FileIngestionMethod mode; - Base base = SRI; + HashFormat hashFormat = SRI; bool truncate = false; HashType ht = htSHA256; std::vector paths; @@ -22,25 +22,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&base, SRI}, + .handler = {&hashFormat, SRI}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&base, Base64}, + .handler = {&hashFormat, Base64}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&base, Base32}, + .handler = {&hashFormat, Base32}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&base, Base16}, + .handler = {&hashFormat, Base16}, }); addFlag(Flag::mkHashTypeFlag("type", &ht)); @@ -94,18 +94,18 @@ struct CmdHashBase : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(base, base == SRI)); + logger->cout(h.to_string(hashFormat, hashFormat == SRI)); } } }; struct CmdToBase : Command { - Base base; + HashFormat hashFormat; std::optional ht; std::vector args; - CmdToBase(Base base) : base(base) + CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { addFlag(Flag::mkHashTypeOptFlag("type", &ht)); expectArgs("strings", &args); @@ -114,16 +114,16 @@ struct CmdToBase : Command std::string description() override { return fmt("convert a hash to %s representation", - base == Base16 ? "base-16" : - base == Base32 ? "base-32" : - base == Base64 ? "base-64" : + hashFormat == Base16 ? "base-16" : + hashFormat == Base32 ? "base-32" : + hashFormat == Base64 ? "base-64" : "SRI"); } void run() override { for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == SRI)); } }; @@ -162,7 +162,7 @@ static int compatNixHash(int argc, char * * argv) { std::optional ht; bool flat = false; - Base base = Base16; + HashFormat hashFormat = Base16; bool truncate = false; enum { opHash, opTo } op = opHash; std::vector ss; @@ -173,10 +173,10 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base16") base = Base16; - else if (*arg == "--base32") base = Base32; - else if (*arg == "--base64") base = Base64; - else if (*arg == "--sri") base = SRI; + else if (*arg == "--base16") hashFormat = Base16; + else if (*arg == "--base32") hashFormat = Base32; + else if (*arg == "--base64") hashFormat = Base64; + else if (*arg == "--sri") hashFormat = SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); @@ -184,19 +184,19 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base16") { op = opTo; - base = Base16; + hashFormat = Base16; } else if (*arg == "--to-base32") { op = opTo; - base = Base32; + hashFormat = Base32; } else if (*arg == "--to-base64") { op = opTo; - base = Base64; + hashFormat = Base64; } else if (*arg == "--to-sri") { op = opTo; - base = SRI; + hashFormat = SRI; } else if (*arg != "" && arg->at(0) == '-') return false; @@ -209,14 +209,14 @@ static int compatNixHash(int argc, char * * argv) CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); if (!ht.has_value()) ht = htMD5; cmd.ht = ht.value(); - cmd.base = base; + cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(base); + CmdToBase cmd(hashFormat); cmd.args = ss; if (ht.has_value()) cmd.ht = ht; cmd.run(); From 5043e6cf4ea537dfe599470797c5b310ab0e94b9 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Wed, 18 Oct 2023 17:08:06 +0800 Subject: [PATCH 031/100] Document HashFormat --- src/libutil/hash.hh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 73c2183ef..0db9bcd6d 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -23,7 +23,21 @@ extern std::set hashTypes; extern const std::string base32Chars; -enum HashFormat : int { Base64, Base32, Base16, SRI }; +/** + * @brief Enumeration representing the hash formats. + */ +enum HashFormat : int { + /// @brief Base 64 encoding. + /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). + Base64, + /// @brief Nix-specific base-32 encoding. @see base32Chars + Base32, + /// @brief Lowercase hexadecimal encoding. @see base16Chars + Base16, + /// @brief ":", format of the SRI integrity attribute. + /// @see W3C recommendation [Subresource Intergrity](https://www.w3.org/TR/SRI/). + SRI +}; struct Hash From e026f3e1ae533ab476e58258c7d84814189df9ff Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 13 Oct 2023 09:48:15 +0800 Subject: [PATCH 032/100] treewide: Reference HashFormat members with scope Base* -> HashFormat::Base* --- perl/lib/Nix/Store.xs | 12 +++--- src/libexpr/eval-cache.cc | 2 +- src/libexpr/primops.cc | 4 +- src/libexpr/primops/fetchTree.cc | 4 +- src/libfetchers/fetchers.cc | 4 +- src/libfetchers/git.cc | 4 +- src/libfetchers/github.cc | 8 ++-- src/libfetchers/mercurial.cc | 4 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 6 +-- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/content-address.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 12 +++--- src/libstore/downstream-placeholder.cc | 4 +- src/libstore/export-import.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 14 +++---- src/libstore/nar-info-disk-cache.cc | 4 +- src/libstore/nar-info.cc | 4 +- src/libstore/optimise-store.cc | 4 +- src/libstore/path-info.cc | 4 +- src/libstore/path.cc | 2 +- src/libstore/realisation.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 6 +-- src/libstore/store-api.hh | 2 +- src/libutil/hash.cc | 16 +++---- src/libutil/hash.hh | 4 +- src/nix-store/nix-store.cc | 8 ++-- src/nix/flake.cc | 8 ++-- src/nix/hash.cc | 46 ++++++++++----------- src/nix/path-info.cc | 2 +- src/nix/prefetch.cc | 4 +- src/nix/verify.cc | 4 +- 37 files changed, 108 insertions(+), 108 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index e96885e4c..08f812b31 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -78,7 +78,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -206,7 +206,7 @@ SV * hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath(parseHashType(algo), path).first; - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -217,7 +217,7 @@ SV * hashFile(char * algo, int base32, char * path) PPCODE: try { Hash h = hashFile(parseHashType(algo), path); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -228,7 +228,7 @@ SV * hashString(char * algo, int base32, char * s) PPCODE: try { Hash h = hashString(parseHashType(algo), s); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -239,7 +239,7 @@ SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { auto h = Hash::parseAny(s, parseHashType(algo)); - auto s = h.to_string(toBase32 ? Base32 : Base16, false); + auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 391b32a77..10fc799a9 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -50,7 +50,7 @@ struct AttrDb Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); - Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite"; state->db = SQLite(dbPath); state->db.isCache(); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3bed7c5aa..830579dbf 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1758,7 +1758,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[1]); - v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); + v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -3760,7 +3760,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(Base16, false)); + v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashString({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 3431ff013..976037ff9 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -31,7 +31,7 @@ void emitTreeAttrs( auto narHash = input.getNarHash(); assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( @@ -297,7 +297,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); + *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); } state.allowAndSetStorePathString(storePath, v); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e3d3ac562..c54c39cf0 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -147,12 +147,12 @@ std::pair Input::fetch(ref store) const }; auto narHash = store->queryPathInfo(tree.storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true)); + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); + to_string(), tree.actualPath, prevNarHash->to_string(HashFormat::SRI, true), narHash.to_string(HashFormat::SRI, true)); } if (auto prevLastModified = getLastModified()) { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7e9f34790..c1a6dce43 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -46,7 +46,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(Base32, false); + hashString(htSHA256, key).to_string(HashFormat::Base32, false); } // Returns the name of the HEAD branch. @@ -417,7 +417,7 @@ struct GitInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) - throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); }; auto getLockedAttrs = [&]() diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 6c5c792ec..d7450defe 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -125,7 +125,7 @@ struct GitArchiveInputScheme : InputScheme auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; - if (rev) path += "/" + rev->to_string(Base16, false); + if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); return ParsedURL { .scheme = type(), .path = path, @@ -296,7 +296,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://api.%s/repos/%s/%s/tarball/%s"; const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); return DownloadUrl { url, headers }; } @@ -362,7 +362,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; @@ -449,7 +449,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 51fd1ed42..b0d2d1909 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -206,7 +206,7 @@ struct MercurialInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && hash->type != htSHA1) - throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); }; @@ -252,7 +252,7 @@ struct MercurialInputScheme : InputScheme } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e3a41e75e..269a56526 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -250,7 +250,7 @@ struct CurlInputScheme : InputScheme // NAR hashes are preferred over file hashes since tar/zip // files don't have a canonical representation. if (auto narHash = input.getNarHash()) - url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b4fea693f..2a91233ec 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -164,7 +164,7 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6a02f5ad4..40a0385af 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1066,7 +1066,7 @@ void LocalDerivationGoal::initTmpDir() { env[i.first] = i.second; } else { auto hash = hashString(htSHA256, i.first); - std::string fn = ".attr-" + hash.to_string(Base32, false); + std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -2583,8 +2583,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true))); } if (!newInfo0.references.empty()) delayedException = std::make_exception_ptr( diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 7d7924d77..357800333 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index ae91b859b..52c60154c 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -60,7 +60,7 @@ std::string ContentAddress::render() const + makeFileIngestionPrefix(method); }, }, method.raw) - + this->hash.to_string(Base32, true); + + this->hash.to_string(HashFormat::Base32, true); } /** diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8cbf6f044..5860dd548 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -334,7 +334,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash.to_string(HashFormat::Base16, false); break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fc17e520c..a5ceb29dc 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -542,7 +542,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(HashFormat::Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -775,7 +775,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut auto & dof = std::get(i.second.raw); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.hash.to_string(Base16, false) + ":" + + dof.ca.hash.to_string(HashFormat::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -820,7 +820,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].value.insert(outputName); + inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); } } @@ -925,7 +925,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.hash.to_string(Base16, false); + << dof.ca.hash.to_string(HashFormat::Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -957,7 +957,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); + return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); } @@ -1162,7 +1162,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.hash.to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 7e3f7548d..ca9f7476e 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -5,7 +5,7 @@ namespace nix { std::string DownstreamPlaceholder::render() const { - return "/" + hash.to_string(Base32, false); + return "/" + hash.to_string(HashFormat::Base32, false); } @@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( xpSettings.require(Xp::DynamicDerivations); auto compressed = compressHash(placeholder.hash, 20); auto clearText = "nix-computed-output:" - + compressed.to_string(Base32, false) + + compressed.to_string(HashFormat::Base32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { hashString(htSHA256, clearText) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 87b2f8741..91b7e30db 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashSink.currentHash().first; if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); teeSink << exportMagic diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 516cbef83..fb7895817 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -43,7 +43,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(Base32, false); + std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 703ded0b2..46c90ee31 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -209,7 +209,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << ServeProto::Command::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(HashFormat::Base16, false); ServeProto::write(*this, *conn, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17b4ecc73..1c2f6023a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -826,7 +826,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmts->RegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -933,7 +933,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -1236,7 +1236,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1252,8 +1252,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - specified.hash.to_string(Base32, true), - actualHash.hash.to_string(Base32, true)); + specified.hash.to_string(HashFormat::Base32, true), + actualHash.hash.to_string(HashFormat::Base32, true)); } } @@ -1545,7 +1545,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1578,7 +1578,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); + printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c7176d30f..cdbcf7e74 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -332,9 +332,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash.to_string(HashFormat::Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d17253741..ee2ddfd81 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -105,10 +105,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash && fileHash->type == htSHA256); - res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; + res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 4a79cf4a1..23c6a41e4 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -146,10 +146,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false); /* Maybe delete the link, if it has been corrupted. */ if (pathExists(linkPath)) { diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ccb57104f..7ad3247da 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -12,7 +12,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash.to_string(HashFormat::Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -161,7 +161,7 @@ void ValidPathInfo::write( if (includePath) sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") - << narHash.to_string(Base16, false); + << narHash.to_string(HashFormat::Base16, false); WorkerProto::write(store, WorkerProto::WriteConn { .to = sink }, references); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 3c6b9fc10..ec3e53232 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName) } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name))) { checkName(baseName, name()); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 559483ce3..4ba2123d8 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -39,7 +39,7 @@ struct DrvOutput { std::string to_string() const; std::string strHash() const - { return drvHash.to_string(Base16, true); } + { return drvHash.to_string(HashFormat::Base16, true); } static DrvOutput parse(const std::string &); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a639346d1..1704bfbb0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -541,7 +541,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fa23f976e..0c58cca0c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -154,7 +154,7 @@ StorePath Store::makeStorePath(std::string_view type, StorePath Store::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { - return makeStorePath(type, hash.to_string(Base16, true), name); + return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } @@ -192,7 +192,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf hashString(htSHA256, "fixed:out:" + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + info.hash.to_string(HashFormat::Base16, true) + ":"), name); } } @@ -884,7 +884,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash.to_string(HashFormat::Base16, false) + "\n"; s += fmt("%1%\n", info->narSize); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f30ccc950..2d27f7d51 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -676,7 +676,7 @@ public: */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - HashFormat hashFormat = Base32, + HashFormat hashFormat = HashFormat::Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 401a65ec3..b58b5f08d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -111,26 +111,26 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { assert(hash.type); - return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); + return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false); } std::string Hash::to_string(HashFormat hashFormat, bool includeType) const { std::string s; - if (hashFormat == SRI || includeType) { + if (hashFormat == HashFormat::SRI || includeType) { s += printHashType(type); - s += hashFormat == SRI ? '-' : ':'; + s += hashFormat == HashFormat::SRI ? '-' : ':'; } switch (hashFormat) { - case Base16: + case HashFormat::Base16: s += printHash16(*this); break; - case Base32: + case HashFormat::Base32: s += printHash32(*this); break; - case Base64: - case SRI: + case HashFormat::Base64: + case HashFormat::SRI: s += base64Encode(std::string_view((const char *) hash, hashSize)); break; } @@ -267,7 +267,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht) if (!ht) throw BadHash("empty hash requires explicit hash type"); Hash h(*ht); - warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; } else return Hash::parseAny(hashStr, ht); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0db9bcd6d..783e30496 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -132,12 +132,12 @@ public: std::string gitRev() const { - return to_string(Base16, false); + return to_string(HashFormat::Base16, false); } std::string gitShortRev() const { - return std::string(to_string(Base16, false), 0, 7); + return std::string(to_string(HashFormat::Base16, false), 0, 7); } static Hash dummy; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9b6c80a75..5e494bcbf 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -406,7 +406,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -769,8 +769,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), - current.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + current.first.to_string(HashFormat::Base32, true)); status = 1; } } @@ -892,7 +892,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(Base32, true) + out << info->narHash.to_string(HashFormat::Base32, true) << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a77b5fcb8..ceb112c03 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -179,7 +179,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); + j["revision"] = rev->to_string(HashFormat::Base16, false); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) @@ -206,7 +206,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", - rev->to_string(Base16, false)); + rev->to_string(HashFormat::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -1345,7 +1345,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(tree.storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(HashFormat::SRI, true); res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); logger->cout(res.dump()); @@ -1353,7 +1353,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), store->printStorePath(tree.storePath), - hash.to_string(SRI, true)); + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 6af6bd19f..d6595dcca 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -11,7 +11,7 @@ using namespace nix; struct CmdHashBase : Command { FileIngestionMethod mode; - HashFormat hashFormat = SRI; + HashFormat hashFormat = HashFormat::SRI; bool truncate = false; HashType ht = htSHA256; std::vector paths; @@ -22,25 +22,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&hashFormat, SRI}, + .handler = {&hashFormat, HashFormat::SRI}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&hashFormat, Base64}, + .handler = {&hashFormat, HashFormat::Base64}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&hashFormat, Base32}, + .handler = {&hashFormat, HashFormat::Base32}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&hashFormat, Base16}, + .handler = {&hashFormat, HashFormat::Base16}, }); addFlag(Flag::mkHashTypeFlag("type", &ht)); @@ -94,7 +94,7 @@ struct CmdHashBase : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(hashFormat, hashFormat == SRI)); + logger->cout(h.to_string(hashFormat, hashFormat == HashFormat::SRI)); } } }; @@ -114,16 +114,16 @@ struct CmdToBase : Command std::string description() override { return fmt("convert a hash to %s representation", - hashFormat == Base16 ? "base-16" : - hashFormat == Base32 ? "base-32" : - hashFormat == Base64 ? "base-64" : + hashFormat == HashFormat::Base16 ? "base-16" : + hashFormat == HashFormat::Base32 ? "base-32" : + hashFormat == HashFormat::Base64 ? "base-64" : "SRI"); } void run() override { for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; @@ -133,10 +133,10 @@ struct CmdHash : NixMultiCommand : MultiCommand({ {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, - {"to-base16", []() { return make_ref(Base16); }}, - {"to-base32", []() { return make_ref(Base32); }}, - {"to-base64", []() { return make_ref(Base64); }}, - {"to-sri", []() { return make_ref(SRI); }}, + {"to-base16", []() { return make_ref(HashFormat::Base16); }}, + {"to-base32", []() { return make_ref(HashFormat::Base32); }}, + {"to-base64", []() { return make_ref(HashFormat::Base64); }}, + {"to-sri", []() { return make_ref(HashFormat::SRI); }}, }) { } @@ -162,7 +162,7 @@ static int compatNixHash(int argc, char * * argv) { std::optional ht; bool flat = false; - HashFormat hashFormat = Base16; + HashFormat hashFormat = HashFormat::Base16; bool truncate = false; enum { opHash, opTo } op = opHash; std::vector ss; @@ -173,10 +173,10 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base16") hashFormat = Base16; - else if (*arg == "--base32") hashFormat = Base32; - else if (*arg == "--base64") hashFormat = Base64; - else if (*arg == "--sri") hashFormat = SRI; + else if (*arg == "--base16") hashFormat = HashFormat::Base16; + else if (*arg == "--base32") hashFormat = HashFormat::Base32; + else if (*arg == "--base64") hashFormat = HashFormat::Base64; + else if (*arg == "--sri") hashFormat = HashFormat::SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); @@ -184,19 +184,19 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base16") { op = opTo; - hashFormat = Base16; + hashFormat = HashFormat::Base16; } else if (*arg == "--to-base32") { op = opTo; - hashFormat = Base32; + hashFormat = HashFormat::Base32; } else if (*arg == "--to-base64") { op = opTo; - hashFormat = Base64; + hashFormat = HashFormat::Base64; } else if (*arg == "--to-sri") { op = opTo; - hashFormat = SRI; + hashFormat = HashFormat::SRI; } else if (*arg != "" && arg->at(0) == '-') return false; diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 613c5b191..c16864d30 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -90,7 +90,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << store->pathInfoToJSON( // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, SRI, AllowInvalid).dump(); + true, showClosureSize, HashFormat::SRI, AllowInvalid).dump(); } else { diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b67d381ca..3ed7946a8 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -310,13 +310,13 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(HashFormat::SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", url, store->printStorePath(storePath), - hash.to_string(SRI, true)); + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 0b306cc11..adaa33c0c 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -108,8 +108,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + hash.first.to_string(HashFormat::Base32, true)); } } From 231b0fca6df668f904dfb86ef858db556caebdff Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 13 Oct 2023 18:48:36 +0800 Subject: [PATCH 033/100] Migrate HashFormat to scoped enumeration (enum struct) --- src/libutil/hash.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 783e30496..d2bf1cb23 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -26,7 +26,7 @@ extern const std::string base32Chars; /** * @brief Enumeration representing the hash formats. */ -enum HashFormat : int { +enum struct HashFormat : int { /// @brief Base 64 encoding. /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). Base64, From 6b47635180c740b7478f187b3ae9e35b36deeed7 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 9 Oct 2023 11:03:16 +0800 Subject: [PATCH 034/100] Add helper function parseHashFormat[Opt] printHashFormat Add hash format analogy of parseHashTypeOpt, parseHashType, and printHashType. Co-authored-by: Valentin Gagarin --- src/libutil/hash.cc | 35 +++++++++++++++++++++++++++++++++++ src/libutil/hash.hh | 15 +++++++++++++++ src/libutil/tests/hash.cc | 15 +++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index b58b5f08d..e297c245b 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -386,6 +386,41 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } +std::optional parseHashFormatOpt(std::string_view hashFormatName) +{ + if (hashFormatName == "base16") return HashFormat::Base16; + if (hashFormatName == "base32") return HashFormat::Base32; + if (hashFormatName == "base64") return HashFormat::Base64; + if (hashFormatName == "sri") return HashFormat::SRI; + return std::nullopt; +} + +HashFormat parseHashFormat(std::string_view hashFormatName) +{ + auto opt_f = parseHashFormatOpt(hashFormatName); + if (opt_f) + return *opt_f; + throw UsageError("unknown hash format '%1%', expect 'base16', 'base32', 'base64', or 'sri'", hashFormatName); +} + +std::string_view printHashFormat(HashFormat HashFormat) +{ + switch (HashFormat) { + case HashFormat::Base64: + return "base64"; + case HashFormat::Base32: + return "base32"; + case HashFormat::Base16: + return "base16"; + case HashFormat::SRI: + return "sri"; + default: + // illegal hash base enum value internally, as opposed to external input + // which should be validated with nice error message. + assert(false); + } +} + std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index d2bf1cb23..cab3e6eca 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -181,6 +181,21 @@ HashResult hashPath(HashType ht, const Path & path, */ Hash compressHash(const Hash & hash, unsigned int newSize); +/** + * Parse a string representing a hash format. + */ +HashFormat parseHashFormat(std::string_view hashFormatName); + +/** + * std::optional version of parseHashFormat that doesn't throw error. + */ +std::optional parseHashFormatOpt(std::string_view hashFormatName); + +/** + * The reverse of parseHashFormat. + */ +std::string_view printHashFormat(HashFormat hashFormat); + /** * Parse a string representing a hash type. */ diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index d14e5b26d..9a5ebbb30 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -79,6 +79,21 @@ namespace nix { "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); } + + /* ---------------------------------------------------------------------------- + * parseHashFormat, parseHashFormatOpt, printHashFormat + * --------------------------------------------------------------------------*/ + + TEST(hashFormat, testRoundTripPrintParse) { + for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) { + ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat); + ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat); + } + } + + TEST(hashFormat, testParseHashFormatOptException) { + ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); + } } namespace rc { From 5088e6563a4e88b7e6615623d3ed655fdebd3345 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li <44064051+ShamrockLee@users.noreply.github.com> Date: Sun, 29 Jan 2023 14:03:06 +0800 Subject: [PATCH 035/100] primops: add builtins.convertHash Co-authored-by: Valentin Gagarin --- doc/manual/src/release-notes/rl-next.md | 2 + src/libexpr/primops.cc | 95 +++++++++++++++++++ .../functional/lang/eval-okay-convertHash.exp | 1 + .../functional/lang/eval-okay-convertHash.nix | 31 ++++++ 4 files changed, 129 insertions(+) create mode 100644 tests/functional/lang/eval-okay-convertHash.exp create mode 100644 tests/functional/lang/eval-okay-convertHash.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c905f445f..46ce2abac 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,4 +8,6 @@ - Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. +- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). + - `builtins.fetchTree` is now marked as stable. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 830579dbf..14beb069f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3774,6 +3774,101 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); +static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash"); + auto &inputAttrs = args[0]->attrs; + + Bindings::iterator iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'"); + auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); + + Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo")); + std::optional ht = std::nullopt; + if (iteratorHashAlgo != inputAttrs->end()) { + ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); + } + + Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'"); + HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); + + v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI)); +} + +static RegisterPrimOp primop_convertHash({ + .name = "__convertHash", + .args = {"args"}, + .doc = R"( + Return the specified representation of a hash string, based on the attributes presented in *args*: + + - `hash` + + The hash to be converted. + The hash format is detected automatically. + + - `hashAlgo` + + The algorithm used to create the hash. Must be one of + - `"md5"` + - `"sha1"` + - `"sha256"` + - `"sha512"` + + The attribute may be omitted when `hash` is an [SRI hash](https://www.w3.org/TR/SRI/#the-integrity-attribute) or when the hash is prefixed with the hash algorithm name followed by a colon. + That `:` syntax is supported for backwards compatibility with existing tooling. + + - `toHashFormat` + + The format of the resulting hash. Must be one of + - `"base16"` + - `"base32"` + - `"base64"` + - `"sri"` + + The result hash is the *toHashFormat* representation of the hash *hash*. + + > **Example** + > + > Convert a SHA256 hash in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > hashAlgo = "sha256"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + + > **Example** + > + > Convert a SHA256 hash in SRI to Base16: + > + > ```nix + > builtins.convertHash { + > hash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; + > toHashFormat = "base16"; + > } + > ``` + > + > "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + > **Example** + > + > Convert a hash in the form `:` in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + )", + .fun = prim_convertHash, +}); + struct RegexCache { // TODO use C++20 transparent comparison when available diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp new file mode 100644 index 000000000..60e0a3c49 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.exp @@ -0,0 +1 @@ +{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix new file mode 100644 index 000000000..cf4909aaf --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.nix @@ -0,0 +1,31 @@ +let + hashAlgos = [ "md5" "md5" "md5" "sha1" "sha1" "sha1" "sha256" "sha256" "sha256" "sha512" "sha512" "sha512" ]; + hashesBase16 = import ./eval-okay-hashstring.exp; + map2 = f: { fsts, snds }: if fsts == [ ] then [ ] else [ (f (builtins.head fsts) (builtins.head snds)) ] ++ map2 f { fsts = builtins.tail fsts; snds = builtins.tail snds; }; + map2' = f: fsts: snds: map2 f { inherit fsts snds; }; + getOutputHashes = hashes: { + hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + getOutputHashesColon = hashes: { + hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + outputHashes = getOutputHashes hashesBase16; +in +# map2'` +assert map2' (s1: s2: s1 + s2) [ "a" "b" ] [ "c" "d" ] == [ "ac" "bd" ]; +# hashesBase16 +assert outputHashes.hashesBase16 == hashesBase16; +# standard SRI hashes +assert outputHashes.hashesSRI == (map2' (hashAlgo: hashBody: hashAlgo + "-" + hashBody) hashAlgos outputHashes.hashesBase64); +# without prefix +assert builtins.all (x: getOutputHashes x == outputHashes) (builtins.attrValues outputHashes); +# colon-separated. +# Note that colon prefix must not be applied to the standard SRI. e.g. "sha256:sha256-..." is illegal. +assert builtins.all (x: getOutputHashesColon x == outputHashes) (with outputHashes; [ hashesBase16 hashesBase32 hashesBase64 ]); +outputHashes From d2c0051784e4a338254445f8f680abf1e37e9024 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Oct 2023 23:35:07 +0200 Subject: [PATCH 036/100] Remove obsolete corepkgs references --- doc/manual/local.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 2e9f08678..8bf16e9dd 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -125,7 +125,7 @@ $(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $ @mv $@.tmp $@ $(d)/xp-features.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp @mv $@.tmp $@ $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @@ -141,7 +141,7 @@ $(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin @mv $@.tmp $@ $(d)/language.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ # Generate the HTML manual. From 34c559352579b9e1a501ec2fc435f3e21cf6e78f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Oct 2023 10:57:45 +0200 Subject: [PATCH 037/100] add a link to all maintainer meeting notes linking to the discourse category will by default show a view sorted by most recent post, which makes it hard to find particular meeting notes. this also adds a procedural detail about the notes, to make that more explicit and less dependent on being present in the meetings. --- maintainers/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 8eeb47e8b..5be4f9d04 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -50,7 +50,9 @@ The team meets twice a week: 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. -Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +[All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). ## Project board protocol From 9adac237e70003625b626cef04979ce746e9dd7e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Oct 2023 11:40:27 +0200 Subject: [PATCH 038/100] update link to label GitHub now displays a banner and has a dedicated page[1] for good first issues, but that uses a different label name as we had in place. I renamed the label on GitHub, this is updating the link. [1]: https://github.com/NixOS/nix/contribute --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73210b303..556365375 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 1. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. - Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. + Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue) should be relatively easy to fix and are likely to get merged quickly. Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. From 36b15d905e8ca6f5eb4f73fb842600b6e091708e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 19 Oct 2023 11:47:10 +0200 Subject: [PATCH 039/100] link to popular issues from the contributing guide this also adds a hint to contributors about making far-reaching changes, complementing the recent update to the maintainers' handbook on how to deal with those. --- CONTRIBUTING.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73210b303..71970f307 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,9 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. + If you are proficient with C++, addressing one of the [popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) will be highly appreciated by maintainers and Nix users all over the world. + For far-reaching changes, please investigate possible blockers and design implications, and coordinate with maintainers before investing too much time in writing code that may not end up getting merged. + If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. 2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. From 12214fef09151e26abfcaea0095ab157388822e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:19:10 +0200 Subject: [PATCH 040/100] InputAccessor::fetchToStore(): Support arbitrary ingestion methods --- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 5 +---- src/libfetchers/input-accessor.cc | 8 +++++--- src/libfetchers/input-accessor.hh | 3 +++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a28bb2101..3b0fe93e1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2329,7 +2329,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); + auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f1c24a75c..f416aa639 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2233,10 +2233,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - // FIXME - if (method != FileIngestionMethod::Recursive) - throw Error("'recursive = false' is not implemented"); - auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, &filter, state.repair); + auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index fec16e99f..63f1f4ad2 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -96,6 +96,7 @@ StorePath InputAccessor::fetchToStore( ref store, const CanonPath & path, std::string_view name, + FileIngestionMethod method, PathFilter * filter, RepairFlag repair) { @@ -107,8 +108,8 @@ StorePath InputAccessor::fetchToStore( auto storePath = settings.readOnlyMode - ? store->computeStorePathFromDump(*source, name).first - : store->addToStoreFromDump(*source, name, FileIngestionMethod::Recursive, htSHA256, repair); + ? store->computeStorePathFromDump(*source, name, method, htSHA256).first + : store->addToStoreFromDump(*source, name, method, htSHA256, repair); return storePath; } @@ -140,10 +141,11 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path) StorePath SourcePath::fetchToStore( ref store, std::string_view name, + FileIngestionMethod method, PathFilter * filter, RepairFlag repair) const { - return accessor->fetchToStore(store, path, name, filter, repair); + return accessor->fetchToStore(store, path, name, method, filter, repair); } std::string_view SourcePath::baseName() const diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index a0f08e295..003c7226c 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -6,6 +6,7 @@ #include "canon-path.hh" #include "repair-flag.hh" #include "hash.hh" +#include "content-address.hh" namespace nix { @@ -73,6 +74,7 @@ struct InputAccessor : public std::enable_shared_from_this ref store, const CanonPath & path, std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); @@ -181,6 +183,7 @@ struct SourcePath StorePath fetchToStore( ref store, std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair) const; From f16af08e8393420f6234ab59bde716dd145703d7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:20:50 +0200 Subject: [PATCH 041/100] Fix macOS compilation --- src/libexpr/eval.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3b0fe93e1..7180b9fbe 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -616,7 +616,9 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & SourcePath EvalState::checkSourcePath(const SourcePath & path_) { - if (path_.accessor != rootFS) return path_; + // Don't check non-rootFS accessors, they're in a different namespace. + if (path_.accessor != ref(rootFS)) return path_; + if (!allowedPaths) return path_; auto i = resolvedPaths.find(path_.path.abs()); From 06c57899e306bd6a19fce5d83b2ffce487067cdb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:22:05 +0200 Subject: [PATCH 042/100] Remove FIXME --- src/libexpr/primops.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f416aa639..b971fd052 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -595,7 +595,9 @@ struct CompareValues case nString: return v1->string_view().compare(v2->string_view()) < 0; case nPath: - // FIXME: handle accessor? + // Note: we don't take the accessor into account + // since it's not obvious how to compare them in a + // reproducible way. return strcmp(v1->_path.path, v2->_path.path) < 0; case nList: // Lexicographic comparison From 9bc7b4f463b6a06746bd1658d0170865e3fc8337 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 19 Oct 2023 14:39:41 +0200 Subject: [PATCH 043/100] doc: generic closure supported key types (#9183) * doc: generic closure supported key types Co-authored-by: Valentin Gagarin --- src/libexpr/primops.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 14beb069f..e5c5eb5b1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -734,6 +734,14 @@ static RegisterPrimOp primop_genericClosure(PrimOp { ``` [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] ``` + + `key` can be one of the following types: + - [Number](@docroot@/language/values.md#type-number) + - [Boolean](@docroot@/language/values.md#type-boolean) + - [String](@docroot@/language/values.md#type-string) + - [Path](@docroot@/language/values.md#type-path) + - [List](@docroot@/language/values.md#list) + )", .fun = prim_genericClosure, }); From fb6a3910c4b4ebec2ab2e422a7454b8f3ba71f85 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 14:45:26 +0200 Subject: [PATCH 044/100] Move most of InputAccessor into libutil --- src/libfetchers/input-accessor.cc | 102 ---------------------------- src/libfetchers/input-accessor.hh | 85 +---------------------- src/libutil/source-accessor.cc | 108 ++++++++++++++++++++++++++++++ src/libutil/source-accessor.hh | 96 ++++++++++++++++++++++++++ 4 files changed, 206 insertions(+), 185 deletions(-) create mode 100644 src/libutil/source-accessor.cc create mode 100644 src/libutil/source-accessor.hh diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 63f1f4ad2..9bddf7c2e 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -3,95 +3,6 @@ namespace nix { -static std::atomic nextNumber{0}; - -InputAccessor::InputAccessor() - : number(++nextNumber) -{ -} - -// FIXME: merge with archive.cc. -void InputAccessor::dumpPath( - const CanonPath & path, - Sink & sink, - PathFilter & filter) -{ - auto dumpContents = [&](const CanonPath & path) - { - // FIXME: pipe - auto s = readFile(path); - sink << "contents" << s.size(); - sink(s); - writePadding(s.size(), sink); - }; - - std::function dump; - - dump = [&](const CanonPath & path) { - checkInterrupt(); - - auto st = lstat(path); - - sink << "("; - - if (st.type == tRegular) { - sink << "type" << "regular"; - if (st.isExecutable) - sink << "executable" << ""; - dumpContents(path); - } - - else if (st.type == tDirectory) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (/* archiveSettings.useCaseHack */ false) { // FIXME - std::string name(i.first); - size_t pos = i.first.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%s'", path + i.first); - name.erase(pos); - } - if (!unhacked.emplace(name, i.first).second) - throw Error("file name collision in between '%s' and '%s'", - (path + unhacked[name]), - (path + i.first)); - } else - unhacked.emplace(i.first, i.first); - - for (auto & i : unhacked) - if (filter((path + i.first).abs())) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + i.second); - sink << ")"; - } - } - - else if (st.type == tSymlink) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error("file '%s' has an unsupported type", path); - - sink << ")"; - }; - - sink << narVersionMagic1; - dump(path); -} - -Hash InputAccessor::hashPath( - const CanonPath & path, - PathFilter & filter, - HashType ht) -{ - HashSink sink(ht); - dumpPath(path, sink, filter); - return sink.finish().first; -} - StorePath InputAccessor::fetchToStore( ref store, const CanonPath & path, @@ -114,19 +25,6 @@ StorePath InputAccessor::fetchToStore( return storePath; } -std::optional InputAccessor::maybeLstat(const CanonPath & path) -{ - // FIXME: merge these into one operation. - if (!pathExists(path)) - return {}; - return lstat(path); -} - -std::string InputAccessor::showPath(const CanonPath & path) -{ - return path.abs(); -} - SourcePath InputAccessor::root() { return {ref(shared_from_this()), CanonPath::root}; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 003c7226c..524a94cd1 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -1,11 +1,9 @@ #pragma once +#include "source-accessor.hh" #include "ref.hh" #include "types.hh" -#include "archive.hh" -#include "canon-path.hh" #include "repair-flag.hh" -#include "hash.hh" #include "content-address.hh" namespace nix { @@ -16,60 +14,8 @@ struct SourcePath; class StorePath; class Store; -struct InputAccessor : public std::enable_shared_from_this +struct InputAccessor : SourceAccessor, std::enable_shared_from_this { - const size_t number; - - InputAccessor(); - - virtual ~InputAccessor() - { } - - virtual std::string readFile(const CanonPath & path) = 0; - - virtual bool pathExists(const CanonPath & path) = 0; - - enum Type { - tRegular, tSymlink, tDirectory, - /** - Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. - - Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. - - Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. - */ - tMisc - }; - - struct Stat - { - Type type = tMisc; - //uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only - }; - - virtual Stat lstat(const CanonPath & path) = 0; - - std::optional maybeLstat(const CanonPath & path); - - typedef std::optional DirEntry; - - typedef std::map DirEntries; - - virtual DirEntries readDirectory(const CanonPath & path) = 0; - - virtual std::string readLink(const CanonPath & path) = 0; - - virtual void dumpPath( - const CanonPath & path, - Sink & sink, - PathFilter & filter = defaultPathFilter); - - Hash hashPath( - const CanonPath & path, - PathFilter & filter = defaultPathFilter, - HashType ht = htSHA256); - StorePath fetchToStore( ref store, const CanonPath & path, @@ -78,34 +24,7 @@ struct InputAccessor : public std::enable_shared_from_this PathFilter * filter = nullptr, RepairFlag repair = NoRepair); - /* Return a corresponding path in the root filesystem, if - possible. This is only possible for inputs that are - materialized in the root filesystem. */ - virtual std::optional getPhysicalPath(const CanonPath & path) - { return std::nullopt; } - - bool operator == (const InputAccessor & x) const - { - return number == x.number; - } - - bool operator < (const InputAccessor & x) const - { - return number < x.number; - } - - void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); - - virtual std::string showPath(const CanonPath & path); - SourcePath root(); - - /* Return the maximum last-modified time of the files in this - tree, if available. */ - virtual std::optional getLastModified() - { - return std::nullopt; - } }; /** diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc new file mode 100644 index 000000000..23ec55c89 --- /dev/null +++ b/src/libutil/source-accessor.cc @@ -0,0 +1,108 @@ +#include "source-accessor.hh" +#include "archive.hh" + +namespace nix { + +static std::atomic nextNumber{0}; + +SourceAccessor::SourceAccessor() + : number(++nextNumber) +{ +} + +// FIXME: merge with archive.cc. +void SourceAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) +{ + auto dumpContents = [&](const CanonPath & path) + { + // FIXME: pipe + auto s = readFile(path); + sink << "contents" << s.size(); + sink(s); + writePadding(s.size(), sink); + }; + + std::function dump; + + dump = [&](const CanonPath & path) { + checkInterrupt(); + + auto st = lstat(path); + + sink << "("; + + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } + + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (/* archiveSettings.useCaseHack */ false) { // FIXME + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); + + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; + } + } + + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error("file '%s' has an unsupported type", path); + + sink << ")"; + }; + + sink << narVersionMagic1; + dump(path); +} + +Hash SourceAccessor::hashPath( + const CanonPath & path, + PathFilter & filter, + HashType ht) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish().first; +} + +std::optional SourceAccessor::maybeLstat(const CanonPath & path) +{ + // FIXME: merge these into one operation. + if (!pathExists(path)) + return {}; + return lstat(path); +} + +std::string SourceAccessor::showPath(const CanonPath & path) +{ + return path.abs(); +} + +} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh new file mode 100644 index 000000000..c2d35d6b5 --- /dev/null +++ b/src/libutil/source-accessor.hh @@ -0,0 +1,96 @@ +#pragma once + +#include "canon-path.hh" +#include "hash.hh" + +namespace nix { + +/** + * A read-only filesystem abstraction. This is used by the Nix + * evaluator and elsewhere for accessing sources in various + * filesystem-like entities (such as the real filesystem, tarballs or + * Git repositories). + */ +struct SourceAccessor +{ + const size_t number; + + SourceAccessor(); + + virtual ~SourceAccessor() + { } + + virtual std::string readFile(const CanonPath & path) = 0; + + virtual bool pathExists(const CanonPath & path) = 0; + + enum Type { + tRegular, tSymlink, tDirectory, + /** + Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. + + Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. + + Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. + */ + tMisc + }; + + struct Stat + { + Type type = tMisc; + //uint64_t fileSize = 0; // regular files only + bool isExecutable = false; // regular files only + }; + + virtual Stat lstat(const CanonPath & path) = 0; + + std::optional maybeLstat(const CanonPath & path); + + typedef std::optional DirEntry; + + typedef std::map DirEntries; + + virtual DirEntries readDirectory(const CanonPath & path) = 0; + + virtual std::string readLink(const CanonPath & path) = 0; + + virtual void dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter = defaultPathFilter); + + Hash hashPath( + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashType ht = htSHA256); + + /* Return a corresponding path in the root filesystem, if + possible. This is only possible for filesystems that are + materialized in the root filesystem. */ + virtual std::optional getPhysicalPath(const CanonPath & path) + { return std::nullopt; } + + bool operator == (const SourceAccessor & x) const + { + return number == x.number; + } + + bool operator < (const SourceAccessor & x) const + { + return number < x.number; + } + + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + + virtual std::string showPath(const CanonPath & path); + + /* Return the maximum last-modified time of the files in this + tree, if available. */ + virtual std::optional getLastModified() + { + return std::nullopt; + } +}; + +} From 9f572eb0e35d3017a160cef89b7417cf6b4a53e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 15:07:56 +0200 Subject: [PATCH 045/100] Unify the two implementations of dumpPath() --- src/libutil/archive.cc | 183 +++++++++++++++++++-------------- src/libutil/source-accessor.cc | 72 ------------- 2 files changed, 108 insertions(+), 147 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 0f0cae7c0..e48fc7522 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -14,6 +14,7 @@ #include "archive.hh" #include "util.hh" #include "config.hh" +#include "source-accessor.hh" namespace nix { @@ -36,91 +37,134 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, off_t size, - Sink & sink) +void SourceAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) { - sink << "contents" << size; + auto dumpContents = [&](const CanonPath & path) + { + /* It would be nice if this was streaming, but we need the + size before the contents. */ + auto s = readFile(path); + sink << "contents" << s.size(); + sink(s); + writePadding(s.size(), sink); + }; - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError("opening file '%1%'", path); + std::function dump; - std::vector buf(65536); - size_t left = size; + dump = [&](const CanonPath & path) { + checkInterrupt(); - while (left > 0) { - auto n = std::min(left, buf.size()); - readFull(fd.get(), buf.data(), n); - left -= n; - sink({buf.data(), n}); - } + auto st = lstat(path); - writePadding(size, sink); + sink << "("; + + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } + + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (archiveSettings.useCaseHack) { + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); + + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; + } + } + + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error("file '%s' has an unsupported type", path); + + sink << ")"; + }; + + sink << narVersionMagic1; + dump(path); } -static time_t dump(const Path & path, Sink & sink, PathFilter & filter) +struct FSSourceAccessor : SourceAccessor { - checkInterrupt(); + time_t mtime = 0; // most recent mtime seen - auto st = lstat(path); - time_t result = st.st_mtime; - - sink << "("; - - if (S_ISREG(st.st_mode)) { - sink << "type" << "regular"; - if (st.st_mode & S_IXUSR) - sink << "executable" << ""; - dumpContents(path, st.st_size, sink); + std::string readFile(const CanonPath & path) override + { + return nix::readFile(path.abs()); } - else if (S_ISDIR(st.st_mode)) { - sink << "type" << "directory"; + bool pathExists(const CanonPath & path) override + { + return nix::pathExists(path.abs()); + } - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (archiveSettings.useCaseHack) { - std::string name(i.name); - size_t pos = i.name.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%1%'", path + "/" + i.name); - name.erase(pos); - } - if (!unhacked.emplace(name, i.name).second) - throw Error("file name collision in between '%1%' and '%2%'", - (path + "/" + unhacked[name]), - (path + "/" + i.name)); - } else - unhacked.emplace(i.name, i.name); + Stat lstat(const CanonPath & path) override + { + auto st = nix::lstat(path.abs()); + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; + } - for (auto & i : unhacked) - if (filter(path + "/" + i.first)) { - sink << "entry" << "(" << "name" << i.first << "node"; - auto tmp_mtime = dump(path + "/" + i.second, sink, filter); - if (tmp_mtime > result) { - result = tmp_mtime; - } - sink << ")"; + DirEntries readDirectory(const CanonPath & path) override + { + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; } + res.emplace(entry.name, type); + } + return res; } - else if (S_ISLNK(st.st_mode)) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error("file '%1%' has an unsupported type", path); - - sink << ")"; - - return result; -} + std::string readLink(const CanonPath & path) override + { + return nix::readLink(path.abs()); + } +}; time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - sink << narVersionMagic1; - return dump(path, sink, filter); + FSSourceAccessor accessor; + accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); + return accessor.mtime; } void dumpPath(const Path & path, Sink & sink, PathFilter & filter) @@ -141,17 +185,6 @@ static SerialisationError badArchive(const std::string & s) } -#if 0 -static void skipGeneric(Source & source) -{ - if (readString(source) == "(") { - while (readString(source) != ")") - skipGeneric(source); - } -} -#endif - - static void parseContents(ParseSink & sink, Source & source, const Path & path) { uint64_t size = readLongLong(source); diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 23ec55c89..e3adee5f1 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -10,78 +10,6 @@ SourceAccessor::SourceAccessor() { } -// FIXME: merge with archive.cc. -void SourceAccessor::dumpPath( - const CanonPath & path, - Sink & sink, - PathFilter & filter) -{ - auto dumpContents = [&](const CanonPath & path) - { - // FIXME: pipe - auto s = readFile(path); - sink << "contents" << s.size(); - sink(s); - writePadding(s.size(), sink); - }; - - std::function dump; - - dump = [&](const CanonPath & path) { - checkInterrupt(); - - auto st = lstat(path); - - sink << "("; - - if (st.type == tRegular) { - sink << "type" << "regular"; - if (st.isExecutable) - sink << "executable" << ""; - dumpContents(path); - } - - else if (st.type == tDirectory) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (/* archiveSettings.useCaseHack */ false) { // FIXME - std::string name(i.first); - size_t pos = i.first.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%s'", path + i.first); - name.erase(pos); - } - if (!unhacked.emplace(name, i.first).second) - throw Error("file name collision in between '%s' and '%s'", - (path + unhacked[name]), - (path + i.first)); - } else - unhacked.emplace(i.first, i.first); - - for (auto & i : unhacked) - if (filter((path + i.first).abs())) { - sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + i.second); - sink << ")"; - } - } - - else if (st.type == tSymlink) - sink << "type" << "symlink" << "target" << readLink(path); - - else throw Error("file '%s' has an unsupported type", path); - - sink << ")"; - }; - - sink << narVersionMagic1; - dump(path); -} - Hash SourceAccessor::hashPath( const CanonPath & path, PathFilter & filter, From 50156302c033323895ababb6c190086cbc5cec46 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 15:20:10 +0200 Subject: [PATCH 046/100] Deduplicate FSSourceAccessor and FSInputAccessor --- src/libfetchers/fs-input-accessor.cc | 31 +++++------------ src/libutil/archive.cc | 52 +--------------------------- src/libutil/source-accessor.cc | 49 ++++++++++++++++++++++++++ src/libutil/source-accessor.hh | 24 +++++++++++++ 4 files changed, 82 insertions(+), 74 deletions(-) diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index a35955465..e40faf03f 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -3,7 +3,7 @@ namespace nix { -struct FSInputAccessorImpl : FSInputAccessor +struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor { CanonPath root; std::optional> allowedPaths; @@ -23,28 +23,20 @@ struct FSInputAccessorImpl : FSInputAccessor { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return nix::readFile(absPath.abs()); + return PosixSourceAccessor::readFile(absPath); } bool pathExists(const CanonPath & path) override { auto absPath = makeAbsPath(path); - return isAllowed(absPath) && nix::pathExists(absPath.abs()); + return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); } Stat lstat(const CanonPath & path) override { auto absPath = makeAbsPath(path); checkAllowed(absPath); - auto st = nix::lstat(absPath.abs()); - return Stat { - .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : - tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; + return PosixSourceAccessor::lstat(absPath); } DirEntries readDirectory(const CanonPath & path) override @@ -52,16 +44,9 @@ struct FSInputAccessorImpl : FSInputAccessor auto absPath = makeAbsPath(path); checkAllowed(absPath); DirEntries res; - for (auto & entry : nix::readDirectory(absPath.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = Type::tRegular; break; - case DT_LNK: type = Type::tSymlink; break; - case DT_DIR: type = Type::tDirectory; break; - } - if (isAllowed(absPath + entry.name)) - res.emplace(entry.name, type); - } + for (auto & entry : PosixSourceAccessor::readDirectory(absPath)) + if (isAllowed(absPath + entry.first)) + res.emplace(entry); return res; } @@ -69,7 +54,7 @@ struct FSInputAccessorImpl : FSInputAccessor { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return nix::readLink(absPath.abs()); + return PosixSourceAccessor::readLink(absPath); } CanonPath makeAbsPath(const CanonPath & path) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index e48fc7522..6508ba807 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -110,59 +110,9 @@ void SourceAccessor::dumpPath( } -struct FSSourceAccessor : SourceAccessor -{ - time_t mtime = 0; // most recent mtime seen - - std::string readFile(const CanonPath & path) override - { - return nix::readFile(path.abs()); - } - - bool pathExists(const CanonPath & path) override - { - return nix::pathExists(path.abs()); - } - - Stat lstat(const CanonPath & path) override - { - auto st = nix::lstat(path.abs()); - mtime = std::max(mtime, st.st_mtime); - return Stat { - .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : - tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; - } - - DirEntries readDirectory(const CanonPath & path) override - { - DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = Type::tRegular; break; - case DT_LNK: type = Type::tSymlink; break; - case DT_DIR: type = Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; - } - - std::string readLink(const CanonPath & path) override - { - return nix::readLink(path.abs()); - } -}; - - time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - FSSourceAccessor accessor; + PosixSourceAccessor accessor; accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); return accessor.mtime; } diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index e3adee5f1..d5c8cbcdd 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -33,4 +33,53 @@ std::string SourceAccessor::showPath(const CanonPath & path) return path.abs(); } +std::string PosixSourceAccessor::readFile(const CanonPath & path) +{ + return nix::readFile(path.abs()); +} + +bool PosixSourceAccessor::pathExists(const CanonPath & path) +{ + return nix::pathExists(path.abs()); +} + +SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +{ + auto st = nix::lstat(path.abs()); + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; +} + +SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +{ + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +std::string PosixSourceAccessor::readLink(const CanonPath & path) +{ + return nix::readLink(path.abs()); +} + +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + return path; +} + } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index c2d35d6b5..a251ae31c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -93,4 +93,28 @@ struct SourceAccessor } }; +/** + * A source accessor that uses the Unix filesystem. + */ +struct PosixSourceAccessor : SourceAccessor +{ + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + std::string readFile(const CanonPath & path) override; + + bool pathExists(const CanonPath & path) override; + + Stat lstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; +}; + } From 5be7705ddf9397846ca9391d464af1cc27110f06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Oct 2023 19:20:21 +0200 Subject: [PATCH 047/100] Remove stuff we don't need yet --- src/libexpr/eval.cc | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7180b9fbe..8235b83c5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -505,18 +505,7 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) - , rootFS( - makeFSInputAccessor( - CanonPath::root, - evalSettings.restrictEval || evalSettings.pureEval - ? std::optional>(std::set()) - : std::nullopt, - [](const CanonPath & path) -> RestrictedPathError { - auto modeInformation = evalSettings.pureEval - ? "in pure evaluation mode (use '--impure' to override)" - : "in restricted mode"; - throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); - })) + , rootFS(makeFSInputAccessor(CanonPath::root)) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -541,9 +530,6 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { - // For now, don't rely on FSInputAccessor for access control. - rootFS->allowPath(CanonPath::root); - countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); From 87c4f4a972ce732962cc0881d848d8bf979893c2 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 19 Oct 2023 18:24:13 +0100 Subject: [PATCH 048/100] libutil: Move some non-template implememntations from config.hh to config.cc --- src/libutil/config.cc | 45 +++++++++++++++++++++++++++++++++++++++++++ src/libutil/config.hh | 31 +++++++---------------------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8672edaa8..f5c7fb7d5 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -9,6 +9,10 @@ namespace nix { +Config::Config(StringMap initials) + : AbstractConfig(std::move(initials)) +{ } + bool Config::set(const std::string & name, const std::string & value) { bool append = false; @@ -59,6 +63,10 @@ void Config::addSetting(AbstractSetting * setting) } } +AbstractConfig::AbstractConfig(StringMap initials) + : unknownSettings(std::move(initials)) +{ } + void AbstractConfig::warnUnknownSettings() { for (auto & s : unknownSettings) @@ -199,6 +207,13 @@ AbstractSetting::AbstractSetting( { } +AbstractSetting::~AbstractSetting() +{ + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); +} + nlohmann::json AbstractSetting::toJSON() { return nlohmann::json(toJSONObject()); @@ -220,6 +235,9 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } + +bool AbstractSetting::isOverridden() const { return overridden; } + template<> std::string BaseSetting::parse(const std::string & str) const { return str; @@ -385,11 +403,33 @@ static Path parsePath(const AbstractSetting & s, const std::string & str) return canonPath(str); } +PathSetting::PathSetting(Config * options, + const Path & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting(def, true, name, description, aliases) +{ + options->addSetting(this); +} + Path PathSetting::parse(const std::string & str) const { return parsePath(*this, str); } + +OptionalPathSetting::OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting>(def, true, name, description, aliases) +{ + options->addSetting(this); +} + + std::optional OptionalPathSetting::parse(const std::string & str) const { if (str == "") @@ -398,6 +438,11 @@ std::optional OptionalPathSetting::parse(const std::string & str) const return parsePath(*this, str); } +void OptionalPathSetting::operator =(const std::optional & v) +{ + this->assign(v); +} + bool GlobalConfig::set(const std::string & name, const std::string & value) { for (auto & config : *configRegistrations) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 96c2cd75d..47c7be473 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -52,9 +52,7 @@ class AbstractConfig protected: StringMap unknownSettings; - AbstractConfig(const StringMap & initials = {}) - : unknownSettings(initials) - { } + AbstractConfig(StringMap initials = {}); public: @@ -163,9 +161,7 @@ private: public: - Config(const StringMap & initials = {}) - : AbstractConfig(initials) - { } + Config(StringMap initials = {}); bool set(const std::string & name, const std::string & value) override; @@ -206,12 +202,7 @@ protected: const std::set & aliases, std::optional experimentalFeature = std::nullopt); - virtual ~AbstractSetting() - { - // Check against a gcc miscompilation causing our constructor - // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). - assert(created == 123); - } + virtual ~AbstractSetting(); virtual void set(const std::string & value, bool append = false) = 0; @@ -229,7 +220,7 @@ protected: virtual void convertToArg(Args & args, const std::string & category); - bool isOverridden() const { return overridden; } + bool isOverridden() const; }; /** @@ -365,11 +356,7 @@ public: const Path & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); Path parse(const std::string & str) const override; @@ -391,15 +378,11 @@ public: const std::optional & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting>(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); std::optional parse(const std::string & str) const override; - void operator =(const std::optional & v) { this->assign(v); } + void operator =(const std::optional & v); }; struct GlobalConfig : public AbstractConfig From 55f06b6f30fa212d48cfd6d121a845863fb2980b Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 19 Oct 2023 18:24:54 +0100 Subject: [PATCH 049/100] libutil: Remove non-needed constructor --- src/libutil/config.cc | 4 ++-- src/libutil/config.hh | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index f5c7fb7d5..8e06273ee 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -33,9 +33,9 @@ bool Config::set(const std::string & name, const std::string & value) void Config::addSetting(AbstractSetting * setting) { - _settings.emplace(setting->name, Config::SettingData(false, setting)); + _settings.emplace(setting->name, Config::SettingData{false, setting}); for (auto & alias : setting->aliases) - _settings.emplace(alias, Config::SettingData(true, setting)); + _settings.emplace(alias, Config::SettingData{true, setting}); bool set = false; diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 47c7be473..f09a9b0b8 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -148,9 +148,6 @@ public: { bool isAlias; AbstractSetting * setting; - SettingData(bool isAlias, AbstractSetting * setting) - : isAlias(isAlias), setting(setting) - { } }; typedef std::map Settings; From b0f4ac29d3c0534b91fff7b37191d4d7d5a96395 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 19 Oct 2023 18:25:17 +0100 Subject: [PATCH 050/100] libutil: Use c++ style cast --- src/libutil/config.hh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index f09a9b0b8..38c3ce0c4 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -312,8 +312,7 @@ public: template std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) { - str << (const T &) opt; - return str; + return str << static_cast(opt); } template From 42f26eb42e3b115f39570bb14445d7e5f33220d7 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 20 Oct 2023 02:45:47 +0200 Subject: [PATCH 051/100] doc: complexity for '?' operator (#9184) Co-authored-by: Valentin Gagarin Co-authored-by: Robert Hensing --- doc/manual/src/language/operators.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index a22c71109..07b43a881 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -63,6 +63,8 @@ The result is a [Boolean] value. [Has attribute]: #has-attribute +After evaluating *attrset* and *attrpath*, the computational complexity is O(log(*n*)) for *n* attributes in the *attrset* + ## Arithmetic Numbers are type-compatible: From e58566a057692bbfbd30a6639248adfcf81df108 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 20 Oct 2023 05:11:03 +0200 Subject: [PATCH 052/100] doc: add reference to hasAttr in `?` operator (#9185) Co-authored-by: Valentin Gagarin --- doc/manual/src/language/operators.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 07b43a881..cc825b4cf 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -59,6 +59,8 @@ An attribute path is a dot-separated list of [attribute names](./values.md#attri Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. The result is a [Boolean] value. +See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) + [Boolean]: ./values.md#type-boolean [Has attribute]: #has-attribute From 9277eb276bf0a942e88fcf499f6a6b9c262be853 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 19 Oct 2023 16:59:35 +0200 Subject: [PATCH 053/100] libstore: Add apple-virt to system features when available I'm sure that we'll adjust the implementation over time, but this at least discerns between an apple silicon bare metal machine and a tart VM. --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libstore/globals.cc | 31 +++++++++++++++++++++++++ src/libstore/globals.hh | 4 ++++ 3 files changed, 37 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 46ce2abac..d0b516dfb 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,6 +8,8 @@ - Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. +- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. + - Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). - `builtins.fetchTree` is now marked as stable. diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5a4cb1824..9c25d9868 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -24,6 +24,9 @@ #include "config-impl.hh" +#ifdef __APPLE__ +#include +#endif namespace nix { @@ -154,6 +157,29 @@ unsigned int Settings::getDefaultCores() return concurrency; } +#if __APPLE__ +static bool hasVirt() { + + int hasVMM; + int hvSupport; + size_t size; + + size = sizeof(hasVMM); + if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) { + if (hasVMM) + return false; + } + + // whether the kernel and hardware supports virt + size = sizeof(hvSupport); + if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) { + return hvSupport == 1; + } else { + return false; + } +} +#endif + StringSet Settings::getDefaultSystemFeatures() { /* For backwards compatibility, accept some "features" that are @@ -170,6 +196,11 @@ StringSet Settings::getDefaultSystemFeatures() features.insert("kvm"); #endif + #if __APPLE__ + if (hasVirt()) + features.insert("apple-virt"); + #endif + return features; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 5ec332417..ff7d3df1e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -714,6 +714,10 @@ public: System features are user-defined, but Nix sets the following defaults: + - `apple-virt` + + Included on darwin if virtualization is available. + - `kvm` Included by default if `/dev/kvm` is accessible. From bb645c5d02025158c8dbb0c2a60e7cba8251f062 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 20 Oct 2023 10:19:07 +0200 Subject: [PATCH 054/100] system-features doc: kvm is Linux-only --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ff7d3df1e..e90f70f5f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -720,7 +720,7 @@ public: - `kvm` - Included by default if `/dev/kvm` is accessible. + Included on Linux if `/dev/kvm` is accessible. - `nixos-test`, `benchmark`, `big-parallel` From df10dc630fd13082e13400ec46acefe4ad85a715 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 12:36:18 +0200 Subject: [PATCH 055/100] Doxygen Co-authored-by: John Ericson --- src/libutil/source-accessor.hh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index a251ae31c..4074b2e63 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -65,9 +65,11 @@ struct SourceAccessor PathFilter & filter = defaultPathFilter, HashType ht = htSHA256); - /* Return a corresponding path in the root filesystem, if - possible. This is only possible for filesystems that are - materialized in the root filesystem. */ + /** + * Return a corresponding path in the root filesystem, if + * possible. This is only possible for filesystems that are + * materialized in the root filesystem. + */ virtual std::optional getPhysicalPath(const CanonPath & path) { return std::nullopt; } From bacceaea919ddb26d1882fec16462cdcb4e1d740 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 12:40:46 +0200 Subject: [PATCH 056/100] Move getLastModified(), remove setPathDisplay() --- src/libfetchers/input-accessor.hh | 9 +++++++++ src/libutil/source-accessor.hh | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 524a94cd1..5dc05a363 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -16,6 +16,15 @@ class Store; struct InputAccessor : SourceAccessor, std::enable_shared_from_this { + /** + * Return the maximum last-modified time of the files in this + * tree, if available. + */ + virtual std::optional getLastModified() + { + return std::nullopt; + } + StorePath fetchToStore( ref store, const CanonPath & path, diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 4074b2e63..53408eb6c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -83,16 +83,7 @@ struct SourceAccessor return number < x.number; } - void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); - virtual std::string showPath(const CanonPath & path); - - /* Return the maximum last-modified time of the files in this - tree, if available. */ - virtual std::optional getLastModified() - { - return std::nullopt; - } }; /** From 173abec0bce03016b001c415822793a309c40e0b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 13:04:39 +0200 Subject: [PATCH 057/100] coerceToPath(): Handle __toString, add tests --- src/libexpr/eval.cc | 27 ++++++++++--------- src/libexpr/tests/error_traces.cc | 8 +++--- .../functional/lang/eval-okay-pathexists.nix | 2 ++ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8235b83c5..e3ecb987c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2335,29 +2335,32 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext { try { forceValue(v, pos); - - if (v.type() == nString) { - copyContext(v, context); - auto s = v.string_view(); - if (!hasPrefix(s, "/")) - error("string '%s' doesn't represent an absolute path", s).atPos(pos).debugThrow(); - return rootPath(CanonPath(s)); - } } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; } + /* Handle path values directly, without coercing to a string. */ if (v.type() == nPath) return v.path(); + /* Similarly, handle __toString where the result may be a path + value. */ if (v.type() == nAttrs) { - auto i = v.attrs->find(sOutPath); - if (i != v.attrs->end()) - return coerceToPath(pos, *i->value, context, errorCtx); + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToPath(pos, v1, context, errorCtx); + } } - error("cannot coerce %1% to a path", showType(v)).withTrace(pos, errorCtx).debugThrow(); + /* Any other value should be coercable to a string, interpreted + relative to the root filesystem. */ + auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); + if (path == "" || path[0] != '/') + error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); + return rootPath(CanonPath(path)); } diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 6da8a5954..9cd9f0a60 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,7 +309,7 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a path", "a Boolean"), + hintfmt("cannot coerce %s to a string", "a Boolean"), hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -377,7 +377,7 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a path", "a list"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix index c5e7a62de..31697f66a 100644 --- a/tests/functional/lang/eval-okay-pathexists.nix +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -25,5 +25,7 @@ builtins.pathExists (./lib.nix) && builtins.pathExists (builtins.toString ./. + "/../lang/..//") && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) +&& builtins.pathExists (builtins.toPath { __toString = x: builtins.toString ./lib.nix; }) +&& builtins.pathExists (builtins.toPath { outPath = builtins.toString ./lib.nix; }) && builtins.pathExists ./lib.nix && !builtins.pathExists ./bla.nix From 7a086a32bce3338bdc1ce669a51599a99bc17b9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 13:32:15 +0200 Subject: [PATCH 058/100] fetchToStore(): Handle flat ingestion method and add test --- src/libfetchers/input-accessor.cc | 5 ++++- tests/functional/lang/eval-okay-path.exp | 2 +- tests/functional/lang/eval-okay-path.nix | 22 +++++++++++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 9bddf7c2e..488350849 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -14,7 +14,10 @@ StorePath InputAccessor::fetchToStore( Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); auto source = sinkToSource([&](Sink & sink) { - dumpPath(path, sink, filter ? *filter : defaultPathFilter); + if (method == FileIngestionMethod::Recursive) + dumpPath(path, sink, filter ? *filter : defaultPathFilter); + else + sink(readFile(path)); // FIXME: stream }); auto storePath = diff --git a/tests/functional/lang/eval-okay-path.exp b/tests/functional/lang/eval-okay-path.exp index 3ce7f8283..635e2243a 100644 --- a/tests/functional/lang/eval-okay-path.exp +++ b/tests/functional/lang/eval-okay-path.exp @@ -1 +1 @@ -"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" +[ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" ] diff --git a/tests/functional/lang/eval-okay-path.nix b/tests/functional/lang/eval-okay-path.nix index e67168cf3..599b33541 100644 --- a/tests/functional/lang/eval-okay-path.nix +++ b/tests/functional/lang/eval-okay-path.nix @@ -1,7 +1,15 @@ -builtins.path - { path = ./.; - filter = path: _: baseNameOf path == "data"; - recursive = true; - sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; - name = "output"; - } +[ + (builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + }) + (builtins.path + { path = ./data; + recursive = false; + sha256 = "0k4lwj58f2w5yh92ilrwy9917pycipbrdrr13vbb3yd02j09vfxm"; + name = "output"; + }) +] From 57db3be9e448042814d386def2d8af16a2a4c4b9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 16:36:41 +0200 Subject: [PATCH 059/100] SourceAccessor::readFile(): Support reading into a sink --- src/libfetchers/fs-input-accessor.cc | 7 +++- src/libfetchers/input-accessor.cc | 2 +- src/libutil/archive.cc | 15 ++++--- src/libutil/source-accessor.cc | 58 +++++++++++++++++++++++++++- src/libutil/source-accessor.hh | 25 +++++++++++- 5 files changed, 94 insertions(+), 13 deletions(-) diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index e40faf03f..3444c4643 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -19,11 +19,14 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor { } - std::string readFile(const CanonPath & path) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { auto absPath = makeAbsPath(path); checkAllowed(absPath); - return PosixSourceAccessor::readFile(absPath); + PosixSourceAccessor::readFile(absPath, sink, sizeCallback); } bool pathExists(const CanonPath & path) override diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 488350849..d1d450cf7 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -17,7 +17,7 @@ StorePath InputAccessor::fetchToStore( if (method == FileIngestionMethod::Recursive) dumpPath(path, sink, filter ? *filter : defaultPathFilter); else - sink(readFile(path)); // FIXME: stream + readFile(path, sink); }); auto storePath = diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6508ba807..0cd54e5db 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -44,12 +44,15 @@ void SourceAccessor::dumpPath( { auto dumpContents = [&](const CanonPath & path) { - /* It would be nice if this was streaming, but we need the - size before the contents. */ - auto s = readFile(path); - sink << "contents" << s.size(); - sink(s); - writePadding(s.size(), sink); + sink << "contents"; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + sink << _size; + }); + assert(size); + writePadding(*size, sink); }; std::function dump; diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index d5c8cbcdd..2d03d3d7a 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -10,6 +10,28 @@ SourceAccessor::SourceAccessor() { } +std::string SourceAccessor::readFile(const CanonPath & path) +{ + StringSink sink; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + }); + assert(size && *size == sink.s.size()); + return std::move(sink.s); +} + +void SourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + auto s = readFile(path); + sizeCallback(s.size()); + sink(s); +} + Hash SourceAccessor::hashPath( const CanonPath & path, PathFilter & filter, @@ -33,9 +55,41 @@ std::string SourceAccessor::showPath(const CanonPath & path) return path.abs(); } -std::string PosixSourceAccessor::readFile(const CanonPath & path) +void PosixSourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) { - return nix::readFile(path.abs()); + // FIXME: add O_NOFOLLOW since symlinks should be resolved by the + // caller? + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting file"); + + sizeCallback(st.st_size); + + off_t left = st.st_size; + + std::vector buf(64 * 1024); + while (left) { + checkInterrupt(); + ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file '%s'", showPath(path)); + } + else if (rd == 0) + throw SysError("unexpected end-of-file reading '%s'", showPath(path)); + else { + assert(rd <= left); + sink({(char *) buf.data(), (size_t) rd}); + left -= rd; + } + } } bool PosixSourceAccessor::pathExists(const CanonPath & path) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 53408eb6c..f3504c9bb 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -5,6 +5,8 @@ namespace nix { +struct Sink; + /** * A read-only filesystem abstraction. This is used by the Nix * evaluator and elsewhere for accessing sources in various @@ -20,7 +22,23 @@ struct SourceAccessor virtual ~SourceAccessor() { } - virtual std::string readFile(const CanonPath & path) = 0; + /** + * Return the contents of a file as a string. + */ + virtual std::string readFile(const CanonPath & path); + + /** + * Write the contents of a file as a sink. `sizeCallback` must be + * called with the size of the file before any data is written to + * the sink. + * + * Note: subclasses of `SourceAccessor` need to implement at least + * one of the `readFile()` variants. + */ + virtual void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback = [](uint64_t size){}); virtual bool pathExists(const CanonPath & path) = 0; @@ -97,7 +115,10 @@ struct PosixSourceAccessor : SourceAccessor */ time_t mtime = 0; - std::string readFile(const CanonPath & path) override; + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override; bool pathExists(const CanonPath & path) override; From bcf5c31950a01d73c17cee301d3dd363c91cd23d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 16:58:33 +0200 Subject: [PATCH 060/100] Add future FIXME --- src/libexpr/eval.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e3ecb987c..d26cde423 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2461,6 +2461,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nPath: return + // FIXME: compare accessors by their fingerprint. v1._path.accessor == v2._path.accessor && strcmp(v1._path.path, v2._path.path) == 0; From af302267e5e02c8b373779fac979f20e094e7cfa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 17:18:42 +0200 Subject: [PATCH 061/100] Input::hasAllInfo(): Remove --- src/libfetchers/fetchers.cc | 9 +-------- src/libfetchers/fetchers.hh | 11 ----------- src/libfetchers/git.cc | 9 --------- src/libfetchers/github.cc | 5 ----- src/libfetchers/indirect.cc | 5 ----- src/libfetchers/mercurial.cc | 7 ------- src/libfetchers/path.cc | 5 ----- src/libfetchers/tarball.cc | 6 ------ 8 files changed, 1 insertion(+), 56 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c54c39cf0..000609f09 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -89,11 +89,6 @@ Attrs Input::toAttrs() const return attrs; } -bool Input::hasAllInfo() const -{ - return getNarHash() && scheme && scheme->hasAllInfo(*this); -} - bool Input::operator ==(const Input & other) const { return attrs == other.attrs; @@ -117,7 +112,7 @@ std::pair Input::fetch(ref store) const /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ - if (hasAllInfo()) { + if (getNarHash()) { try { auto storePath = computeStorePath(*store); @@ -175,8 +170,6 @@ std::pair Input::fetch(ref store) const input.locked = true; - assert(input.hasAllInfo()); - return {std::move(tree), input}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b55aabb6f..f52175e41 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -79,15 +79,6 @@ public: */ bool isLocked() const { return locked; } - /** - * Check whether the input carries all necessary info required - * for cache insertion and substitution. - * These fields are used to uniquely identify cached trees - * within the "tarball TTL" window without necessarily - * indicating that the input's origin is unchanged. - */ - bool hasAllInfo() const; - bool operator ==(const Input & other) const; bool contains(const Input & other) const; @@ -144,8 +135,6 @@ struct InputScheme virtual ParsedURL toURL(const Input & input) const; - virtual bool hasAllInfo(const Input & input) const = 0; - virtual Input applyOverrides( const Input & input, std::optional ref, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c1a6dce43..26b8987d6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -321,15 +321,6 @@ struct GitInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - bool maybeDirty = !input.getRef(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - return - maybeGetIntAttr(input.attrs, "lastModified") - && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); - } - Input applyOverrides( const Input & input, std::optional ref, diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index d7450defe..b824140a6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -132,11 +132,6 @@ struct GitArchiveInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) const override - { - return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); - } - Input applyOverrides( const Input & _input, std::optional ref, diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index e09d8bfb8..947849802 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -78,11 +78,6 @@ struct IndirectInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - return false; - } - Input applyOverrides( const Input & _input, std::optional ref, diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index b0d2d1909..f830a3271 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -98,13 +98,6 @@ struct MercurialInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - // FIXME: ugly, need to distinguish between dirty and clean - // default trees. - return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); - } - Input applyOverrides( const Input & input, std::optional ref, diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 092975f5d..d829609b5 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -66,11 +66,6 @@ struct PathInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) const override - { - return true; - } - std::optional getSourcePath(const Input & input) override { return getStrAttr(input.attrs, "path"); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 269a56526..ba47d59a7 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -253,12 +253,6 @@ struct CurlInputScheme : InputScheme url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } - - bool hasAllInfo(const Input & input) const override - { - return true; - } - }; struct FileInputScheme : CurlInputScheme From 3e6b9f9357e4573740a4d9e477c4f4fa69c74fa5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 01:17:33 -0400 Subject: [PATCH 062/100] Remove `prevInfos` as its dead code It is unused since 8e0946e8df968391d1430af8377bdb51204e4666 removed support for the repeat and enforce-determinism options. --- src/libstore/build/local-derivation-goal.hh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 0a05081c7..8191af7a6 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -120,14 +120,6 @@ struct LocalDerivationGoal : public DerivationGoal */ OutputPathMap scratchOutputs; - /** - * Path registration info from the previous round, if we're - * building multiple times. Since this contains the hash, it - * allows us to compare whether two rounds produced the same - * result. - */ - std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } From 862d16436b7606bba9064e2bf2a4c48bd883bb42 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 01:22:34 -0400 Subject: [PATCH 063/100] Remove the `ValidPathInfo` `==` operator It is dead code. It was added in 8e0946e8df968391d1430af8377bdb51204e4666 as part of the repeated / enforce-determinism feature, but that was removed in 8fdd156a650f9b2ce9ae8cd74edcf16225478292. It is not good because it skips many fields. For testing purposes we will soon want to add a new one that doesn't skip fields, but we want to make sure making == sensitive to those fields won't change how Nix works. Proving in this commit that the old version is dead code achieves that. --- src/libstore/path-info.hh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 221523622..fea6d0e5f 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -72,14 +72,6 @@ struct ValidPathInfo */ std::optional ca; - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } - /** * Return a fingerprint of the store path to be used in binary * cache signatures. It contains the store path, the base-32 From 0f7e9d051340531332b7c6854675006cc8a16f50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 19:13:35 +0200 Subject: [PATCH 064/100] Input: Remove 'direct' field --- src/libfetchers/fetchers.cc | 5 +++++ src/libfetchers/fetchers.hh | 6 ++++-- src/libfetchers/indirect.cc | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 000609f09..5f6167868 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -84,6 +84,11 @@ std::string Input::to_string() const return toURL().to_string(); } +bool Input::isDirect() const +{ + return !scheme || scheme->isDirect(*this); +} + Attrs Input::toAttrs() const { return attrs; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index f52175e41..4119dd17b 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -35,7 +35,6 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; bool locked = false; - bool direct = true; /** * path of the parent of this input, used for relative path resolution @@ -71,7 +70,7 @@ public: * Check whether this is a "direct" input, that is, not * one that goes through a registry. */ - bool isDirect() const { return direct; } + bool isDirect() const; /** * Check whether this is a "locked" input, that is, @@ -152,6 +151,9 @@ struct InputScheme * Is this `InputScheme` part of an experimental feature? */ virtual std::optional experimentalFeature(); + + virtual bool isDirect(const Input & input) const + { return true; } }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 947849802..9a71df3d4 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -41,7 +41,6 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? Input input; - input.direct = false; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -63,7 +62,6 @@ struct IndirectInputScheme : InputScheme throw BadURL("'%s' is not a valid flake ID", id); Input input; - input.direct = false; input.attrs = attrs; return input; } @@ -98,6 +96,9 @@ struct IndirectInputScheme : InputScheme { return Xp::Flakes; } + + bool isDirect(const Input & input) const override + { return false; } }; static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); From 85e5ac403fa9adb82105664bb6c4df501d4a0a25 Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Fri, 20 Oct 2023 10:28:26 -0700 Subject: [PATCH 065/100] docker: publish images to ghcr.io (#8066) * docker: publish images to ghcr.io docker.com announced their intention to remove the free plan used by OSS. The nixos/nix image is essential to various CI runs to build with nix. To provide a continuity plan, this commit pushes the image to ghcr.io as well. Co-authored-by: Sandro --- .github/workflows/ci.yml | 23 +++++++++++++++++++ .../src/installation/installing-docker.md | 8 +++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c9c24dad..afe4dc2e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,9 @@ jobs: docker_push_image: needs: [check_secrets, tests] + permissions: + contents: read + packages: write if: >- github.event_name == 'push' && github.ref_name == 'master' && @@ -126,6 +129,9 @@ jobs: - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master + # We'll deploy the newly built image to both Docker Hub and Github Container Registry. + # + # Push to Docker Hub first - name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -133,3 +139,20 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - run: docker push nixos/nix:$NIX_VERSION - run: docker push nixos/nix:master + # Push to GitHub Container Registry as well + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION + docker tag nix:$NIX_VERSION $IMAGE_ID:master + docker push $IMAGE_ID:$NIX_VERSION + docker push $IMAGE_ID:master diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index 9d6d8f2d9..6f77d6a57 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -3,14 +3,14 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker run -ti nixos/nix -Unable to find image 'nixos/nix:latest' locally -latest: Pulling from nixos/nix +$ docker run -ti ghcr.io/nixos/nix +Unable to find image 'ghcr.io/nixos/nix:latest' locally +latest: Pulling from ghcr.io/nixos/nix 5843afab3874: Pull complete b52bf13f109c: Pull complete 1e2415612aa3: Pull complete Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff -Status: Downloaded newer image for nixos/nix:latest +Status: Downloaded newer image for ghcr.io/nixos/nix:latest 35ca4ada6e96:/# nix --version nix (Nix) 2.3.12 35ca4ada6e96:/# exit From 935c9981de03248fd5687c5d0363f572af723144 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Oct 2023 19:50:21 +0200 Subject: [PATCH 066/100] Remove fetchers::Tree and move tarball-related stuff into its own header --- src/libcmd/common-eval-args.cc | 5 ++-- src/libexpr/flake/flake.cc | 34 ++++++++++----------- src/libexpr/flake/flake.hh | 6 ++-- src/libexpr/flake/flakeref.cc | 6 ++-- src/libexpr/flake/flakeref.hh | 2 +- src/libexpr/parser.y | 6 ++-- src/libexpr/primops/fetchMercurial.cc | 6 ++-- src/libexpr/primops/fetchTree.cc | 13 ++++---- src/libfetchers/fetchers.cc | 18 +++++------ src/libfetchers/fetchers.hh | 41 ++----------------------- src/libfetchers/github.cc | 5 ++-- src/libfetchers/registry.cc | 2 +- src/libfetchers/tarball.cc | 7 +++-- src/libfetchers/tarball.hh | 43 +++++++++++++++++++++++++++ src/nix-channel/nix-channel.cc | 2 +- src/nix/flake.cc | 18 +++++------ 16 files changed, 111 insertions(+), 103 deletions(-) create mode 100644 src/libfetchers/tarball.hh diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e36bda52f..e6df49a7a 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -9,6 +9,7 @@ #include "flake/flakeref.hh" #include "store-api.hh" #include "command.hh" +#include "tarball.hh" namespace nix { @@ -168,14 +169,14 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; + auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index a6212c12f..b32c7c086 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -15,7 +15,7 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +typedef std::pair FetchedFlake; typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( @@ -34,7 +34,7 @@ static std::optional lookupInFlakeCache( return std::nullopt; } -static std::tuple fetchOrSubstituteTree( +static std::tuple fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, bool allowLookup, @@ -61,16 +61,16 @@ static std::tuple fetchOrSubstituteTree( flakeCache.push_back({originalRef, *fetched}); } - auto [tree, lockedRef] = *fetched; + auto [storePath, lockedRef] = *fetched; debug("got tree '%s' from '%s'", - state.store->printStorePath(tree.storePath), lockedRef); + state.store->printStorePath(storePath), lockedRef); - state.allowPath(tree.storePath); + state.allowPath(storePath); - assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(tree), resolvedRef, lockedRef}; + return {std::move(storePath), resolvedRef, lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) @@ -202,21 +202,21 @@ static Flake getFlake( FlakeCache & flakeCache, InputPath lockRootPath) { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. - auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true); + auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true); auto flakeFile = canonPath(flakeDir + "/flake.nix", true); - if (!isInDir(flakeFile, sourceInfo.actualPath)) + if (!isInDir(flakeFile, state.store->toRealPath(storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", - lockedRef, state.store->printStorePath(sourceInfo.storePath)); + lockedRef, state.store->printStorePath(storePath)); Flake flake { .originalRef = originalRef, .resolvedRef = resolvedRef, .lockedRef = lockedRef, - .sourceInfo = std::make_shared(std::move(sourceInfo)) + .storePath = storePath, }; if (!pathExists(flakeFile)) @@ -346,7 +346,7 @@ LockedFlake lockFlake( // FIXME: symlink attack auto oldLockFile = LockFile::read( lockFlags.referenceLockFilePath.value_or( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); + state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); @@ -574,7 +574,7 @@ LockedFlake lockFlake( oldLock ? std::dynamic_pointer_cast(oldLock) : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), + state.store->toRealPath(inputFlake.storePath) + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -598,7 +598,7 @@ LockedFlake lockFlake( }; // Bring in the current ref for relative path resolution if we have it - auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); + auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true); computeLocks( flake.inputs, @@ -729,7 +729,7 @@ void callFlake(EvalState & state, emitTreeAttrs( state, - *lockedFlake.flake.sourceInfo, + lockedFlake.flake.storePath, lockedFlake.flake.lockedRef.input, *vRootSrc, false, @@ -893,7 +893,7 @@ Fingerprint LockedFlake::getFingerprint() const // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%s;%d;%d;%s", - flake.sourceInfo->storePath.to_string(), + flake.storePath.to_string(), flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index c1d1b71e5..d5ad3eade 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -10,8 +10,6 @@ namespace nix { class EvalState; -namespace fetchers { struct Tree; } - namespace flake { struct FlakeInput; @@ -84,7 +82,7 @@ struct Flake */ bool forceDirty = false; std::optional description; - std::shared_ptr sourceInfo; + StorePath storePath; FlakeInputs inputs; /** * 'nixConfig' attribute @@ -193,7 +191,7 @@ void callFlake( void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback = false, diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 93ad33a33..16f45ace7 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -272,10 +272,10 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } -std::pair FlakeRef::fetchTree(ref store) const +std::pair FlakeRef::fetchTree(ref store) const { - auto [tree, lockedInput] = input.fetch(store); - return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; + auto [storePath, lockedInput] = input.fetch(store); + return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; } std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index ffb2e50de..5d78f49b6 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -63,7 +63,7 @@ struct FlakeRef static FlakeRef fromAttrs(const fetchers::Attrs & attrs); - std::pair fetchTree(ref store) const; + std::pair fetchTree(ref store) const; }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 70228e1e2..8b73bd056 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -646,7 +646,7 @@ formal #include "eval.hh" #include "filetransfer.hh" -#include "fetchers.hh" +#include "tarball.hh" #include "store-api.hh" #include "flake/flake.hh" @@ -783,7 +783,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath; res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ @@ -797,7 +797,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; + auto storePath = flakeRef.resolve(store).fetchTree(store).first; res = { store->toRealPath(storePath) }; } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index b9ff01c16..e76ce455d 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -71,10 +71,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to @@ -86,7 +86,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("revCount").mkInt(*revCount); v.mkAttrs(attrs2); - state.allowPath(tree.storePath); + state.allowPath(storePath); } static RegisterPrimOp r_fetchMercurial({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 976037ff9..a99b0e500 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -5,6 +5,7 @@ #include "fetchers.hh" #include "filetransfer.hh" #include "registry.hh" +#include "tarball.hh" #include "url.hh" #include @@ -15,7 +16,7 @@ namespace nix { void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback, @@ -25,7 +26,7 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -165,11 +166,11 @@ static void fetchTree( state.checkURI(input.toURLString()); - auto [tree, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetch(state.store); - state.allowPath(tree.storePath); + state.allowPath(storePath); - emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); + emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -288,7 +289,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; if (expectedHash) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5f6167868..5688c4dc1 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -109,7 +109,7 @@ bool Input::contains(const Input & other) const return false; } -std::pair Input::fetch(ref store) const +std::pair Input::fetch(ref store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); @@ -126,7 +126,7 @@ std::pair Input::fetch(ref store) const debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); - return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this}; + return {std::move(storePath), *this}; } catch (Error & e) { debug("substitution of input '%s' failed: %s", to_string(), e.what()); } @@ -141,18 +141,16 @@ std::pair Input::fetch(ref store) const } }(); - Tree tree { - .actualPath = store->toRealPath(storePath), - .storePath = storePath, - }; - - auto narHash = store->queryPathInfo(tree.storePath)->narHash; + auto narHash = store->queryPathInfo(storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(HashFormat::SRI, true), narHash.to_string(HashFormat::SRI, true)); + to_string(), + store->printStorePath(storePath), + prevNarHash->to_string(HashFormat::SRI, true), + narHash.to_string(HashFormat::SRI, true)); } if (auto prevLastModified = getLastModified()) { @@ -175,7 +173,7 @@ std::pair Input::fetch(ref store) const input.locked = true; - return {std::move(tree), input}; + return {std::move(storePath), input}; } Input Input::applyOverrides( diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 4119dd17b..ac605ff8e 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -13,12 +13,6 @@ namespace nix { class Store; } namespace nix::fetchers { -struct Tree -{ - Path actualPath; - StorePath storePath; -}; - struct InputScheme; /** @@ -83,10 +77,10 @@ public: bool contains(const Input & other) const; /** - * Fetch the input into the Nix store, returning the location in - * the Nix store and the locked input. + * Fetch the entire input into the Nix store, returning the + * location in the Nix store and the locked input. */ - std::pair fetch(ref store) const; + std::pair fetch(ref store) const; Input applyOverrides( std::optional ref, @@ -158,33 +152,4 @@ struct InputScheme void registerInputScheme(std::shared_ptr && fetcher); -struct DownloadFileResult -{ - StorePath storePath; - std::string etag; - std::string effectiveUrl; - std::optional immutableUrl; -}; - -DownloadFileResult downloadFile( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); - -struct DownloadTarballResult -{ - Tree tree; - time_t lastModified; - std::optional immutableUrl; -}; - -DownloadTarballResult downloadTarball( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); - } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index b824140a6..617fc7468 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -7,6 +7,7 @@ #include "git.hh" #include "fetchers.hh" #include "fetch-settings.hh" +#include "tarball.hh" #include #include @@ -213,10 +214,10 @@ struct GitArchiveInputScheme : InputScheme {"rev", rev->gitRev()}, {"lastModified", uint64_t(result.lastModified)} }, - result.tree.storePath, + result.storePath, true); - return {result.tree.storePath, input}; + return {result.storePath, input}; } std::optional experimentalFeature() override diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 43c03beec..a0fff9ceb 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,5 +1,5 @@ #include "registry.hh" -#include "fetchers.hh" +#include "tarball.hh" #include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ba47d59a7..e1ea9b58b 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,3 +1,4 @@ +#include "tarball.hh" #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" @@ -133,7 +134,7 @@ DownloadTarballResult downloadTarball( if (cached && !cached->expired) return { - .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, + .storePath = std::move(cached->storePath), .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; @@ -174,7 +175,7 @@ DownloadTarballResult downloadTarball( locked); return { - .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, + .storePath = std::move(*unpackedStorePath), .lastModified = lastModified, .immutableUrl = res.immutableUrl, }; @@ -307,7 +308,7 @@ struct TarballInputScheme : CurlInputScheme if (result.lastModified && !input.attrs.contains("lastModified")) input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); - return {result.tree.storePath, std::move(input)}; + return {result.storePath, std::move(input)}; } }; diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh new file mode 100644 index 000000000..9e6b50b31 --- /dev/null +++ b/src/libfetchers/tarball.hh @@ -0,0 +1,43 @@ +#pragma once + +#include "types.hh" +#include "path.hh" + +#include + +namespace nix { +class Store; +} + +namespace nix::fetchers { + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; + std::optional immutableUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +struct DownloadTarballResult +{ + StorePath storePath; + time_t lastModified; + std::optional immutableUrl; +}; + +DownloadTarballResult downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +} diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 95f401441..4504441fa 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -4,9 +4,9 @@ #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" -#include "fetchers.hh" #include "eval-settings.hh" // for defexpr #include "util.hh" +#include "tarball.hh" #include #include diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ceb112c03..b4e4156c0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -186,7 +186,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["path"] = store->printStorePath(flake.storePath); j["locks"] = lockedFlake.lockFile.toJSON(); logger->cout("%s", j.dump()); } else { @@ -202,7 +202,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON *flake.description); logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", - store->printStorePath(flake.sourceInfo->storePath)); + store->printStorePath(flake.storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -976,7 +976,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - sources.insert(flake.flake.sourceInfo->storePath); + sources.insert(flake.flake.storePath); // FIXME: use graph output, handle cycles. std::function traverse; @@ -988,7 +988,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + : (*inputNode)->lockedRef.input.fetch(store).first; if (json) { auto& jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); @@ -1005,7 +1005,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (json) { nlohmann::json jsonRoot = { - {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, + {"path", store->printStorePath(flake.flake.storePath)}, {"inputs", traverse(*flake.lockFile.root)}, }; logger->cout("%s", jsonRoot); @@ -1339,12 +1339,12 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON { auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); - auto [tree, lockedRef] = resolvedRef.fetchTree(store); - auto hash = store->queryPathInfo(tree.storePath)->narHash; + auto [storePath, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(storePath)->narHash; if (json) { auto res = nlohmann::json::object(); - res["storePath"] = store->printStorePath(tree.storePath); + res["storePath"] = store->printStorePath(storePath); res["hash"] = hash.to_string(HashFormat::SRI, true); res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); @@ -1352,7 +1352,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), - store->printStorePath(tree.storePath), + store->printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); } } From 97a0c08873496dbd6f53adb253e159ecf700d616 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 Oct 2023 21:17:28 +0200 Subject: [PATCH 067/100] Expand derivation examples (#9048) Also use fancier formatting so the example blocks are easier to discern from the description. Co-authored-by: John Ericson --- doc/manual/src/glossary.md | 1 + doc/manual/src/language/derivations.md | 214 +++++++++++++++++++------ 2 files changed, 162 insertions(+), 53 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 611d9576f..d49d5e52e 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -207,6 +207,7 @@ - [output]{#gloss-output} A [store object] produced by a [derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. [output]: #gloss-output diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index d80a7dd48..2aded5527 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -8,8 +8,6 @@ It outputs an attribute set, and produces a [store derivation] as a side effect [store derivation]: @docroot@/glossary.md#gloss-store-derivation - - ## Input attributes ### Required @@ -17,11 +15,22 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) A symbolic name for the derivation. - It is added to the [store derivation]'s [path](@docroot@/glossary.md#gloss-store-path) and its [output paths][output path]. + It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). - Example: `name = "hello";` + [store path]: @docroot@/glossary.md#gloss-store-path + + > **Example** + > + > ```nix + > derivation { + > name = "hello"; + > # ... + > } + > ``` + > + > The store derivation's path will be `/nix/store/-hello.drv`. + > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` - The store derivation's path will be `/nix/store/-hello.drv`, and the output paths will be of the form `/nix/store/-hello[-]` - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -29,77 +38,175 @@ It outputs an attribute set, and produces a [store derivation] as a side effect A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. - Examples: - - `system = "x86_64-linux";` - - `system = builtins.currentSystem;` - - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. - [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system + > **Example** + > + > Declare a derivation to be built on a specific system type: + > + > ```nix + > derivation { + > # ... + > system = "x86_64-linux"; + > # ... + > } + > ``` + + > **Example** + > + > Declare a derivation to be built on the system type that evaluates the expression: + > + > ```nix + > derivation { + > # ... + > system = builtins.currentSystem; + > # ... + > } + > ``` + > + > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + - [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) Path to an executable that will perform the build. - Examples: + > **Example** + > + > Use the file located at `/bin/bash` as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > # ... + > }; + > ``` - `builder = "/bin/bash";` + - `builder = ./builder.sh;` + > **Example** + > + > Copy a local file to the Nix store for use as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = ./builder.sh; + > # ... + > }; + > ``` - `builder = "${pkgs.python}/bin/python";` + + + > **Example** + > + > Use a file from another derivation as the builder executable: + > + > ```nix + > let pkgs = import {}; in + > derivation { + > # ... + > builder = "${pkgs.python}/bin/python"; + > # ... + > }; + > ``` ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ ]` +- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ ]` Command-line arguments to be passed to the [`builder`](#attr-builder) executable. - Example: `args = [ "-c" "echo hello world > $out" ];` + > **Example** + > + > Pass arguments to Bash to interpret a shell command: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > args = [ "-c" "echo hello world > $out" ]; + > # ... + > }; + > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ "out" ]` +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ "out" ]` Symbolic outputs of the derivation. - Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [output path]. + Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [store path]. - [output path]: @docroot@/glossary.md#gloss-output-path - - By default, a derivation produces a single output path called `out`. - However, derivations can produce multiple output paths. + By default, a derivation produces a single output called `out`. + However, derivations can produce multiple outputs. This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. - Examples: + > **Example** + > + > Imagine a library package that provides a dynamic library, header files, and documentation. + > A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + > Thus, the library package could specify: + > + > ```nix + > derivation { + > # ... + > outputs = [ "lib" "dev" "doc" ]; + > # ... + > } + > ``` + > + > This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. + > The builder would typically do something like + > + > ```bash + > ./configure \ + > --libdir=$lib/lib \ + > --includedir=$dev/include \ + > --docdir=$doc/share/doc + > ``` + > + > for an Autoconf-style package. - Imagine a library package that provides a dynamic library, header files, and documentation. - A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. - Thus, the library package could specify: + The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. - ```nix - derivation { - # ... - outputs = [ "lib" "dev" "doc" ]; - # ... - } - ``` + > **Example** + > + > + > ```nix + > derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > } + > ``` + > + > The store derivation path will be `/nix/store/-example.drv`. + > The output paths will be + > - `/nix/store/-example-lib` + > - `/nix/store/-example-dev` + > - `/nix/store/-example-doc` + > - `/nix/store/-example` - This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. - The builder would typically do something like + You can refer to each output of a derivation by selecting it as an attribute. + The first element of `outputs` determines the *default output* and ends up at the top-level. - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$dev/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. - - You can refer to each output of a derivation by selecting it as an attribute, e.g. `myPackage.lib` or `myPackage.doc`. - - The first element of `outputs` determines the *default output*. - Therefore, in the given example, `myPackage` is equivalent to `myPackage.lib`. + > **Example** + > + > Select an output by attribute name: + > + > ```nix + > let + > myPackage = derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > }; + > in myPackage.dev + > ``` + > + > Since `lib` is the first output, `myPackage` is equivalent to `myPackage.lib`. @@ -123,8 +230,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect reside in the Nix store. - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. + present derivation. The environment variable is set to the [store path] of the derivation's default [output](#attr-outputs). - Lists of the previous types are also allowed. They are simply concatenated, separated by spaces. @@ -132,6 +238,8 @@ It outputs an attribute set, and produces a [store derivation] as a side effect - `true` is passed as the string `1`, `false` and `null` are passed as an empty string. + + ## Builder execution The [`builder`](#attr-builder) is executed as follows: From 96c58550b839ca415a2cef0f7a5ae51f3e86f028 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 16 Oct 2023 01:45:06 -0400 Subject: [PATCH 068/100] Test more derived paths --- src/libstore/tests/worker-protocol.cc | 11 ++++++++++- .../libstore/worker-protocol/derived-path.bin | Bin 120 -> 248 bytes 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index 63fecce96..b10192cc1 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -72,10 +72,19 @@ VERSIONED_CHARACTERIZATION_TEST( derivedPath, "derived-path", defaultVersion, - (std::tuple { + (std::tuple { DerivedPath::Opaque { .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, }, + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, DerivedPath::Built { .drvPath = makeConstantStorePathRef(StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", diff --git a/unit-test-data/libstore/worker-protocol/derived-path.bin b/unit-test-data/libstore/worker-protocol/derived-path.bin index bb1a81ac6d096cddf510a457e7506e6a81ee66a3..0729b2690e2f99dc8122b013e4ceaa4a51ebd3d6 100644 GIT binary patch delta 39 mcmb>U!8pN!(Rd=G8JAv4Q5gdWm`(%= Date: Fri, 26 May 2023 14:48:11 -0400 Subject: [PATCH 069/100] Systematize the worker protocol derived path serialiser It was some ad-hoc functions to account for versions, while the already factored-out serializer just supported the latest version. Now, we can fold that version-specific logic into the factored out one, and so we do. --- src/libstore/daemon.cc | 18 ++-------- src/libstore/remote-store.cc | 33 ++---------------- src/libstore/tests/worker-protocol.cc | 29 +++++++++++++-- src/libstore/worker-protocol.cc | 26 ++++++++++++-- .../worker-protocol/derived-path-1.29.bin | Bin 0 -> 184 bytes ...derived-path.bin => derived-path-1.30.bin} | Bin 6 files changed, 56 insertions(+), 50 deletions(-) create mode 100644 unit-test-data/libstore/worker-protocol/derived-path-1.29.bin rename unit-test-data/libstore/worker-protocol/{derived-path.bin => derived-path-1.30.bin} (100%) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e117b507f..27c0b6431 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -261,18 +261,6 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, WorkerProto::Version clientVersion, WorkerProto::ReadConn conn) -{ - std::vector reqs; - if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = WorkerProto::Serialise>::read(store, conn); - } else { - for (auto & s : readStrings(conn.from)) - reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); - } - return reqs; -} - static void performOp(TunnelLogger * logger, ref store, TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) @@ -538,7 +526,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPaths: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -563,7 +551,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPathsWithResults: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; mode = (BuildMode) readInt(from); @@ -938,7 +926,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryMissing: { - auto targets = readDerivedPaths(*store, clientVersion, rconn); + auto targets = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1704bfbb0..482c2ae01 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -655,33 +655,6 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, } catch (...) { return callback.rethrow(); } } -static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector & reqs) -{ - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) { - WorkerProto::write(store, conn, reqs); - } else { - Strings ss; - for (auto & p : reqs) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(store)); - }, - [&](const StorePath & drvPath) { - throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", - store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn.daemonVersion), - GET_PROTOCOL_MINOR(conn.daemonVersion)); - }, - [&](std::monostate) { - throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); - }, - }, sOrDrvPath); - } - conn.to << ss; - } -} - void RemoteStore::copyDrvsFromEvalStore( const std::vector & paths, std::shared_ptr evalStore) @@ -711,7 +684,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeDerivedPaths(*this, *conn, drvPaths); + WorkerProto::write(*this, *conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -735,7 +708,7 @@ std::vector RemoteStore::buildPathsWithResults( if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; - writeDerivedPaths(*this, *conn, paths); + WorkerProto::write(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); return WorkerProto::Serialise>::read(*this, *conn); @@ -929,7 +902,7 @@ void RemoteStore::queryMissing(const std::vector & targets, // to prevent a deadlock. goto fallback; conn->to << WorkerProto::Op::QueryMissing; - writeDerivedPaths(*this, *conn, targets); + WorkerProto::write(*this, *conn, targets); conn.processStderr(); willBuild = WorkerProto::Serialise::read(*this, *conn); willSubstitute = WorkerProto::Serialise::read(*this, *conn); diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index b10192cc1..b3e857bc4 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -69,9 +69,32 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - derivedPath, - "derived-path", - defaultVersion, + derivedPath_1_29, + "derived-path-1.29", + 1 << 8 | 29, + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + derivedPath_1_30, + "derived-path-1.30", + 1 << 8 | 30, (std::tuple { DerivedPath::Opaque { .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 415e66f16..0f3f20ca5 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -51,12 +51,34 @@ void WorkerProto::Serialise>::write(const Store & sto DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); - return DerivedPath::parseLegacy(store, s); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + return DerivedPath::parseLegacy(store, s); + } else { + return parsePathWithOutputs(store, s).toDerivedPath(); + } } void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) { - conn.to << req.to_string_legacy(store); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + conn.to << req.to_string_legacy(store); + } else { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(req); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + conn.to << s.to_string(store); + }, + [&](const StorePath & drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn.version), + GET_PROTOCOL_MINOR(conn.version)); + }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but protocols do not support that. Try upgrading the Nix on the other end of this connection"); + }, + }, sOrDrvPath); + } } diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin new file mode 100644 index 0000000000000000000000000000000000000000..05ea7678aa058344a5ca4cb1e9e7333e9cf8fe20 GIT binary patch literal 184 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F&X-j5{vXwipsz`&B@oVSfNwN F001Y$HlP3i literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/derived-path.bin b/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path.bin rename to unit-test-data/libstore/worker-protocol/derived-path-1.30.bin From ab822af0dfee341be131ad285a2cba5362a3f2dc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 28 Feb 2021 18:42:46 +0000 Subject: [PATCH 070/100] Factor out serialization for `BuildResult` Worker Protocol: Note that the worker protocol already had a serialization for `BuildResult`; this was added in a4604f19284254ac98f19a13ff7c2216de7fe176. It didn't have any versioning support because at that time reusable seralizers were not away for the protocol version. It could thus only be used for new messages also introduced in that commit. Now that we do support versioning in reusable serializers, we can expand it to support all known versions and use it in many more places. The exist test data becomes the version 1.29 tests: note that those files' contents are unchanged. 1.28 and 1.27 tests are added to cover the older code-paths. The keyered build result test only has 1.29 because the keying was also added in a4604f19284254ac98f19a13ff7c2216de7fe176; the older serializations are always used unkeyed. Serve Protocol: Conversely, no attempt was made to factor out such a serializer for the serve protocol, so our work there in this commit for that protocol proceeds from scratch. --- src/libstore/daemon.cc | 11 +- src/libstore/legacy-ssh-store.cc | 15 +-- src/libstore/remote-store.cc | 15 +-- src/libstore/serve-protocol.cc | 43 +++++++ src/libstore/serve-protocol.hh | 6 + src/libstore/tests/serve-protocol.cc | 113 +++++++++++++++++- src/libstore/tests/worker-protocol.cc | 80 ++++++++++++- src/libstore/worker-protocol.cc | 49 ++++---- src/nix-store/nix-store.cc | 12 +- .../serve-protocol/build-result-2.2.bin | Bin 0 -> 80 bytes .../serve-protocol/build-result-2.3.bin | Bin 0 -> 176 bytes .../build-result-2.6.bin} | Bin .../worker-protocol/build-result-1.27.bin | Bin 0 -> 80 bytes .../worker-protocol/build-result-1.28.bin | Bin 0 -> 648 bytes .../worker-protocol/build-result-1.29.bin | Bin 0 -> 744 bytes ...result.bin => keyed-build-result-1.29.bin} | Bin 16 files changed, 268 insertions(+), 76 deletions(-) create mode 100644 unit-test-data/libstore/serve-protocol/build-result-2.2.bin create mode 100644 unit-test-data/libstore/serve-protocol/build-result-2.3.bin rename unit-test-data/libstore/{worker-protocol/build-result.bin => serve-protocol/build-result-2.6.bin} (100%) create mode 100644 unit-test-data/libstore/worker-protocol/build-result-1.27.bin create mode 100644 unit-test-data/libstore/worker-protocol/build-result-1.28.bin create mode 100644 unit-test-data/libstore/worker-protocol/build-result-1.29.bin rename unit-test-data/libstore/worker-protocol/{keyed-build-result.bin => keyed-build-result-1.29.bin} (100%) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 27c0b6431..19afe4388 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -635,16 +635,7 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); - to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; - } - if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); - } + WorkerProto::write(*store, wconn, res); break; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d6e24521a..c712f7eb1 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -319,20 +319,7 @@ public: conn->to.flush(); - BuildResult status; - status.status = (BuildResult::Status) readInt(conn->from); - conn->from >> status.errorMsg; - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = ServeProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - status.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return status; + return ServeProto::Serialise::read(*this, *conn); } void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 482c2ae01..ef648e01b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -788,20 +788,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res; - res.status = (BuildResult::Status) readInt(conn->from); - conn->from >> res.errorMsg; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { - conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return res; + return WorkerProto::Serialise::read(*this, *conn); } diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 16a62b5bc..97a0ddf0e 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" +#include "build-result.hh" #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "archive.hh" @@ -12,4 +13,46 @@ namespace nix { /* protocol-specific definitions */ +BuildResult ServeProto::Serialise::read(const Store & store, ServeProto::ReadConn conn) +{ + BuildResult status; + status.status = (BuildResult::Status) readInt(conn.from); + conn.from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.from + >> status.timesBuilt + >> status.isNonDeterministic + >> status.startTime + >> status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + auto builtOutputs = ServeProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } + return status; +} + +void ServeProto::Serialise::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status) +{ + conn.to + << status.status + << status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << status.timesBuilt + << status.isNonDeterministic + << status.startTime + << status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + ServeProto::write(store, conn, builtOutputs); + } +} + } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index a627c6ad6..ba159f6e9 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -16,6 +16,9 @@ namespace nix { class Store; struct Source; +// items being serialised +struct BuildResult; + /** * The "serve protocol", used by ssh:// stores. @@ -136,6 +139,9 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ }; +template<> +DECLARE_SERVE_SERIALISER(BuildResult); + template DECLARE_SERVE_SERIALISER(std::vector); template diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc index ba798ab1c..c8ac87a04 100644 --- a/src/libstore/tests/serve-protocol.cc +++ b/src/libstore/tests/serve-protocol.cc @@ -19,7 +19,7 @@ struct ServeProtoTest : VersionedProtoTest * For serializers that don't care about the minimum version, we * used the oldest one: 1.0. */ - ServeProto::Version defaultVersion = 1 << 8 | 0; + ServeProto::Version defaultVersion = 2 << 8 | 0; }; VERSIONED_CHARACTERIZATION_TEST( @@ -114,6 +114,117 @@ VERSIONED_CHARACTERIZATION_TEST( }, })) +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_2, + "build-result-2.2", + 2 << 8 | 2, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_3, + "build-result-2.3", + 2 << 8 | 3, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_6, + "build-result-2.6", + 2 << 8 | 6, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index b3e857bc4..e0ce340d8 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -167,9 +167,77 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - buildResult, - "build-result", - defaultVersion, + buildResult_1_27, + "build-result-1.27", + 1 << 8 | 27, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_28, + "build-result-1.28", + 1 << 8 | 28, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29, + "build-result-1.29", + 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t { @@ -226,9 +294,9 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, - keyedBuildResult, - "keyed-build-result", - defaultVersion, + keyedBuildResult_1_29, + "keyed-build-result-1.29", + 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t { diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 0f3f20ca5..f69322a61 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -103,17 +103,21 @@ BuildResult WorkerProto::Serialise::read(const Store & store, Worke { BuildResult res; res.status = (BuildResult::Status) readInt(conn.from); - conn.from - >> res.errorMsg - >> res.timesBuilt - >> res.isNonDeterministic - >> res.startTime - >> res.stopTime; - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); + conn.from >> res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.from + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + auto builtOutputs = WorkerProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } return res; } @@ -121,15 +125,20 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto { conn.to << res.status - << res.errorMsg - << res.timesBuilt - << res.isNonDeterministic - << res.startTime - << res.stopTime; - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputs); + << res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.to + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + WorkerProto::write(store, conn, builtOutputs); + } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e78720f92..e4dd94585 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -959,17 +959,7 @@ static void opServe(Strings opFlags, Strings opArgs) MonitorFdHup monitor(in.fd); auto status = store->buildDerivation(drvPath, drv); - out << status.status << status.errorMsg; - - if (GET_PROTOCOL_MINOR(clientVersion) >= 3) - out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : status.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - ServeProto::write(*store, wconn, builtOutputs); - } - + ServeProto::write(*store, wconn, status); break; } diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/unit-test-data/libstore/serve-protocol/build-result-2.2.bin new file mode 100644 index 0000000000000000000000000000000000000000..ae684778bc26addba4bf1b3e49cc30edf5f038fa GIT binary patch literal 80 gcmZQ&fBf0AiU4H~;_u literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/unit-test-data/libstore/serve-protocol/build-result-2.3.bin new file mode 100644 index 0000000000000000000000000000000000000000..d51e08dfc0d1715210e8a616a5403f03f4b2a46c GIT binary patch literal 176 vcmZQ&fBf0AiU4H~;_u literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/unit-test-data/libstore/worker-protocol/build-result-1.28.bin new file mode 100644 index 0000000000000000000000000000000000000000..74bcd5cf98b828fb63a931ef7810f7fabc805ca5 GIT binary patch literal 648 zcmc&wK?=e!5EQ|aK0%e!+`)6%Si?(*`6_8xaxz$KI6Xx-2t!SGa{k;2f8i+RMR2A;`6>RitBjDY7{lnANJD3OVh8WW_c zkTRG+zDX!Yj%68orEsd#Y$KG=*{FoWL-7`MFAQl%7RmZ0!PYe3jk66aF4r+L$O`tu z&uq-x(J#Q)LAOdzsy>VTJDdcofzX)BfXe~O6CwQ^%woR?}>#sX4il$bfs;n zmv%`YOQ~vvTo;t-%xH@l(n4vSK8bRZQHc`kI^B)Ih0TkNGRhXy8rqZN5BnYj(ieFo zAJ+t*u7l`;??iPt&V)lzi91dfGZFf@g4iVAZN4+jUVRU7o>ol_o!fedeM@Pl_mAVf T^ROZOQyyvZZF!s Date: Tue, 8 Mar 2022 23:09:26 +0000 Subject: [PATCH 071/100] Move `ValidPathInfo` serialization code to `worker-protocol.{cc.hh}` It does not belong with the data type itself. This also materializes the fact that `copyPath` does not do any version negotiation just just hard-codes "16". The non-standard interface of these serializers makes it harder to test, but this is fixed in the next commit which then adds those tests. --- src/libstore/daemon.cc | 4 +-- src/libstore/path-info.cc | 54 --------------------------------- src/libstore/path-info.hh | 5 --- src/libstore/remote-store.cc | 11 +++++-- src/libstore/store-api.cc | 11 ++++++- src/libstore/worker-protocol.cc | 45 ++++++++++++++++++++++++++- src/libstore/worker-protocol.hh | 12 ++++++++ 7 files changed, 76 insertions(+), 66 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 19afe4388..0afaaccba 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -422,7 +422,7 @@ static void performOp(TunnelLogger * logger, ref store, }(); logger->stopWork(); - pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion)); + WorkerProto::Serialise::write(*store, wconn, *pathInfo); } else { HashType hashAlgo; std::string baseName; @@ -819,7 +819,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false); + WorkerProto::Serialise::write(*store, wconn, *info, false); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index fb507a847..51fb5f02a 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,6 +1,4 @@ #include "path-info.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "store-api.hh" namespace nix { @@ -99,7 +97,6 @@ Strings ValidPathInfo::shortRefs() const return refs; } - ValidPathInfo::ValidPathInfo( const Store & store, std::string_view name, @@ -128,55 +125,4 @@ ValidPathInfo::ValidPathInfo( }, std::move(ca).raw); } - -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) -{ - return read(source, store, format, store.parseStorePath(readString(source))); -} - -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path) -{ - auto deriver = readString(source); - auto narHash = Hash::parseAny(readString(source), htSHA256); - ValidPathInfo info(path, narHash); - if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { - .from = source, - .version = format, - }); - source >> info.registrationTime >> info.narSize; - if (format >= 16) { - source >> info.ultimate; - info.sigs = readStrings(source); - info.ca = ContentAddress::parseOpt(readString(source)); - } - return info; -} - - -void ValidPathInfo::write( - Sink & sink, - const Store & store, - unsigned int format, - bool includePath) const -{ - if (includePath) - sink << store.printStorePath(path); - sink << (deriver ? store.printStorePath(*deriver) : "") - << narHash.to_string(HashFormat::Base16, false); - WorkerProto::write(store, - WorkerProto::WriteConn { - .to = sink, - .version = format, - }, - references); - sink << registrationTime << narSize; - if (format >= 16) { - sink << ultimate - << sigs - << renderContentAddress(ca); - } -} - } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index fea6d0e5f..919fe32f1 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -121,11 +121,6 @@ struct ValidPathInfo std::string_view name, ContentAddressWithReferences && ca, Hash narHash); virtual ~ValidPathInfo() { } - - static ValidPathInfo read(Source & source, const Store & store, unsigned int format); - static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path); - - void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const; }; typedef std::map ValidPathInfos; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ef648e01b..aaea5c4a2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -332,7 +332,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } info = std::make_shared( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path})); + WorkerProto::Serialise::read(*this, *conn, StorePath{path})); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -445,7 +445,7 @@ ref RemoteStore::addCAToStore( } return make_ref( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion))); + WorkerProto::Serialise::read(*this, *conn)); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); @@ -570,7 +570,12 @@ void RemoteStore::addMultipleToStore( auto source = sinkToSource([&](Sink & sink) { sink << pathsToCopy.size(); for (auto & [pathInfo, pathSource] : pathsToCopy) { - pathInfo.write(sink, *this, 16); + WorkerProto::Serialise::write(*this, + WorkerProto::WriteConn { + .to = sink, + .version = 16, + }, + pathInfo); pathSource->drainInto(sink); } }); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0c58cca0c..785b26062 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,6 +11,9 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +// FIXME this should not be here, see TODO below on +// `addMultipleToStore`. +#include "worker-protocol.hh" #include #include @@ -357,7 +360,13 @@ void Store::addMultipleToStore( { auto expected = readNum(source); for (uint64_t i = 0; i < expected; ++i) { - auto info = ValidPathInfo::read(source, *this, 16); + // FIXME we should not be using the worker protocol here, let + // alone the worker protocol with a hard-coded version! + auto info = WorkerProto::Serialise::read(*this, + WorkerProto::ReadConn { + .from = source, + .version = 16, + }); info.ultimate = false; addToStore(info, source, repair, checkSigs); } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index f69322a61..94f8c8811 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -6,7 +6,7 @@ #include "worker-protocol.hh" #include "worker-protocol-impl.hh" #include "archive.hh" -#include "derivations.hh" +#include "path-info.hh" #include @@ -142,4 +142,47 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } +ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +{ + auto path = WorkerProto::Serialise::read(store, conn); + return WorkerProto::Serialise::read(store, conn, std::move(path)); +} + +ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn, StorePath && path) +{ + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + ValidPathInfo info(path, narHash); + if (deriver != "") info.deriver = store.parseStorePath(deriver); + info.references = WorkerProto::Serialise::read(store, conn); + conn.from >> info.registrationTime >> info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.from >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + } + return info; +} + +void WorkerProto::Serialise::write( + const Store & store, + WriteConn conn, + const ValidPathInfo & pathInfo, + bool includePath) +{ + if (includePath) + conn.to << store.printStorePath(pathInfo.path); + conn.to + << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") + << pathInfo.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(store, conn, pathInfo.references); + conn.to << pathInfo.registrationTime << pathInfo.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.to + << pathInfo.ultimate + << pathInfo.sigs + << renderContentAddress(pathInfo.ca); + } +} + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 2d55d926a..d35e3c678 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -31,6 +31,7 @@ struct Source; struct DerivedPath; struct BuildResult; struct KeyedBuildResult; +struct ValidPathInfo; enum TrustedFlag : bool; @@ -220,4 +221,15 @@ template DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ +/* These are a non-standard form for historical reasons. */ + +template<> +struct WorkerProto::Serialise +{ + static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn); + static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn, StorePath && path); + + static void write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo, bool includePath = true); +}; + } From 70f8b96c11af275b5766dca0a49737803d1e0339 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 17:13:44 -0400 Subject: [PATCH 072/100] Factor out `UnkeyedValidPathInfo` and test This makes the path info serialisers ideomatic again, which allows me to test them. --- src/libstore/daemon.cc | 2 +- src/libstore/path-info.cc | 23 ++- src/libstore/path-info.hh | 31 ++-- src/libstore/remote-store.cc | 3 +- src/libstore/tests/worker-protocol.cc | 153 ++++++++++++++++++ src/libstore/worker-protocol.cc | 24 +-- src/libstore/worker-protocol.hh | 16 +- .../unkeyed-valid-path-info-1.15.bin | Bin 0 -> 328 bytes .../worker-protocol/valid-path-info-1.15.bin | Bin 0 -> 488 bytes .../worker-protocol/valid-path-info-1.16.bin | Bin 0 -> 952 bytes 10 files changed, 218 insertions(+), 34 deletions(-) create mode 100644 unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin create mode 100644 unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin create mode 100644 unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0afaaccba..007ffc05a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -819,7 +819,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - WorkerProto::Serialise::write(*store, wconn, *info, false); + WorkerProto::write(*store, wconn, static_cast(*info)); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 51fb5f02a..ab39e71f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,6 +3,25 @@ namespace nix { +GENERATE_CMP_EXT( + , + UnkeyedValidPathInfo, + me->deriver, + me->narHash, + me->references, + me->registrationTime, + me->narSize, + //me->id, + me->ultimate, + me->sigs, + me->ca); + +GENERATE_CMP_EXT( + , + ValidPathInfo, + me->path, + static_cast(*me)); + std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) @@ -102,8 +121,8 @@ ValidPathInfo::ValidPathInfo( std::string_view name, ContentAddressWithReferences && ca, Hash narHash) - : path(store.makeFixedOutputPathFromCA(name, ca)) - , narHash(narHash) + : UnkeyedValidPathInfo(narHash) + , path(store.makeFixedOutputPathFromCA(name, ca)) { std::visit(overloaded { [this](TextInfo && ti) { diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 919fe32f1..a82e643ae 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -32,9 +32,8 @@ struct SubstitutablePathInfo typedef std::map SubstitutablePathInfos; -struct ValidPathInfo +struct UnkeyedValidPathInfo { - StorePath path; std::optional deriver; /** * \todo document this @@ -72,6 +71,20 @@ struct ValidPathInfo */ std::optional ca; + UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default; + + UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; + + DECLARE_CMP(UnkeyedValidPathInfo); + + virtual ~UnkeyedValidPathInfo() { } +}; + +struct ValidPathInfo : UnkeyedValidPathInfo { + StorePath path; + + DECLARE_CMP(ValidPathInfo); + /** * Return a fingerprint of the store path to be used in binary * cache signatures. It contains the store path, the base-32 @@ -84,11 +97,11 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - /** - * @return The `ContentAddressWithReferences` that determines the - * store path for a content-addressed store object, `std::nullopt` - * for an input-addressed store object. - */ + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ std::optional contentAddressWithReferences() const; /** @@ -114,8 +127,8 @@ struct ValidPathInfo ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; - ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; + ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(path) { }; ValidPathInfo(const Store & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index aaea5c4a2..7bdc25433 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -332,7 +332,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } info = std::make_shared( - WorkerProto::Serialise::read(*this, *conn, StorePath{path})); + StorePath{path}, + WorkerProto::Serialise::read(*this, *conn)); } callback(std::move(info)); } catch (...) { callback.rethrow(); } diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc index e0ce340d8..ad5943c69 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/src/libstore/tests/worker-protocol.cc @@ -329,6 +329,159 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + unkeyedValidPathInfo_1_15, + "unkeyed-valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_15, + "valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_16, + "valid-path-info-1.16", + 1 << 8 | 16, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.ultimate = true; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalTrustedFlag, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 94f8c8811..d618b9bd8 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -145,14 +145,24 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); - return WorkerProto::Serialise::read(store, conn, std::move(path)); + return ValidPathInfo { + std::move(path), + WorkerProto::Serialise::read(store, conn), + }; } -ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn, StorePath && path) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo) +{ + WorkerProto::write(store, conn, pathInfo.path); + WorkerProto::write(store, conn, static_cast(pathInfo)); +} + + +UnkeyedValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { auto deriver = readString(conn.from); auto narHash = Hash::parseAny(readString(conn.from), htSHA256); - ValidPathInfo info(path, narHash); + UnkeyedValidPathInfo info(narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise::read(store, conn); conn.from >> info.registrationTime >> info.narSize; @@ -164,14 +174,8 @@ ValidPathInfo WorkerProto::Serialise::read(const Store & store, R return info; } -void WorkerProto::Serialise::write( - const Store & store, - WriteConn conn, - const ValidPathInfo & pathInfo, - bool includePath) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { - if (includePath) - conn.to << store.printStorePath(pathInfo.path); conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") << pathInfo.narHash.to_string(HashFormat::Base16, false); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index d35e3c678..dcd54ad16 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -32,6 +32,7 @@ struct DerivedPath; struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; +struct UnkeyedValidPathInfo; enum TrustedFlag : bool; @@ -207,6 +208,10 @@ DECLARE_WORKER_SERIALISER(BuildResult); template<> DECLARE_WORKER_SERIALISER(KeyedBuildResult); template<> +DECLARE_WORKER_SERIALISER(ValidPathInfo); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); +template<> DECLARE_WORKER_SERIALISER(std::optional); template @@ -221,15 +226,4 @@ template DECLARE_WORKER_SERIALISER(std::map); #undef COMMA_ -/* These are a non-standard form for historical reasons. */ - -template<> -struct WorkerProto::Serialise -{ - static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn); - static ValidPathInfo read(const Store & store, WorkerProto::ReadConn conn, StorePath && path); - - static void write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo, bool includePath = true); -}; - } diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin new file mode 100644 index 0000000000000000000000000000000000000000..e69ccbe83862b29e3f79810c0c2ff4561a42f86f GIT binary patch literal 328 zcmbu4%?`pK5QOo8D!&WM#q>Qq0R{CUX=_wYUVUsEAs(9a)(w2^a=MH48Q8;TGRcmlTf(6rfXPFjSJ46(yf&k8}#9A2z znmI2!XwnU&AmvqJYIhQk7+x#{!gQkU@$Il_5ceP*O@N zxu!r+#$?P>4g#ry<$$?w`2MnGPahlJ`HLM!?tH6m@;lyRZZ9kI;P`*_d+&94ym9_l z{ZG@h-`{zOWuKw$x?n$F_k7^>Jh*~Rnvu&5}L}Qp+gUA>+#+bF!C(UNw&;S4c literal 0 HcmV?d00001 From 201a4af9a4447a151682de6705a4a74783bf0e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=A2=D1=80=D0=BE?= =?UTF-8?q?=D1=84=D0=B8=D0=BC=D0=BE=D0=B2?= <62882157+trofkm@users.noreply.github.com> Date: Mon, 23 Oct 2023 01:56:46 +0300 Subject: [PATCH 073/100] Clean up `app.cc` (#9201) - Rename `expected` to `expectedType` - Use early `return` and `continue` to reduce nesting --- .gitignore | 2 ++ src/nix/app.cc | 40 ++++++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 2d3314015..04d96ca2c 100644 --- a/.gitignore +++ b/.gitignore @@ -138,7 +138,9 @@ nix-rust/target result +# IDE .vscode/ +.idea/ # clangd and possibly more .cache/ diff --git a/src/nix/app.cc b/src/nix/app.cc index 34fac9935..935ed18ec 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -20,20 +20,24 @@ StringPairs resolveRewrites( const std::vector & dependencies) { StringPairs res; - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - for (auto & dep : dependencies) { - if (auto drvDep = std::get_if(&dep.path)) { - for (auto & [ outputName, outputPath ] : drvDep->outputs) { - res.emplace( - DownstreamPlaceholder::fromSingleDerivedPathBuilt( - SingleDerivedPath::Built { - .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), - .output = outputName, - }).render(), - store.printStorePath(outputPath) - ); - } - } + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + return res; + } + for (auto &dep: dependencies) { + auto drvDep = std::get_if(&dep.path); + if (!drvDep) { + continue; + } + + for (const auto & [ outputName, outputPath ] : drvDep->outputs) { + res.emplace( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), + .output = outputName, + }).render(), + store.printStorePath(outputPath) + ); } } return res; @@ -58,11 +62,11 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto type = cursor->getAttr("type")->getString(); - std::string expected = !attrPath.empty() && + std::string expectedType = !attrPath.empty() && (state.symbols[attrPath[0]] == "apps" || state.symbols[attrPath[0]] == "defaultApp") ? "app" : "derivation"; - if (type != expected) - throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); + if (type != expectedType) + throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expectedType); if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); @@ -91,7 +95,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) }, c.raw)); } - return UnresolvedApp{App { + return UnresolvedApp { App { .context = std::move(context2), .program = program, }}; From 256dfb98e864e474921ac5a8264e33de6978f47d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 23 Oct 2023 04:05:02 +0200 Subject: [PATCH 074/100] remove Basic Package Management section (#7974) this is the first thing most beginners see, and it misleads them into assuming `nix-env` is appropriate for doing anything but setting and reverting profile generations. this chapter is the root of most evil around the ecosystem, and today we finally close it for good. --- doc/manual/src/SUMMARY.md.in | 1 - .../package-management/basic-package-mgmt.md | 179 ------------------ 2 files changed, 180 deletions(-) delete mode 100644 doc/manual/src/package-management/basic-package-mgmt.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index e49e77cf5..60ebeb138 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,7 +17,6 @@ - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) - [Package Management](package-management/package-management.md) - - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md deleted file mode 100644 index 07b92fb76..000000000 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ /dev/null @@ -1,179 +0,0 @@ -# Basic Package Management - -The main command for package management is -[`nix-env`](../command-ref/nix-env.md). You can use it to install, -upgrade, and erase packages, and to query what packages are installed -or are available for installation. - -In Nix, different users can have different “views” on the set of -installed applications. That is, there might be lots of applications -present on the system (possibly in many different versions), but users -can have a specific selection of those active — where “active” just -means that it appears in a directory in the user’s `PATH`. Such a view -on the set of installed applications is called a *user environment*, -which is just a directory tree consisting of symlinks to the files of -the active applications. - -Components are installed from a set of *Nix expressions* that tell Nix -how to build those packages, including, if necessary, their -dependencies. There is a collection of Nix expressions called the -Nixpkgs package collection that contains packages ranging from basic -development stuff such as GCC and Glibc, to end-user applications like -Mozilla Firefox. (Nix is however not tied to the Nixpkgs package -collection; you could write your own Nix expressions based on Nixpkgs, -or completely new ones.) - -You can manually download the latest version of Nixpkgs from -. However, it’s much more -convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes -it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is -automatically added to your list of “subscribed” channels when you -install Nix. If this is not the case for some reason, you can add it -as follows: - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -$ nix-channel --update -``` - -> **Note** -> -> On NixOS, you’re automatically subscribed to a NixOS channel -> corresponding to your NixOS major release (e.g. -> ). A NixOS channel is identical -> to the Nixpkgs channel, except that it contains only Linux binaries -> and is updated only if a set of regression tests succeed. - -You can view the set of available packages in Nixpkgs: - -```console -$ nix-env --query --available --attr-path -nixpkgs.aterm aterm-2.2 -nixpkgs.bash bash-3.0 -nixpkgs.binutils binutils-2.15 -nixpkgs.bison bison-1.875d -nixpkgs.blackdown blackdown-1.4.2 -nixpkgs.bzip2 bzip2-1.0.2 -… -``` - -The flag `-q` specifies a query operation, `-a` means that you want -to show the “available” (i.e., installable) packages, as opposed to the -installed packages, and `-P` prints the attribute paths that can be used -to unambiguously select a package for installation (listed in the first column). -If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, -then you need to pass the path to your Nixpkgs tree using the `-f` flag: - -```console -$ nix-env --query --available --attr-path --file /path/to/nixpkgs -aterm aterm-2.2 -bash bash-3.0 -… -``` - -where */path/to/nixpkgs* is where you’ve unpacked or checked out -Nixpkgs. - -You can filter the packages by name: - -```console -$ nix-env --query --available --attr-path firefox -nixpkgs.firefox-esr firefox-91.3.0esr -nixpkgs.firefox firefox-94.0.1 -``` - -and using regular expressions: - -```console -$ nix-env --query --available --attr-path 'firefox.*' -``` - -It is also possible to see the *status* of available packages, i.e., -whether they are installed into the user environment and/or present in -the system: - -```console -$ nix-env --query --available --attr-path --status -… --PS nixpkgs.bash bash-3.0 ---S nixpkgs.binutils binutils-2.15 -IPS nixpkgs.bison bison-1.875d -… -``` - -The first character (`I`) indicates whether the package is installed in -your current user environment. The second (`P`) indicates whether it is -present on your system (in which case installing it into your user -environment would be a very quick operation). The last one (`S`) -indicates whether there is a so-called *substitute* for the package, -which is Nix’s mechanism for doing binary deployment. It just means that -Nix knows that it can fetch a pre-built package from somewhere -(typically a network server) instead of building it locally. - -You can install a package using `nix-env --install --attr `. For instance, - -```console -$ nix-env --install --attr nixpkgs.subversion -``` - -will install the package called `subversion` from `nixpkgs` channel (which is, of course, the -[Subversion version management system](http://subversion.tigris.org/)). - -> **Note** -> -> When you ask Nix to install a package, it will first try to get it in -> pre-compiled form from a *binary cache*. By default, Nix will use the -> binary cache ; it contains binaries for most -> packages in Nixpkgs. Only if no binary is available in the binary -> cache, Nix will build the package from source. So if `nix-env -> -iA nixpkgs.subversion` results in Nix building stuff from source, then either -> the package is not built for your platform by the Nixpkgs build -> servers, or your version of Nixpkgs is too old or too new. For -> instance, if you have a very recent checkout of Nixpkgs, then the -> Nixpkgs build servers may not have had a chance to build everything -> and upload the resulting binaries to . The -> Nixpkgs channel is only updated after all binaries have been uploaded -> to the cache, so if you stick to the Nixpkgs channel (rather than -> using a Git checkout of the Nixpkgs tree), you will get binaries for -> most packages. - -Naturally, packages can also be uninstalled. Unlike when installing, you will -need to use the derivation name (though the version part can be omitted), -instead of the attribute path, as `nix-env` does not record which attribute -was used for installing: - -```console -$ nix-env --uninstall subversion -``` - -Upgrading to a new version is just as easy. If you have a new release of -Nix Packages, you can do: - -```console -$ nix-env --upgrade --attr nixpkgs.subversion -``` - -This will *only* upgrade Subversion if there is a “newer” version in the -new set of Nix expressions, as defined by some pretty arbitrary rules -regarding ordering of version numbers (which generally do what you’d -expect of them). To just unconditionally replace Subversion with -whatever version is in the Nix expressions, use `-i` instead of `-u`; -`-i` will remove whatever version is already installed. - -You can also upgrade all packages for which there are newer versions: - -```console -$ nix-env --upgrade -``` - -Sometimes it’s useful to be able to ask what `nix-env` would do, without -actually doing it. For instance, to find out what packages would be -upgraded by `nix-env --upgrade `, you can do - -```console -$ nix-env --upgrade --dry-run -(dry run; not doing anything) -upgrading `libxslt-1.1.0' to `libxslt-1.1.10' -upgrading `graphviz-1.10' to `graphviz-1.12' -upgrading `coreutils-5.0' to `coreutils-5.2.1' -``` From 34a42f0d0ac20a70fdcc3dbf99a212b523c5f2d4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Oct 2023 11:05:50 +0200 Subject: [PATCH 075/100] Move PosixSourceAccessor into its own file --- src/libfetchers/fs-input-accessor.cc | 1 + src/libutil/archive.cc | 2 +- src/libutil/posix-source-accessor.cc | 86 ++++++++++++++++++++++++++++ src/libutil/posix-source-accessor.hh | 34 +++++++++++ src/libutil/source-accessor.cc | 81 -------------------------- src/libutil/source-accessor.hh | 27 --------- 6 files changed, 122 insertions(+), 109 deletions(-) create mode 100644 src/libutil/posix-source-accessor.cc create mode 100644 src/libutil/posix-source-accessor.hh diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 3444c4643..7638d2d82 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -1,4 +1,5 @@ #include "fs-input-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" namespace nix { diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 0cd54e5db..3b1a1e0ef 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -14,7 +14,7 @@ #include "archive.hh" #include "util.hh" #include "config.hh" -#include "source-accessor.hh" +#include "posix-source-accessor.hh" namespace nix { diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc new file mode 100644 index 000000000..48b4fe626 --- /dev/null +++ b/src/libutil/posix-source-accessor.cc @@ -0,0 +1,86 @@ +#include "posix-source-accessor.hh" + +namespace nix { + +void PosixSourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + // FIXME: add O_NOFOLLOW since symlinks should be resolved by the + // caller? + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting file"); + + sizeCallback(st.st_size); + + off_t left = st.st_size; + + std::vector buf(64 * 1024); + while (left) { + checkInterrupt(); + ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file '%s'", showPath(path)); + } + else if (rd == 0) + throw SysError("unexpected end-of-file reading '%s'", showPath(path)); + else { + assert(rd <= left); + sink({(char *) buf.data(), (size_t) rd}); + left -= rd; + } + } +} + +bool PosixSourceAccessor::pathExists(const CanonPath & path) +{ + return nix::pathExists(path.abs()); +} + +SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +{ + auto st = nix::lstat(path.abs()); + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; +} + +SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +{ + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +std::string PosixSourceAccessor::readLink(const CanonPath & path) +{ + return nix::readLink(path.abs()); +} + +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + return path; +} + +} diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh new file mode 100644 index 000000000..608f96ee2 --- /dev/null +++ b/src/libutil/posix-source-accessor.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +/** + * A source accessor that uses the Unix filesystem. + */ +struct PosixSourceAccessor : SourceAccessor +{ + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override; + + bool pathExists(const CanonPath & path) override; + + Stat lstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; +}; + +} diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 2d03d3d7a..d168a9667 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -55,85 +55,4 @@ std::string SourceAccessor::showPath(const CanonPath & path) return path.abs(); } -void PosixSourceAccessor::readFile( - const CanonPath & path, - Sink & sink, - std::function sizeCallback) -{ - // FIXME: add O_NOFOLLOW since symlinks should be resolved by the - // caller? - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("statting file"); - - sizeCallback(st.st_size); - - off_t left = st.st_size; - - std::vector buf(64 * 1024); - while (left) { - checkInterrupt(); - ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading from file '%s'", showPath(path)); - } - else if (rd == 0) - throw SysError("unexpected end-of-file reading '%s'", showPath(path)); - else { - assert(rd <= left); - sink({(char *) buf.data(), (size_t) rd}); - left -= rd; - } - } -} - -bool PosixSourceAccessor::pathExists(const CanonPath & path) -{ - return nix::pathExists(path.abs()); -} - -SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) -{ - auto st = nix::lstat(path.abs()); - mtime = std::max(mtime, st.st_mtime); - return Stat { - .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : - tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; -} - -SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) -{ - DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = Type::tRegular; break; - case DT_LNK: type = Type::tSymlink; break; - case DT_DIR: type = Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; -} - -std::string PosixSourceAccessor::readLink(const CanonPath & path) -{ - return nix::readLink(path.abs()); -} - -std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) -{ - return path; -} - } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index f3504c9bb..fd823aa39 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -104,31 +104,4 @@ struct SourceAccessor virtual std::string showPath(const CanonPath & path); }; -/** - * A source accessor that uses the Unix filesystem. - */ -struct PosixSourceAccessor : SourceAccessor -{ - /** - * The most recent mtime seen by lstat(). This is a hack to - * support dumpPathAndGetMtime(). Should remove this eventually. - */ - time_t mtime = 0; - - void readFile( - const CanonPath & path, - Sink & sink, - std::function sizeCallback) override; - - bool pathExists(const CanonPath & path) override; - - Stat lstat(const CanonPath & path) override; - - DirEntries readDirectory(const CanonPath & path) override; - - std::string readLink(const CanonPath & path) override; - - std::optional getPhysicalPath(const CanonPath & path) override; -}; - } From b461cac21aa5b9127d230ed3f1a1ca8e8266fb60 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Oct 2023 09:03:11 -0400 Subject: [PATCH 076/100] Overhaul completions, redo #6693 (#8131) As I complained in https://github.com/NixOS/nix/pull/6784#issuecomment-1421777030 (a comment on the wrong PR, sorry again!), #6693 introduced a second completions mechanism to fix a bug. Having two completion mechanisms isn't so nice. As @thufschmitt also pointed out, it was a bummer to go from `FlakeRef` to `std::string` when collecting flake refs. Now it is `FlakeRefs` again. The underlying issue that sought to work around was that completion of arguments not at the end can still benefit from the information from latter arguments. To fix this better, we rip out that change and simply defer all completion processing until after all the (regular, already-complete) arguments have been passed. In addition, I noticed the original completion logic used some global variables. I do not like global variables, because even if they save lines of code, they also obfuscate the architecture of the code. I got rid of them moved them to a new `RootArgs` class, which now has `parseCmdline` instead of `Args`. The idea is that we have many argument parsers from subcommands and what-not, but only one root args that owns the other per actual parsing invocation. The state that was global is now part of the root args instead. This did, admittedly, add a bunch of new code. And I do feel bad about that. So I went and added a lot of API docs to try to at least make the current state of things clear to the next person. -- This is needed for RFC 134 (tracking issue #7868). It was very hard to modularize `Installable` parsing when there were two completion arguments. I wouldn't go as far as to say it is *easy* now, but at least it is less hard (and the completions test finally passed). Co-authored-by: Valentin Gagarin --- src/libcmd/command.hh | 26 ++++---- src/libcmd/common-eval-args.cc | 4 +- src/libcmd/installables.cc | 114 +++++++++++++++++++-------------- src/libmain/common-args.cc | 9 +-- src/libmain/shared.hh | 3 +- src/libutil/args.cc | 111 ++++++++++++++++++++------------ src/libutil/args.hh | 109 ++++++++++++++++++++----------- src/libutil/args/root.hh | 72 +++++++++++++++++++++ src/libutil/local.mk | 5 ++ src/nix/bundle.cc | 4 +- src/nix/flake.cc | 14 ++-- src/nix/main.cc | 27 ++++---- src/nix/registry.cc | 4 +- src/nix/why-depends.cc | 8 +-- 14 files changed, 329 insertions(+), 181 deletions(-) create mode 100644 src/libutil/args/root.hh diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 5c4569001..dafc0db3b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -97,8 +97,6 @@ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; - std::optional needsFlakeInputCompletion = {}; - MixFlakeOptions(); /** @@ -109,12 +107,8 @@ struct MixFlakeOptions : virtual Args, EvalCommand * command is operating with (presumably specified via some other * arguments) so that the completions for these flags can use them. */ - virtual std::vector getFlakesForCompletion() + virtual std::vector getFlakeRefsForCompletion() { return {}; } - - void completeFlakeInput(std::string_view prefix); - - void completionHook() override; }; struct SourceExprCommand : virtual Args, MixFlakeOptions @@ -137,7 +131,13 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions /** * Complete an installable from the given prefix. */ - void completeInstallable(std::string_view prefix); + void completeInstallable(AddCompletions & completions, std::string_view prefix); + + /** + * Convenience wrapper around the underlying function to make setting the + * callback easier. + */ + CompleterClosure getCompleteInstallable(); }; /** @@ -170,7 +170,7 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand bool readFromStdIn = false; - std::vector getFlakesForCompletion() override; + std::vector getFlakeRefsForCompletion() override; private: @@ -199,10 +199,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand void run(ref store) override; - std::vector getFlakesForCompletion() override - { - return {_installable}; - } + std::vector getFlakeRefsForCompletion() override; private: @@ -329,9 +326,10 @@ struct MixEnvironment : virtual Args { void setEnviron(); }; -void completeFlakeRef(ref store, std::string_view prefix); +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix); void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e6df49a7a..e53bc4c01 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -133,8 +133,8 @@ MixEvalArgs::MixEvalArgs() if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(openStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, openStore(), prefix); }} }); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 01c01441c..eff18bbf6 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -28,6 +28,20 @@ namespace nix { +static void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix) +{ + for (auto & flakeRef : flakeRefs) { + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions.add(input.first); + } +} + MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; @@ -79,8 +93,8 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, - .completer = {[&](size_t, std::string_view prefix) { - needsFlakeInputCompletion = {std::string(prefix)}; + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); }} }); @@ -95,11 +109,12 @@ MixFlakeOptions::MixFlakeOptions() flake::parseInputPath(inputPath), parseFlakeRef(flakeRef, absPath("."), true)); }}, - .completer = {[&](size_t n, std::string_view prefix) { - if (n == 0) - needsFlakeInputCompletion = {std::string(prefix)}; - else if (n == 1) - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { + if (n == 0) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + } else if (n == 1) { + completeFlakeRef(completions, getEvalState()->store, prefix); + } }} }); @@ -146,30 +161,12 @@ MixFlakeOptions::MixFlakeOptions() } } }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getEvalState()->store, prefix); }} }); } -void MixFlakeOptions::completeFlakeInput(std::string_view prefix) -{ - auto evalState = getEvalState(); - for (auto & flakeRefS : getFlakesForCompletion()) { - auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; - auto flake = flake::getFlake(*evalState, flakeRef, true); - for (auto & input : flake.inputs) - if (hasPrefix(input.first, prefix)) - completions->add(input.first); - } -} - -void MixFlakeOptions::completionHook() -{ - if (auto & prefix = needsFlakeInputCompletion) - completeFlakeInput(*prefix); -} - SourceExprCommand::SourceExprCommand() { addFlag({ @@ -226,11 +223,18 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() }; } -void SourceExprCommand::completeInstallable(std::string_view prefix) +Args::CompleterClosure SourceExprCommand::getCompleteInstallable() +{ + return [this](AddCompletions & completions, size_t, std::string_view prefix) { + completeInstallable(completions, prefix); + }; +} + +void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix) { try { if (file) { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); evalSettings.pureEval = false; auto state = getEvalState(); @@ -265,14 +269,15 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) std::string name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions->add(name); + completions.add(name); else - completions->add(prefix_ + "." + name); + completions.add(prefix_ + "." + name); } } } } else { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, getDefaultFlakeAttrPathPrefixes(), @@ -285,6 +290,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) } void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, @@ -296,9 +302,9 @@ void completeFlakeRefWithFragment( try { auto hash = prefix.find('#'); if (hash == std::string::npos) { - completeFlakeRef(evalState->store, prefix); + completeFlakeRef(completions, evalState->store, prefix); } else { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); auto fragment = prefix.substr(hash + 1); std::string prefixRoot = ""; @@ -341,7 +347,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); + completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -352,7 +358,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->add(flakeRefS + "#" + prefixRoot); + completions.add(flakeRefS + "#" + prefixRoot); } } } @@ -361,15 +367,15 @@ void completeFlakeRefWithFragment( } } -void completeFlakeRef(ref store, std::string_view prefix) +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) return; if (prefix == "") - completions->add("."); + completions.add("."); - completeDir(0, prefix); + Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ for (auto & registry : fetchers::getRegistries(store)) { @@ -378,10 +384,10 @@ void completeFlakeRef(ref store, std::string_view prefix) if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) - completions->add(from2); + completions.add(from2); } else { if (hasPrefix(from, prefix)) - completions->add(from); + completions.add(from); } } } @@ -747,9 +753,7 @@ RawInstallablesCommand::RawInstallablesCommand() expectArgs({ .label = "installables", .handler = {&rawInstallables}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } @@ -762,6 +766,17 @@ void RawInstallablesCommand::applyDefaultInstallables(std::vector & } } +std::vector RawInstallablesCommand::getFlakeRefsForCompletion() +{ + applyDefaultInstallables(rawInstallables); + std::vector res; + for (auto i : rawInstallables) + res.push_back(parseFlakeRefWithFragment( + expandTilde(i), + absPath(".")).first); + return res; +} + void RawInstallablesCommand::run(ref store) { if (readFromStdIn && !isatty(STDIN_FILENO)) { @@ -775,10 +790,13 @@ void RawInstallablesCommand::run(ref store) run(store, std::move(rawInstallables)); } -std::vector RawInstallablesCommand::getFlakesForCompletion() +std::vector InstallableCommand::getFlakeRefsForCompletion() { - applyDefaultInstallables(rawInstallables); - return rawInstallables; + return { + parseFlakeRefWithFragment( + expandTilde(_installable), + absPath(".")).first + }; } void InstallablesCommand::run(ref store, std::vector && rawInstallables) @@ -794,9 +812,7 @@ InstallableCommand::InstallableCommand() .label = "installable", .optional = true, .handler = {&_installable}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index f92920d18..205b77808 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,4 +1,5 @@ #include "common-args.hh" +#include "args/root.hh" #include "globals.hh" #include "loggers.hh" @@ -34,21 +35,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .category = miscCategory, .labels = {"name", "value"}, - .handler = {[](std::string name, std::string value) { + .handler = {[this](std::string name, std::string value) { try { globalConfig.set(name, value); } catch (UsageError & e) { - if (!completions) + if (!getRoot().completions) warn(e.what()); } }}, - .completer = [](size_t index, std::string_view prefix) { + .completer = [](AddCompletions & completions, size_t index, std::string_view prefix) { if (index == 0) { std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->add(s.first, fmt("Set the `%s` setting.", s.first)); + completions.add(s.first, fmt("Set the `%s` setting.", s.first)); } } }); diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 9415be78a..3159fe479 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -3,6 +3,7 @@ #include "util.hh" #include "args.hh" +#include "args/root.hh" #include "common-args.hh" #include "path.hh" #include "derived-path.hh" @@ -66,7 +67,7 @@ template N getIntArg(const std::string & opt, } -struct LegacyArgs : public MixCommonArgs +struct LegacyArgs : public MixCommonArgs, public RootArgs { std::function parseArg; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index e410c7eec..6bc3cae07 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,4 +1,5 @@ #include "args.hh" +#include "args/root.hh" #include "hash.hh" #include "json-utils.hh" @@ -26,6 +27,11 @@ void Args::removeFlag(const std::string & longName) longFlags.erase(flag); } +void Completions::setType(AddCompletions::Type t) +{ + type = t; +} + void Completions::add(std::string completion, std::string description) { description = trim(description); @@ -37,7 +43,7 @@ void Completions::add(std::string completion, std::string description) if (needs_ellipsis) description.append(" [...]"); } - insert(Completion { + completions.insert(Completion { .completion = completion, .description = description }); @@ -46,12 +52,20 @@ void Completions::add(std::string completion, std::string description) bool Completion::operator<(const Completion & other) const { return completion < other.completion || (completion == other.completion && description < other.description); } -CompletionType completionType = ctNormal; -std::shared_ptr completions; - std::string completionMarker = "___COMPLETE___"; -static std::optional needsCompletion(std::string_view s) +RootArgs & Args::getRoot() +{ + Args * p = this; + while (p->parent) + p = p->parent; + + auto * res = dynamic_cast(p); + assert(res); + return *res; +} + +std::optional RootArgs::needsCompletion(std::string_view s) { if (!completions) return {}; auto i = s.find(completionMarker); @@ -60,7 +74,7 @@ static std::optional needsCompletion(std::string_view s) return {}; } -void Args::parseCmdline(const Strings & _cmdline) +void RootArgs::parseCmdline(const Strings & _cmdline) { Strings pendingArgs; bool dashDash = false; @@ -71,7 +85,7 @@ void Args::parseCmdline(const Strings & _cmdline) size_t n = std::stoi(*s); assert(n > 0 && n <= cmdline.size()); *std::next(cmdline.begin(), n - 1) += completionMarker; - completions = std::make_shared(); + completions = std::make_shared(); verbosity = lvlError; } @@ -125,17 +139,23 @@ void Args::parseCmdline(const Strings & _cmdline) for (auto & f : flagExperimentalFeatures) experimentalFeatureSettings.require(f); + /* Now that all the other args are processed, run the deferred completions. + */ + for (auto d : deferredCompletions) + d.completer(*completions, d.n, d.prefix); } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); + auto & rootArgs = getRoot(); + auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; if (auto & f = flag.experimentalFeature) - flagExperimentalFeatures.insert(*f); + rootArgs.flagExperimentalFeatures.insert(*f); std::vector args; bool anyCompleted = false; @@ -146,10 +166,15 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) "flag '%s' requires %d argument(s), but only %d were given", name, flag.handler.arity, n); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { anyCompleted = true; - if (flag.completer) - flag.completer(n, *prefix); + if (flag.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = flag.completer, + .n = n, + .prefix = *prefix, + }); + } } args.push_back(*pos++); } @@ -159,14 +184,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) }; if (std::string(*pos, 0, 2) == "--") { - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) { if (auto & f = flag->experimentalFeature) - flagExperimentalFeatures.insert(*f); - completions->add("--" + name, flag->description); + rootArgs.flagExperimentalFeatures.insert(*f); + rootArgs.completions->add("--" + name, flag->description); } } return false; @@ -183,12 +208,12 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) return process(std::string("-") + c, *i->second); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { if (prefix == "-") { - completions->add("--"); + rootArgs.completions->add("--"); for (auto & [flagName, flag] : shortFlags) if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) - completions->add(std::string("-") + flagName, flag->description); + rootArgs.completions->add(std::string("-") + flagName, flag->description); } } @@ -203,6 +228,8 @@ bool Args::processArgs(const Strings & args, bool finish) return true; } + auto & rootArgs = getRoot(); + auto & exp = expectedArgs.front(); bool res = false; @@ -211,15 +238,23 @@ bool Args::processArgs(const Strings & args, bool finish) (exp.handler.arity != ArityAny && args.size() == exp.handler.arity)) { std::vector ss; + bool anyCompleted = false; for (const auto &[n, s] : enumerate(args)) { - if (auto prefix = needsCompletion(s)) { + if (auto prefix = rootArgs.needsCompletion(s)) { + anyCompleted = true; ss.push_back(*prefix); - if (exp.completer) - exp.completer(n, *prefix); + if (exp.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = exp.completer, + .n = n, + .prefix = *prefix, + }); + } } else ss.push_back(s); } - exp.handler.fun(ss); + if (!anyCompleted) + exp.handler.fun(ss); expectedArgs.pop_front(); res = true; } @@ -271,11 +306,11 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) - completions->add(type); + completions.add(type); } Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) @@ -287,7 +322,7 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) .handler = {[ht](std::string s) { *ht = parseHashType(s); }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } @@ -300,13 +335,13 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< .handler = {[oht](std::string s) { *oht = std::optional { parseHashType(s) }; }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } -static void _completePath(std::string_view prefix, bool onlyDirs) +static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs) { - completionType = ctFilenames; + completions.setType(Completions::Type::Filenames); glob_t globbuf; int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR @@ -320,20 +355,20 @@ static void _completePath(std::string_view prefix, bool onlyDirs) auto st = stat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } - completions->add(globbuf.gl_pathv[i]); + completions.add(globbuf.gl_pathv[i]); } } globfree(&globbuf); } -void completePath(size_t, std::string_view prefix) +void Args::completePath(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, false); + _completePath(completions, prefix, false); } -void completeDir(size_t, std::string_view prefix) +void Args::completeDir(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, true); + _completePath(completions, prefix, true); } Strings argvToStrings(int argc, char * * argv) @@ -368,10 +403,10 @@ MultiCommand::MultiCommand(const Commands & commands_) command = {s, i->second()}; command->second->parent = this; }}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { for (auto & [name, command] : commands) if (hasPrefix(name, prefix)) - completions->add(name); + completions.add(name); }} }); @@ -393,14 +428,6 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } -void MultiCommand::completionHook() -{ - if (command) - return command->second->completionHook(); - else - return Args::completionHook(); -} - nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 6457cceed..a5d7cbe4a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -15,16 +15,14 @@ enum HashType : char; class MultiCommand; +class RootArgs; + +class AddCompletions; + class Args { public: - /** - * Parse the command line, throwing a UsageError if something goes - * wrong. - */ - void parseCmdline(const Strings & cmdline); - /** * Return a short one-line description of the command. */ @@ -123,6 +121,25 @@ protected: { } }; + /** + * The basic function type of the completion callback. + * + * Used to define `CompleterClosure` and some common case completers + * that individual flags/arguments can use. + * + * The `AddCompletions` that is passed is an interface to the state + * stored as part of the root command + */ + typedef void CompleterFun(AddCompletions &, size_t, std::string_view); + + /** + * The closure type of the completion callback. + * + * This is what is actually stored as part of each Flag / Expected + * Arg. + */ + typedef std::function CompleterClosure; + /** * Description of flags / options * @@ -140,7 +157,7 @@ protected: std::string category; Strings labels; Handler handler; - std::function completer; + CompleterClosure completer; std::optional experimentalFeature; @@ -177,7 +194,7 @@ protected: std::string label; bool optional = false; Handler handler; - std::function completer; + CompleterClosure completer; }; /** @@ -211,13 +228,6 @@ protected: */ virtual void initialFlagsProcessed() {} - /** - * Called after the command line has been processed if we need to generate - * completions. Useful for commands that need to know the whole command line - * in order to know what completions to generate. - */ - virtual void completionHook() { } - public: void addFlag(Flag && flag); @@ -252,24 +262,30 @@ public: }); } + static CompleterFun completePath; + + static CompleterFun completeDir; + virtual nlohmann::json toJSON(); friend class MultiCommand; /** * The parent command, used if this is a subcommand. + * + * Invariant: An Args with a null parent must also be a RootArgs + * + * \todo this would probably be better in the CommandClass. + * getRoot() could be an abstract method that peels off at most one + * layer before recuring. */ MultiCommand * parent = nullptr; -private: - /** - * Experimental features needed when parsing args. These are checked - * after flag parsing is completed in order to support enabling - * experimental features coming after the flag that needs the - * experimental feature. + * Traverse parent pointers until we find the \ref RootArgs "root + * arguments" object. */ - std::set flagExperimentalFeatures; + RootArgs & getRoot(); }; /** @@ -320,8 +336,6 @@ public: bool processArgs(const Strings & args, bool finish) override; - void completionHook() override; - nlohmann::json toJSON() override; }; @@ -333,21 +347,40 @@ struct Completion { bool operator<(const Completion & other) const; }; -class Completions : public std::set { + +/** + * The abstract interface for completions callbacks + * + * The idea is to restrict the callback so it can only add additional + * completions to the collection, or set the completion type. By making + * it go through this interface, the callback cannot make any other + * changes, or even view the completions / completion type that have + * been set so far. + */ +class AddCompletions +{ public: - void add(std::string completion, std::string description = ""); + + /** + * The type of completion we are collecting. + */ + enum class Type { + Normal, + Filenames, + Attrs, + }; + + /** + * Set the type of the completions being collected + * + * \todo it should not be possible to change the type after it has been set. + */ + virtual void setType(Type type) = 0; + + /** + * Add a single completion to the collection + */ + virtual void add(std::string completion, std::string description = "") = 0; }; -extern std::shared_ptr completions; - -enum CompletionType { - ctNormal, - ctFilenames, - ctAttrs -}; -extern CompletionType completionType; - -void completePath(size_t, std::string_view prefix); - -void completeDir(size_t, std::string_view prefix); } diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh new file mode 100644 index 000000000..bb98732a1 --- /dev/null +++ b/src/libutil/args/root.hh @@ -0,0 +1,72 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +/** + * The concrete implementation of a collection of completions. + * + * This is exposed so that the main entry point can print out the + * collected completions. + */ +struct Completions final : AddCompletions +{ + std::set completions; + Type type = Type::Normal; + + void setType(Type type) override; + void add(std::string completion, std::string description = "") override; +}; + +/** + * The outermost Args object. This is the one we will actually parse a command + * line with, whereas the inner ones (if they exists) are subcommands (and this + * is also a MultiCommand or something like it). + * + * This Args contains completions state shared between it and all of its + * descendent Args. + */ +class RootArgs : virtual public Args +{ +public: + /** Parse the command line, throwing a UsageError if something goes + * wrong. + */ + void parseCmdline(const Strings & cmdline); + + std::shared_ptr completions; + +protected: + + friend class Args; + + /** + * A pointer to the completion and its two arguments; a thunk; + */ + struct DeferredCompletion { + const CompleterClosure & completer; + size_t n; + std::string prefix; + }; + + /** + * Completions are run after all args and flags are parsed, so completions + * of earlier arguments can benefit from later arguments. + */ + std::vector deferredCompletions; + + /** + * Experimental features needed when parsing args. These are checked + * after flag parsing is completed in order to support enabling + * experimental features coming after the flag that needs the + * experimental feature. + */ + std::set flagExperimentalFeatures; + +private: + + std::optional needsCompletion(std::string_view s); +}; + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk index f880c0fc5..81efaafec 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,8 +6,13 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) +libutil_CXXFLAGS += -I src/libutil + libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context +$(foreach i, $(wildcard $(d)/args/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644))) + ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fbc83b08e..504e35c81 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -21,8 +21,8 @@ struct CmdBundle : InstallableValueCommand .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b4e4156c0..0116eff2e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -36,8 +36,8 @@ public: .label = "flake-url", .optional = true, .handler = {&flakeUrl}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } @@ -52,9 +52,12 @@ public: return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); } - std::vector getFlakesForCompletion() override + std::vector getFlakeRefsForCompletion() override { - return {flakeUrl}; + return { + // Like getFlakeRef but with expandTilde calld first + parseFlakeRef(expandTilde(flakeUrl), absPath(".")) + }; } }; @@ -762,8 +765,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, defaultTemplateAttrPathsPrefixes, diff --git a/src/nix/main.cc b/src/nix/main.cc index ee3878cc7..ffba10099 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,5 +1,6 @@ #include +#include "args/root.hh" #include "command.hh" #include "common-args.hh" #include "eval.hh" @@ -56,7 +57,7 @@ static bool haveInternet() std::string programPath; -struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs { bool useNet = true; bool refresh = false; @@ -241,10 +242,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) static NixArgs & getNixArgs(Command & cmd) { - assert(cmd.parent); - MultiCommand * toplevel = cmd.parent; - while (toplevel->parent) toplevel = toplevel->parent; - return dynamic_cast(*toplevel); + return dynamic_cast(cmd.getRoot()); } struct CmdHelp : Command @@ -412,16 +410,16 @@ void mainWrapped(int argc, char * * argv) Finally printCompletions([&]() { - if (completions) { - switch (completionType) { - case ctNormal: + if (args.completions) { + switch (args.completions->type) { + case Completions::Type::Normal: logger->cout("normal"); break; - case ctFilenames: + case Completions::Type::Filenames: logger->cout("filenames"); break; - case ctAttrs: + case Completions::Type::Attrs: logger->cout("attrs"); break; } - for (auto & s : *completions) + for (auto & s : args.completions->completions) logger->cout(s.completion + "\t" + trim(s.description)); } }); @@ -429,7 +427,7 @@ void mainWrapped(int argc, char * * argv) try { args.parseCmdline(argvToStrings(argc, argv)); } catch (UsageError &) { - if (!args.helpRequested && !completions) throw; + if (!args.helpRequested && !args.completions) throw; } if (args.helpRequested) { @@ -446,10 +444,7 @@ void mainWrapped(int argc, char * * argv) return; } - if (completions) { - args.completionHook(); - return; - } + if (args.completions) return; if (args.showVersion) { printVersion(programName); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index cb94bbd31..f509ccae8 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -175,8 +175,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand .label = "locked", .optional = true, .handler = {&locked}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 592de773c..055cf6d0d 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -38,17 +38,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions expectArgs({ .label = "package", .handler = {&_package}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); expectArgs({ .label = "dependency", .handler = {&_dependency}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); addFlag({ From fa9642ec459859557ce29e5f68413cf3b0846fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 7 Jun 2023 18:12:35 +0200 Subject: [PATCH 077/100] nix-shell: support single quotes in shebangs Single quotes are a basic feature of shell syntax that people expect to work. They are also more convenient for writing literal code expressions with less escaping. --- doc/manual/src/command-ref/nix-shell.md | 6 +++--- doc/manual/src/release-notes/rl-next.md | 2 ++ src/nix-build/nix-build.cc | 26 ++++++++++++++++++------- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 195b72be5..1eaf3c36a 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -235,14 +235,14 @@ package like Terraform: ```bash #! /usr/bin/env nix-shell -#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix-shell -i bash --packages 'terraform.withPlugins (plugins: [ plugins.openstack ])' terraform apply ``` > **Note** > -> You must use double quotes (`"`) when passing a simple Nix expression +> You must use single or double quotes (`'`, `"`) when passing a simple Nix expression > in a nix-shell shebang. Finally, using the merging of multiple nix-shell shebangs the following @@ -251,7 +251,7 @@ branch): ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -i runghc --packages 'haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])' #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index d0b516dfb..276252c37 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -12,4 +12,6 @@ - Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). +- `nix-shell` shebang lines now support single-quoted arguments. + - `builtins.fetchTree` is now marked as stable. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 2895c5e3c..ae05444b7 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -40,7 +40,8 @@ static std::vector shellwords(const std::string & s) std::string cur; enum state { sBegin, - sQuote + sSingleQuote, + sDoubleQuote }; state st = sBegin; auto it = begin; @@ -56,15 +57,26 @@ static std::vector shellwords(const std::string & s) } } switch (*it) { + case '\'': + if (st != sDoubleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sSingleQuote : sBegin; + } + break; case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; + if (st != sSingleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sDoubleQuote : sBegin; + } break; case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; + if (st != sSingleQuote) { + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + } break; } } From 595010b631b0f92ee6eb4afb8494bc9c73e8513b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Thu, 8 Jun 2023 03:11:04 +0200 Subject: [PATCH 078/100] nix-shell: fix shebang whitespace parsing Leading whitespace after `nix-shell` used to produce an empty argument, while an empty argument at the end of the line was ignored. Fix the first issue by consuming the initial whitespace before calling shellwords; fix the second issue by returning immediately if whitespace is found at the end of the string instead of checking for an empty string. Also throw if quotes aren't terminated. --- src/nix-build/nix-build.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index ae05444b7..e62c4f6b1 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -34,7 +34,7 @@ extern char * * environ __attribute__((weak)); */ static std::vector shellwords(const std::string & s) { - std::regex whitespace("^(\\s+).*"); + std::regex whitespace("^\\s+"); auto begin = s.cbegin(); std::vector res; std::string cur; @@ -51,9 +51,10 @@ static std::vector shellwords(const std::string & s) if (regex_search(it, s.cend(), match, whitespace)) { cur.append(begin, it); res.push_back(cur); - cur.clear(); - it = match[1].second; + it = match[0].second; + if (it == s.cend()) return res; begin = it; + cur.clear(); } } switch (*it) { @@ -80,8 +81,9 @@ static std::vector shellwords(const std::string & s) break; } } + if (st != sBegin) throw Error("unterminated quote in shebang line"); cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); + res.push_back(cur); return res; } @@ -140,7 +142,7 @@ static void main_nix_build(int argc, char * * argv) for (auto line : lines) { line = chomp(line); std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) + if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$"))) for (const auto & word : shellwords(match[1].str())) args.push_back(word); } From c82066cf73c1b6d8f910ea68767035d0ead7c7d9 Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 16:56:30 +0300 Subject: [PATCH 079/100] fix: Declare constructor as default --- src/libutil/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index a5d7cbe4a..ebd6fb67b 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -57,7 +57,7 @@ protected: std::function)> fun; size_t arity; - Handler() {} + Handler() = default; Handler(std::function)> && fun) : fun(std::move(fun)) From b205da16ef94d02e08fbe751237e333d8f53aca8 Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 18:06:15 +0300 Subject: [PATCH 080/100] fix: Explicitly pass lambda scope variables. Default capture implicitly also capture *this, which would automatically be used if for example you referenced a method from the enclosing scope. --- src/libutil/args.hh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ebd6fb67b..021c2a937 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -84,29 +84,29 @@ protected: { } Handler(std::vector * dest) - : fun([=](std::vector ss) { *dest = ss; }) + : fun([dest](std::vector ss) { *dest = ss; }) , arity(ArityAny) { } Handler(std::string * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } Handler(std::optional * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } template Handler(T * dest, const T & val) - : fun([=](std::vector ss) { *dest = val; }) + : fun([dest, val](std::vector ss) { *dest = val; }) , arity(0) { } template Handler(I * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) @@ -114,7 +114,7 @@ protected: template Handler(std::optional * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) From a31fc5cc8643beae13b8f071fbeac50aac1a94e2 Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 18:07:17 +0300 Subject: [PATCH 081/100] fix: Use `using` instead of `typedef` for type aliasing. Since C++ 11 we shouldn't use c-style `typedefs`. In addition, `using` can be templated. --- src/libutil/args.hh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 021c2a937..cee672d80 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -130,7 +130,7 @@ protected: * The `AddCompletions` that is passed is an interface to the state * stored as part of the root command */ - typedef void CompleterFun(AddCompletions &, size_t, std::string_view); + using CompleterFun = void(AddCompletions &, size_t, std::string_view); /** * The closure type of the completion callback. @@ -138,7 +138,7 @@ protected: * This is what is actually stored as part of each Flag / Expected * Arg. */ - typedef std::function CompleterClosure; + using CompleterClosure = std::function; /** * Description of flags / options @@ -148,7 +148,7 @@ protected: */ struct Flag { - typedef std::shared_ptr ptr; + using ptr = std::shared_ptr; std::string longName; std::set aliases; @@ -303,7 +303,7 @@ struct Command : virtual public Args */ virtual void run() = 0; - typedef int Category; + using Category = int; static constexpr Category catDefault = 0; @@ -312,7 +312,7 @@ struct Command : virtual public Args virtual Category category() { return catDefault; } }; -typedef std::map()>> Commands; +using Commands = std::map()>>; /** * An argument parser that supports multiple subcommands, From 90e3ed06f84d0f184de9ee60b8219ba40607387e Mon Sep 17 00:00:00 2001 From: Kirill Trofimov Date: Mon, 23 Oct 2023 18:07:57 +0300 Subject: [PATCH 082/100] fix: Use default destructor. --- src/libutil/args.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index cee672d80..ff2bf3cab 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -296,7 +296,7 @@ struct Command : virtual public Args { friend class MultiCommand; - virtual ~Command() { } + virtual ~Command() = default; /** * Entry point to the command From e053eeb272d2a9115afe7f42077002a3de6b8633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sat, 17 Jun 2023 14:54:34 +0200 Subject: [PATCH 083/100] tests: test nix-shell shebang quoting --- tests/functional/nix-shell.sh | 5 +++++ tests/functional/shell.shebang.nix | 10 ++++++++++ 2 files changed, 15 insertions(+) create mode 100755 tests/functional/shell.shebang.nix diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index edaa1249b..13403fadb 100644 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -84,6 +84,11 @@ chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] +# Test nix-shell shebang quoting +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > $TEST_ROOT/shell.shebang.nix +chmod a+rx $TEST_ROOT/shell.shebang.nix +$TEST_ROOT/shell.shebang.nix + # Test 'nix develop'. nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]' diff --git a/tests/functional/shell.shebang.nix b/tests/functional/shell.shebang.nix new file mode 100755 index 000000000..08e43d53c --- /dev/null +++ b/tests/functional/shell.shebang.nix @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --argstr s1 'foo "bar" \baz'"'"'qux' --argstr s2 "foo 'bar' \"\baz" --argstr s3 \foo\ bar\'baz --argstr s4 '' +#! nix-shell shell.shebang.nix --command true +{ s1, s2, s3, s4 }: +assert s1 == ''foo "bar" \baz'qux''; +assert s2 == "foo 'bar' \"baz"; +assert s3 == "foo bar'baz"; +assert s4 == ""; +(import {}).runCommand "nix-shell" {} "" From 765436e300b855922d5793092cc2a533c81d521d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Oct 2023 11:15:52 -0400 Subject: [PATCH 084/100] Add `builtins.addDrvOutputDependencies` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End goal: make `(mkDerivation x).drvPath` behave like a non-DrvDeep context. Problem: users won't be able to recover the DrvDeep behavior when nixpkgs makes this change. Solution: add this primop. The new primop is fairly simple, and is supposed to complement other existing ones (`builtins.storePath`, `builtins.outputOf`) so there are simple ways to construct strings with every type of string context element. (It allows nothing we couldn't already do with `builtins.getContext` and `builtins.appendContext`, which is also true of those other two primops.) This was originally in #8595, but then it was proposed to land some doc changes separately. So now the code changes proper is just moved to this, and the doc will be done in that. Co-authored-by: Robert Hensing Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.nore github.com> Co-authored-by: Valentin Gagarin **Example** + > + > Many operations require a string context to be empty because they are intended only to work with "regular" strings, and also to help users avoid unintentionally loosing track of string context elements. + > `builtins.hasContext` can help create better domain-specific errors in those case. + > + > ```nix + > name: meta: + > + > if builtins.hasContext name + > then throw "package name cannot contain string context" + > else { ${name} = meta; } + > ``` )", .fun = prim_hasContext }); -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; @@ -66,11 +73,83 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p static RegisterPrimOp primop_unsafeDiscardOutputDependency({ .name = "__unsafeDiscardOutputDependency", - .arity = 1, + .args = {"s"}, + .doc = R"( + Create a copy of the given string where every "derivation deep" string context element is turned into a constant string context element. + + This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). + + This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context, + whereas Nix normally tracks all dependencies consistently. + Safe operations "grow" but never "shrink" string contexts. + [`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things). + Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything. + + [`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies + )", .fun = prim_unsafeDiscardOutputDependency }); +static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies"); + + auto contextSize = context.size(); + if (contextSize != 1) { + throw EvalError({ + .msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize), + .errPos = state.positions[pos] + }); + } + NixStringContext context2 { + (NixStringContextElem { std::visit(overloaded { + [&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep { + if (!c.path.isDerivation()) { + throw EvalError({ + .msg = hintfmt("path '%s' is not a derivation", + state.store->printStorePath(c.path)), + .errPos = state.positions[pos], + }); + } + return NixStringContextElem::DrvDeep { + .drvPath = c.path, + }; + }, + [&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep { + throw EvalError({ + .msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output), + .errPos = state.positions[pos], + }); + }, + [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { + /* Reuse original item because we want this to be idempotent. */ + return std::move(c); + }, + }, context.begin()->raw) }), + }; + + v.mkString(*s, context2); +} + +static RegisterPrimOp primop_addDrvOutputDependencies({ + .name = "__addDrvOutputDependencies", + .args = {"s"}, + .doc = R"( + Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element. + + The store path that is the constant string context element should point to a valid derivation, and end in `.drv`. + + The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element. + The latter is supported so this function is idempotent. + + This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies). + )", + .fun = prim_addDrvOutputDependencies +}); + + /* Extract the context of a string as a structured Nix value. The context is represented as an attribute set whose keys are the diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp new file mode 100644 index 000000000..ad91a22aa --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-empty-context.nix:1:1: + + 1| builtins.addDrvOutputDependencies "" + | ^ + 2| + + error: context of string '' must have exactly one element, but has 0 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix new file mode 100644 index 000000000..dc9ee3ba2 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix @@ -0,0 +1 @@ +builtins.addDrvOutputDependencies "" diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp new file mode 100644 index 000000000..bb389db4e --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix:18:4: + + 17| + 18| in builtins.addDrvOutputDependencies combo-path + | ^ + 19| + + error: context of string '/nix/store/pg9yqs4yd85yhdm3f4i5dyaqp5jahrsz-fail.drv/nix/store/2dxd5frb715z451vbf7s8birlf3argbk-fail-2.drv' must have exactly one element, but has 2 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix new file mode 100644 index 000000000..dbde264df --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix @@ -0,0 +1,18 @@ +let + drv0 = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + drv1 = derivation { + name = "fail-2"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + combo-path = "${drv0.drvPath}${drv1.drvPath}"; + +in builtins.addDrvOutputDependencies combo-path diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp new file mode 100644 index 000000000..070381118 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix:9:4: + + 8| + 9| in builtins.addDrvOutputDependencies drv.outPath + | ^ + 10| + + error: `addDrvOutputDependencies` can only act on derivations, not on a derivation output such as 'out' diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix new file mode 100644 index 000000000..e379e1d95 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix @@ -0,0 +1,9 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + +in builtins.addDrvOutputDependencies drv.outPath diff --git a/tests/functional/lang/eval-okay-context-introspection.exp b/tests/functional/lang/eval-okay-context-introspection.exp index 03b400cc8..a136b0035 100644 --- a/tests/functional/lang/eval-okay-context-introspection.exp +++ b/tests/functional/lang/eval-okay-context-introspection.exp @@ -1 +1 @@ -[ true true true true true true ] +[ true true true true true true true true true true true true true ] diff --git a/tests/functional/lang/eval-okay-context-introspection.nix b/tests/functional/lang/eval-okay-context-introspection.nix index 50a78d946..8886cf32e 100644 --- a/tests/functional/lang/eval-okay-context-introspection.nix +++ b/tests/functional/lang/eval-okay-context-introspection.nix @@ -31,11 +31,29 @@ let (builtins.unsafeDiscardStringContext str) (builtins.getContext str); + # Only holds true if string context contains both a `DrvDeep` and + # `Opaque` element. + almostEtaRule = str: + str == builtins.addDrvOutputDependencies + (builtins.unsafeDiscardOutputDependency str); + + addDrvOutputDependencies_idempotent = str: + builtins.addDrvOutputDependencies str == + builtins.addDrvOutputDependencies (builtins.addDrvOutputDependencies str); + + rules = str: [ + (etaRule str) + (almostEtaRule str) + (addDrvOutputDependencies_idempotent str) + ]; + in [ (legit-context == desired-context) (reconstructed-path == combo-path) (etaRule "foo") - (etaRule drv.drvPath) (etaRule drv.foo.outPath) - (etaRule (builtins.unsafeDiscardOutputDependency drv.drvPath)) +] ++ builtins.concatMap rules [ + drv.drvPath + (builtins.addDrvOutputDependencies drv.drvPath) + (builtins.unsafeDiscardOutputDependency drv.drvPath) ] From c9528d20810c577fd394a8668bec645799d5eded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=A2=D1=80=D0=BE?= =?UTF-8?q?=D1=84=D0=B8=D0=BC=D0=BE=D0=B2?= <62882157+trofkm@users.noreply.github.com> Date: Mon, 23 Oct 2023 20:20:23 +0300 Subject: [PATCH 085/100] fix: Remove extra to from README.md (#9213) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 623a9722c..e1cace3b4 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Full reference documentation can be found in the [Nix manual](https://nixos.org/ ## Building And Developing See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to -to set up a development environment and build Nix from source. + set up a development environment and build Nix from source. ## Contributing From cd680bd53d90971211fa8926940bfe6d987ca6cf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 23 Oct 2023 19:22:33 +0200 Subject: [PATCH 086/100] Merge how-to section on S3 buckets into S3 store docs (#7972) Rather than having a misc tutorial page in the grab-bag "package management" section, this information should just be part of the S3 store docs. --------- Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 1 - .../src/advanced-topics/post-build-hook.md | 5 +- .../src/package-management/s3-substituter.md | 115 ------------------ src/libstore/s3-binary-cache-store.md | 100 ++++++++++++++- 4 files changed, 100 insertions(+), 121 deletions(-) delete mode 100644 doc/manual/src/package-management/s3-substituter.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 60ebeb138..f3233f8c9 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,7 +24,6 @@ - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - - [Serving a Nix store via S3](package-management/s3-substituter.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index e4475bd9b..3c1cc9b36 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -17,9 +17,8 @@ the build loop. # Prerequisites -This tutorial assumes you have [configured an S3-compatible binary -cache](../package-management/s3-substituter.md), and that the `root` -user's default AWS profile can upload to the bucket. +This tutorial assumes you have configured an [S3-compatible binary cache](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) as a [substituter](../command-ref/conf-file.md#conf-substituters), +and that the `root` user's default AWS profile can upload to the bucket. # Set up a Signing Key diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md deleted file mode 100644 index d8a1d9105..000000000 --- a/doc/manual/src/package-management/s3-substituter.md +++ /dev/null @@ -1,115 +0,0 @@ -# Serving a Nix store via S3 - -Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) -for storing and fetching store paths from -Amazon S3 and S3-compatible services. This uses the same *binary* -cache mechanism that Nix usually uses to fetch prebuilt binaries from -[cache.nixos.org](https://cache.nixos.org/). - -In this example we will use the bucket named `example-nix-cache`. - -## Anonymous Reads to your S3-compatible binary cache - -If your binary cache is publicly accessible and does not require -authentication, the simplest and easiest way to use Nix with your S3 -compatible binary cache is to use the HTTP URL for that cache. - -For AWS S3 the binary cache URL for example bucket will be exactly - or -. For S3 compatible binary caches, consult that -cache's documentation. - -Your bucket will need the following bucket policy: - -```json -{ - "Id": "DirectReads", - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowDirectReads", - "Action": [ - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ], - "Principal": "*" - } - ] -} -``` - -## Authenticated Reads to your S3 binary cache - -For AWS S3 the binary cache URL for example bucket will be exactly -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Nix supports authenticated reads from Amazon S3 and S3 compatible binary -caches. - -Your bucket will need a bucket policy allowing the desired users to -perform the `s3:GetObject` and `s3:GetBucketLocation` action on all -objects in the bucket. The [anonymous policy given -above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be -updated to have a restricted `Principal` to support this. - -## Authenticated Writes to your S3-compatible binary cache - -Nix support fully supports writing to Amazon S3 and S3 compatible -buckets. The binary cache URL for our example bucket will be -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Your account will need the following IAM policy to upload to the cache: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "UploadToCache", - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListMultipartUploadParts", - "s3:PutObject" - ], - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ] - } - ] -} -``` - -## Examples - -To upload with a specific credential profile for Amazon S3: - -```console -$ nix copy nixpkgs.hello \ - --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' -``` - -To upload to an S3-compatible binary cache: - -```console -$ nix copy nixpkgs.hello --to \ - 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' -``` diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md index 70fe0eb09..675470261 100644 --- a/src/libstore/s3-binary-cache-store.md +++ b/src/libstore/s3-binary-cache-store.md @@ -2,7 +2,103 @@ R"( **Store URL format**: `s3://`*bucket-name* -This store allows reading and writing a binary cache stored in an AWS -S3 bucket. +This store allows reading and writing a binary cache stored in an AWS S3 (or S3-compatible service) bucket. +This store shares many idioms with the [HTTP Binary Cache Store](#http-binary-cache-store). + +For AWS S3, the binary cache URL for a bucket named `example-nix-cache` will be exactly . +For S3 compatible binary caches, consult that cache's documentation. + +### Anonymous reads to your S3-compatible binary cache + +> If your binary cache is publicly accessible and does not require authentication, +> it is simplest to use the [HTTP Binary Cache Store] rather than S3 Binary Cache Store with +> instead of . + +Your bucket will need a +[bucket policy](https://docs.aws.amazon.com/AmazonS3/v1/userguide/bucket-policies.html) +like the following to be accessible: + +```json +{ + "Id": "DirectReads", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowDirectReads", + "Action": [ + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ], + "Principal": "*" + } + ] +} +``` + +### Authentication + +Nix will use the +[default credential provider chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) +for authenticating requests to Amazon S3. + +Note that this means Nix will read environment variables and files with different idioms than with Nix's own settings, as implemented by the AWS SDK. +Consult the documentation linked above for further details. + +### Authenticated reads to your S3 binary cache + +Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket. +The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this. + +### Authenticated writes to your S3-compatible binary cache + +Your account will need an IAM policy to support uploading to the bucket: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToCache", + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ] + } + ] +} +``` + +### Examples + +With bucket policies and authentication set up as described above, uploading works via [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md) (experimental). + +- To upload with a specific credential profile for Amazon S3: + + ```console + $ nix copy nixpkgs.hello \ + --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' + ``` + +- To upload to an S3-compatible binary cache: + + ```console + $ nix copy nixpkgs.hello --to \ + 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' + ``` )" From cde3c63617137a795f9d3e6fa38714d1a2c54bfa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Oct 2023 19:30:00 +0200 Subject: [PATCH 087/100] system-features: Typo There I was, thinking all of Apple's OSes started with lower case. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index e90f70f5f..12fb48d93 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -716,7 +716,7 @@ public: - `apple-virt` - Included on darwin if virtualization is available. + Included on Darwin if virtualization is available. - `kvm` From abb1c829c8ced0680ffa7ef9e45c1844fb24bcb5 Mon Sep 17 00:00:00 2001 From: Vignesh Date: Tue, 24 Oct 2023 14:48:00 +0530 Subject: [PATCH 088/100] Release notes updated for #9150 reverted (#9227) --- doc/manual/src/release-notes/rl-2.15.md | 2 +- doc/manual/src/release-notes/rl-2.7.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 4faf0b143..133121999 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -44,7 +44,7 @@ (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. - `nix store info` and `nix doctor` now display this information. + `nix store ping` and `nix doctor` now display this information. * The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/src/release-notes/rl-2.7.md index dd649e166..2f3879422 100644 --- a/doc/manual/src/release-notes/rl-2.7.md +++ b/doc/manual/src/release-notes/rl-2.7.md @@ -24,7 +24,7 @@ [repository](https://github.com/NixOS/bundlers) has various bundlers implemented. -* `nix store info` now reports the version of the remote Nix daemon. +* `nix store ping` now reports the version of the remote Nix daemon. * `nix flake {init,new}` now display information about which files have been created. From f269911641fcc6cef836a07393feae9d067096a8 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 24 Oct 2023 11:22:02 +0200 Subject: [PATCH 089/100] Document builtins.substring negative length behavior (#9226) --- src/libexpr/primops.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5033d4e2d..704e7007b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3720,10 +3720,11 @@ static RegisterPrimOp primop_substring({ .doc = R"( Return the substring of *s* from character position *start* (zero-based) up to but not including *start + len*. If *start* is - greater than the length of the string, an empty string is returned, - and if *start + len* lies beyond the end of the string, only the - substring up to the end of the string is returned. *start* must be - non-negative. For example, + greater than the length of the string, an empty string is returned. + If *start + len* lies beyond the end of the string or *len* is `-1`, + only the substring up to the end of the string is returned. + *start* must be non-negative. + For example, ```nix builtins.substring 0 3 "nixos" From eaced12c94e13f0fedd8324f4f9bc8684ea351ae Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 19:30:01 +0200 Subject: [PATCH 090/100] Fix signed vs. unsigned comparison warning and improve code --- src/libstore/builtins/buildenv.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 7bba33fb9..c8911d153 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -174,15 +174,19 @@ void builtinBuildenv(const BasicDerivation & drv) /* Convert the stuff we get from the environment back into a * coherent data type. */ Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); + { + auto derivations = tokenizeString(getAttr("derivations")); + + auto itemIt = derivations.begin(); + while (itemIt != derivations.end()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + const bool active = "false" != *itemIt++; + const int priority = stoi(*itemIt++); + const size_t outputs = stoul(*itemIt++); + + for (size_t n {0}; n < outputs; n++) { + pkgs.emplace_back(std::move(*itemIt++), active, priority); + } } } From b113d925de50c5f40c4fd694167adde3eeb2b060 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 19:30:23 +0200 Subject: [PATCH 091/100] Fix warning --- src/libstore/content-address.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 52c60154c..a5f7cdf81 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -29,12 +29,13 @@ std::string ContentAddressMethod::renderPrefix() const ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { - ContentAddressMethod method = FileIngestionMethod::Flat; - if (splitPrefix(m, "r:")) - method = FileIngestionMethod::Recursive; - else if (splitPrefix(m, "text:")) - method = TextIngestionMethod {}; - return method; + if (splitPrefix(m, "r:")) { + return FileIngestionMethod::Recursive; + } + else if (splitPrefix(m, "text:")) { + return TextIngestionMethod {}; + } + return FileIngestionMethod::Flat; } std::string ContentAddressMethod::render(HashType ht) const From 8d9e0b7aed2e3bd218c5344d1dcf4a8632e0d63a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 25 Oct 2023 04:28:35 +0200 Subject: [PATCH 092/100] document the store concept (#9206) * document the store concept and its purpose reword the glossary to link to more existing information instead of repeating it. move the store documentation to the top of the table of contents, in front of the Nix language. this will provide a natural place to document other aspects of the store as well as the various store types. move the package management section after the Nix language and before Advanced Topics to follow the pattern to layer more complex concepts on top of each other. this structure of the manual will also nudge beginners to learn Nix bottom-up and hopefully make more likely that they understand underlying concepts first before delving into complex use cases that may or may not be easy to implement with what's currently there. [John adds this note] The sort of beginner who likes to dive straight into reference documentation should prefer this approach. Conversely, the sort of beginner who would prefer the opposite top-down approach of trying to solve problems before they understand everything that is going on is better off reading other tutorial/guide material anyways, and will just "random-access" the reference manual as a last resort. For such random-access the order doesn't matter, so this restructure doesn't make them any worse off. Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 20 +++++++------ doc/manual/src/architecture/architecture.md | 1 + doc/manual/src/glossary.md | 30 +++++++++---------- .../file-system-object.md | 0 doc/manual/src/store/index.md | 4 +++ 5 files changed, 30 insertions(+), 25 deletions(-) rename doc/manual/src/{architecture => store}/file-system-object.md (100%) create mode 100644 doc/manual/src/store/index.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f3233f8c9..2fe77d2c6 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,14 +16,8 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) -- [Package Management](package-management/package-management.md) - - [Profiles](package-management/profiles.md) - - [Garbage Collection](package-management/garbage-collection.md) - - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - - [Sharing Packages Between Machines](package-management/sharing-packages.md) - - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - - [Copying Closures via SSH](package-management/copy-closure.md) - - [Serving a Nix store via SSH](package-management/ssh-substituter.md) +- [Nix Store](store/index.md) + - [File System Object](store/file-system-object.md) - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) @@ -35,7 +29,16 @@ - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) +- [Package Management](package-management/package-management.md) + - [Profiles](package-management/profiles.md) + - [Garbage Collection](package-management/garbage-collection.md) + - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - [Advanced Topics](advanced-topics/advanced-topics.md) + - [Sharing Packages Between Machines](package-management/sharing-packages.md) + - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) + - [Copying Closures via SSH](package-management/copy-closure.md) + - [Serving a Nix store via SSH](package-management/ssh-substituter.md) + - [Serving a Nix store via S3](package-management/s3-substituter.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) @@ -97,7 +100,6 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) - - [File System Object](architecture/file-system-object.md) - [Protocols](protocols/protocols.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 9e969972e..6e832e1f9 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -59,6 +59,7 @@ The [Nix language](../language/index.md) evaluator transforms Nix expressions in The command line interface and Nix expressions are what users deal with most. > **Note** +> > The Nix language itself does not have a notion of *packages* or *configurations*. > As far as we are concerned here, the inputs and results of a build plan are just data. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index d49d5e52e..ad3cc147b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -58,22 +58,16 @@ - [store]{#gloss-store} - The location in the file system where store objects live. Typically - `/nix/store`. + A collection of store objects, with operations to manipulate that collection. + See [Nix Store] for details. - From the perspective of the location where Nix is - invoked, the Nix store can be referred to - as a "_local_" or a "_remote_" one: + There are many types of stores. + See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. - + A [local store]{#gloss-local-store} exists on the filesystem of - the machine where Nix is invoked. You can use other - local stores by passing the `--store` flag to the - `nix` command. Local stores can be used for building derivations. - - + A *remote store* exists anywhere other than the - local filesystem. One example is the `/nix/store` - directory on another machine, accessed via `ssh` or - served by the `nix-serve` Perl script. + From the perspective of the location where Nix is invoked, the Nix store can be referred to _local_ or _remote_. + Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`. + Local stores can be used for building [derivations](#derivation). + See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details. [store]: #gloss-store [local store]: #gloss-local-store @@ -103,15 +97,19 @@ The Nix data model for representing simplified file system data. - See [File System Object](@docroot@/architecture/file-system-object.md) for details. + See [File System Object](@docroot@/store/file-system-object.md) for details. [file system object]: #gloss-file-system-object - [store object]{#gloss-store-object} - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. + Part of the contents of a [store]. + + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. It can be referred to by a [store path]. + See [Store Object](@docroot@/store/index.md#store-object) for details. + [store object]: #gloss-store-object - [IFD]{#gloss-ifd} diff --git a/doc/manual/src/architecture/file-system-object.md b/doc/manual/src/store/file-system-object.md similarity index 100% rename from doc/manual/src/architecture/file-system-object.md rename to doc/manual/src/store/file-system-object.md diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md new file mode 100644 index 000000000..316e04179 --- /dev/null +++ b/doc/manual/src/store/index.md @@ -0,0 +1,4 @@ +# Nix Store + +The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them. +There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches. From 78278f2b3f1c1c55aec7c147d2e266b805863985 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 25 Oct 2023 12:00:56 +0200 Subject: [PATCH 093/100] add notes on comments in code samples --- doc/manual/src/contributing/documentation.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index f73ab2149..190d367db 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -73,6 +73,17 @@ It should therefore aim to be correct, consistent, complete, and easy to navigat Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. +- Always explain code examples in the text. + + Use comments in code samples very sparingly, for instance to highlight a particular aspect. + Readers tend to glance over large amounts of code when scanning for information. + + Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether. + + If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one. + If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end. + This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject. + - Use British English. This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. From 7bc45c6136af4bb922285ee786f58e54753a2b0d Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Wed, 25 Oct 2023 14:49:30 +0200 Subject: [PATCH 094/100] docs: clarify flake types and implied defaults --- src/nix/flake.md | 77 +++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index f08648417..d8b5bf435 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -98,7 +98,15 @@ Here are some examples of flake references in their URL-like representation: ## Path-like syntax -Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity). +Flakes corresponding to a local path can also be referred to by a direct +path reference, either `/absolute/path/to/the/flake` or`./relative/path/to/the/flake`. +Note that the leading `./` is mandatory for relative paths. If it is +omitted, the path will be interpreted as [URL-like syntax](#url-like-syntax), +which will cause error messages like this: + +```console +error: cannot find flake 'flake:relative/path/to/the/flake' in the flake registries +``` The semantic of such a path is as follows: @@ -153,18 +161,39 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `path`: arbitrary local directories, or local Git trees. The - required attribute `path` specifies the path of the flake. The URL - form is +* `indirect`: *The default*. Indirection through the flake registry. + These have the form ``` - [path:](\?(/(/rev)?)? ``` - where *path* is an absolute path. + These perform a lookup of `` in the flake registry. For + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. - *path* must be a directory in the file system containing a file - named `flake.nix`. + For example, these are valid indirect flake references: + + * `nixpkgs` + * `nixpkgs/nixos-unstable` + * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `nixpkgs/nixos-unstable/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `sub/dir` (if a flake named `sub` is in the registry) + +* `path`: arbitrary local directories. The required attribute `path` + specifies the path of the flake. The URL form is + + ``` + path:(\?)? + ``` + + where *path* is an absolute path to a directory in the file system + containing a file named `flake.nix`. + + If the flake at *path* is not inside a git repository, the `path:` + prefix is implied and can be omitted. *path* generally must be an absolute path. However, on the command line, it can be a relative path (e.g. `.` or `./foo`) which is @@ -173,20 +202,24 @@ Currently the `type` attribute can be one of the following: (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative path). + For example, these are valid path flake references: + + * `path:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is *not* in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is *not* in a git repository) + * `git`: Git repositories. The location of the repository is specified by the attribute `url`. They have the URL form ``` - git(+http|+https|+ssh|+git|+file|):(//)?(\?)? + git(+http|+https|+ssh|+git|+file):(//)?(\?)? ``` - or - - ``` - @: - ``` + If *path* starts with `/` (or `./` when used as an argument on the + command line) and is a local path to a git repository, the leading + `git:` or `+file` prefixes are implied and can be omitted. The `ref` attribute defaults to resolving the `HEAD` reference. @@ -203,6 +236,9 @@ Currently the `type` attribute can be one of the following: For example, the following are valid Git flake references: + * `git:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository) * `git+https://example.org/my/repo` * `git+https://example.org/my/repo?dir=flake1` * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` @@ -324,19 +360,6 @@ Currently the `type` attribute can be one of the following: * `sourcehut:~misterio/nix-colors/182b4b8709b8ffe4e9774a4c5d6877bf6bb9a21c` * `sourcehut:~misterio/nix-colors/21c1a380a6915d890d408e9f22203436a35bb2de?host=hg.sr.ht` -* `indirect`: Indirections through the flake registry. These have the - form - - ``` - [flake:](/(/rev)?)? - ``` - - These perform a lookup of `` in the flake registry. For - example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake - references. The specified `rev` and/or `ref` are merged with the - entry in the registry; see [nix registry](./nix3-registry.md) for - details. - # Flake format As an example, here is a simple `flake.nix` that depends on the From f555c98a343fbed98b0c01b1b19e6796feaea936 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 19:30:37 +0200 Subject: [PATCH 095/100] Improve loop over gid container --- src/libstore/lock.cc | 60 +++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 7202a64b3..165e4969f 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -7,6 +7,31 @@ namespace nix { +#if __linux__ + +static std::vector get_group_list(const char *username, gid_t group_id) +{ + std::vector gids; + gids.resize(32); // Initial guess + + auto getgroupl_failed {[&] { + int ngroups = gids.size(); + int err = getgrouplist(username, group_id, gids.data(), &ngroups); + gids.resize(ngroups); + return err == -1; + }}; + + // The first error means that the vector was not big enough. + // If it happens again, there is some different problem. + if (getgroupl_failed() && getgroupl_failed()) { + throw SysError("failed to get list of supplementary groups for '%s'", username); + } + + return gids; +} +#endif + + struct SimpleUserLock : UserLock { AutoCloseFD fdUserLock; @@ -67,37 +92,14 @@ struct SimpleUserLock : UserLock throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build - user. This is usually either empty or contains a - group such as "kvm". */ - int ngroups = 32; // arbitrary initial guess - std::vector gids; - gids.resize(ngroups); - - int err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - - /* Our initial size of 32 wasn't sufficient, the - correct size has been stored in ngroups, so we try - again. */ - if (err == -1) { - gids.resize(ngroups); - err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - } - - // If it failed once more, then something must be broken. - if (err == -1) - throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); + /* Get the list of supplementary groups of this user. This is + * usually either empty or contains a group such as "kvm". */ // Finally, trim back the GID list to its real size. - for (auto i = 0; i < ngroups; i++) - if (gids[i] != lock->gid) - lock->supplementaryGIDs.push_back(gids[i]); + for (auto gid : get_group_list(pw->pw_name, pw->pw_gid)) { + if (gid != lock->gid) + lock->supplementaryGIDs.push_back(gid); + } #endif return lock; From 46028ff76493439921a5a9200d14c015f0a0c025 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 26 Oct 2023 07:05:48 +0200 Subject: [PATCH 096/100] doc: Fix fetchGit default name (#9241) --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a99b0e500..767f559be 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -392,7 +392,7 @@ static RegisterPrimOp primop_fetchGit({ The URL of the repo. - - `name` (default: *basename of the URL*) + - `name` (default: `source`) The name of the directory the repo should be exported to in the store. From b66381e8d8728c040197ec78ed47d5eff88e1d0e Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 20:16:34 +0200 Subject: [PATCH 097/100] Use using instead of typedef --- src/libstore/path-info.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a82e643ae..580e94189 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,7 +29,7 @@ struct SubstitutablePathInfo uint64_t narSize; }; -typedef std::map SubstitutablePathInfos; +using SubstitutablePathInfos = std::map; struct UnkeyedValidPathInfo @@ -136,6 +136,6 @@ struct ValidPathInfo : UnkeyedValidPathInfo { virtual ~ValidPathInfo() { } }; -typedef std::map ValidPathInfos; +using ValidPathInfos = std::map; } From 28c39c370c99148ccc6576d429b41a1f46b08174 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 24 Oct 2023 20:16:46 +0200 Subject: [PATCH 098/100] Provide default value for id to fix warning --- src/libstore/path-info.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 580e94189..c4c4a6366 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -42,7 +42,7 @@ struct UnkeyedValidPathInfo StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only + uint64_t id = 0; // internal use only /** * Whether the path is ultimately trusted, that is, it's a From e69c764708a78ae2ea38b4c37e5c07119e7097d0 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Thu, 26 Oct 2023 20:53:03 +0100 Subject: [PATCH 099/100] local-derivation-goal.cc: slightly clarify waiting message Before the change builder ID exhaustion printed the following message: [0/1 built] waiting for UID to build '/nix/store/hiy9136x0iyib4ssh3w3r5m8pxjnad50-python3.11-breathe-4.35.0.drv' After the change it should be: [0/1 built] waiting for a free build user ID for '/nix/store/hiy9136x0iyib4ssh3w3r5m8pxjnad50-python3.11-breathe-4.35.0.drv' --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b5b060d95..738e7051e 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -227,7 +227,7 @@ void LocalDerivationGoal::tryLocalBuild() if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } From a419b6149705bddff60215a0d21ef355857ef2c5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 26 Oct 2023 17:58:45 -0400 Subject: [PATCH 100/100] Turn derivation unit tests into unit characterization tests The brings a number of advantages, including: - Easier to update test data if design changes (and I do think our derivation JSON is not yet complaint with the guidelines). - Easier to reuse test data in other implementations, inching closer to compliance tests for Nix *the concept* rather than any one implementation. --- src/libstore/tests/characterization.hh | 5 + src/libstore/tests/common-protocol.cc | 8 +- src/libstore/tests/derivation.cc | 273 ++++++++---------- src/libstore/tests/protocol.hh | 4 +- .../derivation/bad-old-version-dyn-deps.drv | 1 + .../libstore/derivation/bad-version.drv | 1 + .../libstore/derivation/dynDerivationDeps.drv | 1 + .../derivation/dynDerivationDeps.json | 38 +++ .../derivation/output-caFixedFlat.json | 5 + .../derivation/output-caFixedNAR.json | 5 + .../derivation/output-caFixedText.json | 5 + .../derivation/output-caFloating.json | 3 + .../libstore/derivation/output-deferred.json | 1 + .../libstore/derivation/output-impure.json | 4 + .../derivation/output-inputAddressed.json | 3 + unit-test-data/libstore/derivation/simple.drv | 1 + .../libstore/derivation/simple.json | 25 ++ 17 files changed, 228 insertions(+), 155 deletions(-) create mode 100644 unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv create mode 100644 unit-test-data/libstore/derivation/bad-version.drv create mode 100644 unit-test-data/libstore/derivation/dynDerivationDeps.drv create mode 100644 unit-test-data/libstore/derivation/dynDerivationDeps.json create mode 100644 unit-test-data/libstore/derivation/output-caFixedFlat.json create mode 100644 unit-test-data/libstore/derivation/output-caFixedNAR.json create mode 100644 unit-test-data/libstore/derivation/output-caFixedText.json create mode 100644 unit-test-data/libstore/derivation/output-caFloating.json create mode 100644 unit-test-data/libstore/derivation/output-deferred.json create mode 100644 unit-test-data/libstore/derivation/output-impure.json create mode 100644 unit-test-data/libstore/derivation/output-inputAddressed.json create mode 100644 unit-test-data/libstore/derivation/simple.drv create mode 100644 unit-test-data/libstore/derivation/simple.json diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh index 5f366cb42..46bf4b2e5 100644 --- a/src/libstore/tests/characterization.hh +++ b/src/libstore/tests/characterization.hh @@ -20,4 +20,9 @@ static bool testAccept() { return getEnv("_NIX_TEST_ACCEPT") == "1"; } +constexpr std::string_view cannotReadGoldenMaster = + "Cannot read golden master because another test is also updating it"; + +constexpr std::string_view updatingGoldenMaster = + "Updating golden master"; } diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc index 61c2cb70c..b3f4977d2 100644 --- a/src/libstore/tests/common-protocol.cc +++ b/src/libstore/tests/common-protocol.cc @@ -24,14 +24,14 @@ public: { if (testAccept()) { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + GTEST_SKIP() << cannotReadGoldenMaster; } else { - auto expected = readFile(goldenMaster(testStem)); + auto encoded = readFile(goldenMaster(testStem)); T got = ({ - StringSource from { expected }; + StringSource from { encoded }; CommonProto::Serialise::read( *store, CommonProto::ReadConn { .from = from }); @@ -59,7 +59,7 @@ public: { createDirs(dirOf(file)); writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; + GTEST_SKIP() << updatingGoldenMaster; } else { diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index c360c9707..ca0cdff71 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -5,9 +5,12 @@ #include "derivations.hh" #include "tests/libstore.hh" +#include "tests/characterization.hh" namespace nix { +using nlohmann::json; + class DerivationTest : public LibStoreTest { public: @@ -16,6 +19,12 @@ public: * to worry about race conditions if the tests run concurrently. */ ExperimentalFeatureSettings mockXpSettings; + + Path unitTestData = getUnitTestData() + "/libstore/derivation"; + + Path goldenMaster(std::string_view testStem) { + return unitTestData + "/" + testStem; + } }; class CaDerivationTest : public DerivationTest @@ -46,7 +55,7 @@ TEST_F(DerivationTest, BadATerm_version) { ASSERT_THROW( parseDerivation( *store, - R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + readFile(goldenMaster("bad-version.drv")), "whatever", mockXpSettings), FormatError); @@ -56,50 +65,61 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { ASSERT_THROW( parseDerivation( *store, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + readFile(goldenMaster("bad-old-version-dyn-deps.drv")), "dyn-dep-derivation", mockXpSettings), FormatError); } -#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (DerivationOutput { VAL }).toJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME)); \ - } \ - \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - DerivationOutput { VAL }, \ - DerivationOutput::fromJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME, \ - STR ## _json, \ - mockXpSettings)); \ +#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ + if (testAccept()) \ + { \ + GTEST_SKIP() << cannotReadGoldenMaster; \ + } \ + else \ + { \ + auto encoded = json::parse( \ + readFile(goldenMaster("output-" #NAME ".json"))); \ + DerivationOutput got = DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + encoded, \ + mockXpSettings); \ + DerivationOutput expected { VAL }; \ + ASSERT_EQ(got, expected); \ + } \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ + auto file = goldenMaster("output-" #NAME ".json"); \ + \ + json got = DerivationOutput { VAL }.toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME); \ + \ + if (testAccept()) \ + { \ + createDirs(dirOf(file)); \ + writeFile(file, got.dump(2) + "\n"); \ + GTEST_SKIP() << updatingGoldenMaster; \ + } \ + else \ + { \ + auto expected = json::parse(readFile(file)); \ + ASSERT_EQ(got, expected); \ + } \ } TEST_JSON(DerivationTest, inputAddressed, - R"({ - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", (DerivationOutput::InputAddressed { .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), }), "drv-name", "output-name") TEST_JSON(DerivationTest, caFixedFlat, - R"({ - "hashAlgo": "sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" - })", (DerivationOutput::CAFixed { .ca = { .method = FileIngestionMethod::Flat, @@ -109,11 +129,6 @@ TEST_JSON(DerivationTest, caFixedFlat, "drv-name", "output-name") TEST_JSON(DerivationTest, caFixedNAR, - R"({ - "hashAlgo": "r:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", (DerivationOutput::CAFixed { .ca = { .method = FileIngestionMethod::Recursive, @@ -123,11 +138,6 @@ TEST_JSON(DerivationTest, caFixedNAR, "drv-name", "output-name") TEST_JSON(DynDerivationTest, caFixedText, - R"({ - "hashAlgo": "text:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" - })", (DerivationOutput::CAFixed { .ca = { .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), @@ -136,9 +146,6 @@ TEST_JSON(DynDerivationTest, caFixedText, "drv-name", "output-name") TEST_JSON(CaDerivationTest, caFloating, - R"({ - "hashAlgo": "r:sha256" - })", (DerivationOutput::CAFloating { .method = FileIngestionMethod::Recursive, .hashType = htSHA256, @@ -146,15 +153,10 @@ TEST_JSON(CaDerivationTest, caFloating, "drv-name", "output-name") TEST_JSON(DerivationTest, deferred, - R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") TEST_JSON(ImpureDerivationTest, impure, - R"({ - "hashAlgo": "r:sha256", - "impure": true - })", (DerivationOutput::Impure { .method = FileIngestionMethod::Recursive, .hashType = htSHA256, @@ -163,43 +165,79 @@ TEST_JSON(ImpureDerivationTest, impure, #undef TEST_JSON -#define TEST_JSON(FIXTURE, NAME, STR, VAL) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (VAL).toJSON(*store)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - (VAL), \ - Derivation::fromJSON( \ - *store, \ - STR ## _json, \ - mockXpSettings)); \ +#define TEST_JSON(FIXTURE, NAME, VAL) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ + if (testAccept()) \ + { \ + GTEST_SKIP() << cannotReadGoldenMaster; \ + } \ + else \ + { \ + auto encoded = json::parse( \ + readFile(goldenMaster( #NAME ".json"))); \ + Derivation expected { VAL }; \ + Derivation got = Derivation::fromJSON( \ + *store, \ + encoded, \ + mockXpSettings); \ + ASSERT_EQ(got, expected); \ + } \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ + auto file = goldenMaster( #NAME ".json"); \ + \ + json got = Derivation { VAL }.toJSON(*store); \ + \ + if (testAccept()) \ + { \ + createDirs(dirOf(file)); \ + writeFile(file, got.dump(2) + "\n"); \ + GTEST_SKIP() << updatingGoldenMaster; \ + } \ + else \ + { \ + auto expected = json::parse(readFile(file)); \ + ASSERT_EQ(got, expected); \ + } \ } -#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ - ASSERT_EQ( \ - STR, \ - (VAL).unparse(*store, false)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ - auto parsed = parseDerivation( \ - *store, \ - STR, \ - DRV_NAME, \ - mockXpSettings); \ - ASSERT_EQ( \ - (VAL).toJSON(*store), \ - parsed.toJSON(*store)); \ - ASSERT_EQ( \ - (VAL), \ - parsed); \ +#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ + if (testAccept()) \ + { \ + GTEST_SKIP() << cannotReadGoldenMaster; \ + } \ + else \ + { \ + auto encoded = readFile(goldenMaster( #NAME ".drv")); \ + Derivation expected { VAL }; \ + auto got = parseDerivation( \ + *store, \ + std::move(encoded), \ + DRV_NAME, \ + mockXpSettings); \ + ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \ + ASSERT_EQ(got, expected); \ + } \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ + auto file = goldenMaster( #NAME ".drv"); \ + \ + auto got = (VAL).unparse(*store, false); \ + \ + if (testAccept()) \ + { \ + createDirs(dirOf(file)); \ + writeFile(file, got); \ + GTEST_SKIP() << updatingGoldenMaster; \ + } \ + else \ + { \ + auto expected = readFile(file); \ + ASSERT_EQ(got, expected); \ + } \ } Derivation makeSimpleDrv(const Store & store) { @@ -236,36 +274,9 @@ Derivation makeSimpleDrv(const Store & store) { return drv; } -TEST_JSON(DerivationTest, simple, - R"({ - "name": "simple-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": {}, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeSimpleDrv(*store)) +TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store)) TEST_ATERM(DerivationTest, simple, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", makeSimpleDrv(*store), "simple-derivation") @@ -321,45 +332,9 @@ Derivation makeDynDepDerivation(const Store & store) { return drv; } -TEST_JSON(DynDerivationTest, dynDerivationDeps, - R"({ - "name": "dyn-dep-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": { - "cat": { - "dynamicOutputs": {}, - "outputs": ["kitten"] - }, - "goose": { - "dynamicOutputs": {}, - "outputs": ["gosling"] - } - }, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeDynDepDerivation(*store)) +TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store)) TEST_ATERM(DynDerivationTest, dynDerivationDeps, - R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", makeDynDepDerivation(*store), "dyn-dep-derivation") diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh index 496915745..7fdd3e11c 100644 --- a/src/libstore/tests/protocol.hh +++ b/src/libstore/tests/protocol.hh @@ -29,7 +29,7 @@ public: { if (testAccept()) { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + GTEST_SKIP() << cannotReadGoldenMaster; } else { @@ -70,7 +70,7 @@ public: { createDirs(dirOf(file)); writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; + GTEST_SKIP() << updatingGoldenMaster; } else { diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv new file mode 100644 index 000000000..3cd1ded02 --- /dev/null +++ b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/unit-test-data/libstore/derivation/bad-version.drv new file mode 100644 index 000000000..bbf75c114 --- /dev/null +++ b/unit-test-data/libstore/derivation/bad-version.drv @@ -0,0 +1 @@ +DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/unit-test-data/libstore/derivation/dynDerivationDeps.drv new file mode 100644 index 000000000..cfffe48ec --- /dev/null +++ b/unit-test-data/libstore/derivation/dynDerivationDeps.drv @@ -0,0 +1 @@ +DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/unit-test-data/libstore/derivation/dynDerivationDeps.json new file mode 100644 index 000000000..9dbeb1f15 --- /dev/null +++ b/unit-test-data/libstore/derivation/dynDerivationDeps.json @@ -0,0 +1,38 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } + }, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "dyn-dep-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/unit-test-data/libstore/derivation/output-caFixedFlat.json new file mode 100644 index 000000000..fe000ea36 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedFlat.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "sha256", + "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/unit-test-data/libstore/derivation/output-caFixedNAR.json new file mode 100644 index 000000000..1afd60223 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedNAR.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "r:sha256", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/unit-test-data/libstore/derivation/output-caFixedText.json new file mode 100644 index 000000000..0b2cc8bbc --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedText.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "text:sha256", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/unit-test-data/libstore/derivation/output-caFloating.json new file mode 100644 index 000000000..9115de851 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFloating.json @@ -0,0 +1,3 @@ +{ + "hashAlgo": "r:sha256" +} diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/unit-test-data/libstore/derivation/output-deferred.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-deferred.json @@ -0,0 +1 @@ +{} diff --git a/unit-test-data/libstore/derivation/output-impure.json b/unit-test-data/libstore/derivation/output-impure.json new file mode 100644 index 000000000..62b61cdca --- /dev/null +++ b/unit-test-data/libstore/derivation/output-impure.json @@ -0,0 +1,4 @@ +{ + "hashAlgo": "r:sha256", + "impure": true +} diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/unit-test-data/libstore/derivation/output-inputAddressed.json new file mode 100644 index 000000000..86c7f3a05 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-inputAddressed.json @@ -0,0 +1,3 @@ +{ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/simple.drv b/unit-test-data/libstore/derivation/simple.drv new file mode 100644 index 000000000..bda74ad25 --- /dev/null +++ b/unit-test-data/libstore/derivation/simple.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/simple.json b/unit-test-data/libstore/derivation/simple.json new file mode 100644 index 000000000..20d0f8933 --- /dev/null +++ b/unit-test-data/libstore/derivation/simple.json @@ -0,0 +1,25 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "simple-derivation", + "outputs": {}, + "system": "wasm-sel4" +}