Merge remote-tracking branch 'origin/master' into lazy-trees

doc/manual/generate-manpage.nix

@@ -5,93 +5,106 @@ with import <nix/utils.nix>;
 
 let
 
-  showCommand =
+  showCommand = { command, details, filename }:
-    { command, def, filename }:
-    ''
-      **Warning**: This program is **experimental** and its interface is subject to change.
-    ''
-    + "# Name\n\n"
-    + "`${command}` - ${def.description}\n\n"
-    + "# Synopsis\n\n"
-    + showSynopsis { inherit command; args = def.args; }
-    + (if def.commands or {} != {}
-       then
     let
-           categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands)));
+      result = ''
-           listCommands = cmds:
+        > **Warning** \
-             concatStrings (map (name:
+        > This program is **experimental** and its interface is subject to change.
-               "* "
+
-               + "[`${command} ${name}`](./${appendName filename name}.md)"
+        # Name
-               + " - ${cmds.${name}.description}\n")
+
-               (attrNames cmds));
+        `${command}` - ${details.description}
-         in
+
-         "where *subcommand* is one of the following:\n\n"
+        # Synopsis
-         # FIXME: group by category
+
-         + (if length categories > 1
+        ${showSynopsis command details.args}
-            then
+
-              concatStrings (map
+        ${maybeSubcommands}
-                (cat:
+
-                  "**${toString cat.description}:**\n\n"
+        ${maybeDocumentation}
-                  + listCommands (filterAttrs (n: v: v.category == cat) def.commands)
+
-                  + "\n"
+        ${maybeOptions}
-                ) categories)
+      '';
-              + "\n"
+      showSynopsis = command: args:
-            else
+        let
-              listCommands def.commands
+          showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "...");
-              + "\n")
+          arguments = concatStringsSep " " (map showArgument args);
-       else "")
+        in ''
-    + (if def ? doc
+         `${command}` [*option*...] ${arguments}
-       then def.doc + "\n\n"
+        '';
-       else "")
+      maybeSubcommands = if details ? commands && details.commands != {}
-    + (let s = showOptions def.flags; in
+        then ''
-       if s != ""
+           where *subcommand* is one of the following:
-       then "# Options\n\n${s}"
+
-       else "")
+           ${subcommands}
-  ;
+         ''
+        else "";
+      subcommands = if length categories > 1
+        then listCategories
+        else listSubcommands details.commands;
+      categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues details.commands)));
+      listCategories = concatStrings (map showCategory categories);
+      showCategory = cat: ''
+        **${toString cat.description}:**
+
+        ${listSubcommands (filterAttrs (n: v: v.category == cat) details.commands)}
+      '';
+      listSubcommands = cmds: concatStrings (attrValues (mapAttrs showSubcommand cmds));
+      showSubcommand = name: subcmd: ''
+        * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
+      '';
+      maybeDocumentation = if details ? doc then details.doc else "";
+      maybeOptions = if details.flags == {} then "" else ''
+        # Options
+
+        ${showOptions details.flags}
+      '';
+      showOptions = options:
+        let
+          showCategory = cat: ''
+            ${if cat != "" then "**${cat}:**" else ""}
+
+            ${listOptions (filterAttrs (n: v: v.category == cat) options)}
+            '';
+          listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
+          showOption = name: option:
+            let
+              shortName = if option ? shortName then "/ `-${option.shortName}`" else "";
+              labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else "";
+            in trim ''
+              - `--${name}` ${shortName} ${labels}
+
+                ${option.description}
+            '';
+          categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues options)));
+        in concatStrings (map showCategory categories);
+    in squash result;
 
   appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name;
 
-  showOptions = flags:
+  processCommand = { command, details, filename }:
     let
-      categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues flags)));
+      cmd = {
-    in
+        inherit command;
-      concatStrings (map
+        name = filename + ".md";
-        (cat:
+        value = showCommand { inherit command details filename; };
-          (if cat != ""
+      };
-           then "**${cat}:**\n\n"
+      subcommand = subCmd: processCommand {
-           else "")
+        command = command + " " + subCmd;
-          + concatStrings
+        details = details.commands.${subCmd};
-            (map (longName:
+        filename = appendName filename subCmd;
-              let
+      };
-                flag = flags.${longName};
+    in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {});
-              in
-                "  - `--${longName}`"
-                + (if flag ? shortName then " / `-${flag.shortName}`" else "")
-                + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "")
-                + "  \n"
-                + "    " + flag.description + "\n\n"
-            ) (attrNames (filterAttrs (n: v: v.category == cat) flags))))
-        categories);
 
-  showSynopsis =
+  manpages = processCommand {
-    { command, args }:
+    command = "nix";
-    "`${command}` [*option*...] ${concatStringsSep " "
+    details = builtins.fromJSON command;
-      (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n";
+    filename = "nix";
+  };
 
-  processCommand = { command, def, filename }:
+  tableOfContents = let
-    [ { name = filename + ".md"; value = showCommand { inherit command def filename; }; inherit command; } ]
+    showEntry = page:
-    ++ concatMap
+      "    - [${page.command}](command-ref/new-cli/${page.name})";
-      (name: processCommand {
+    in concatStringsSep "\n" (map showEntry manpages) + "\n";
-        filename = appendName filename name;
-        command = command + " " + name;
-        def = def.commands.${name};
-      })
-      (attrNames def.commands or {});
 
-in
+in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; }
-
-let
-  manpages = processCommand { filename = "nix"; command = "nix"; def = builtins.fromJSON command; };
-  summary = concatStrings (map (manpage: "    - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages);
-in
-(listToAttrs manpages) // { "SUMMARY.md" = summary; }

doc/manual/src/command-ref/nix-copy-closure.md

@@ -30,8 +30,8 @@ Since `nix-copy-closure` calls `ssh`, you may be asked to type in the
 appropriate password or passphrase.  In fact, you may be asked _twice_
 because `nix-copy-closure` currently connects twice to the remote
 machine, first to get the set of paths missing on the target machine,
-and second to send the dump of those paths.  If this bothers you, use
+and second to send the dump of those paths.  When using public key
-`ssh-agent`.
+authentication, you can avoid typing the passphrase with `ssh-agent`.
 
 # Options
 

doc/manual/src/contributing/hacking.md

@@ -86,9 +86,7 @@ by:
 $ nix develop
 ```
 
-## Testing
+## Running tests
-
-Nix comes with three different flavors of tests: unit, functional and integration.
 
 ### Unit-tests
 
@@ -111,3 +109,72 @@ These tests include everything that needs to interact with external services or
 Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
 
 You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`
+
+### Installer tests
+
+After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch.
+
+Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91):
+
+- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache:
+  - `x86_64-linux`
+  - `armv6l-linux`
+  - `armv7l-linux`
+  - `x86_64-darwin`
+
+- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command.
+
+#### One-time setup
+
+1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix).
+2. At cachix.org:
+    - Create or log in to an account.
+    - Create a Cachix cache using the format `<github-username>-nix-install-tests`.
+    - Navigate to the new cache > Settings > Auth Tokens.
+    - Generate a new Cachix auth token and copy the generated value.
+3. At github.com:
+    - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret.
+    - Name the secret `CACHIX_AUTH_TOKEN`.
+    - Paste the copied value of the Cachix cache auth token.
+
+#### Using the CI-generated installer for manual testing
+
+After the CI run completes, you can check the output to extract the installer URL:
+1. Click into the detailed view of the CI run.
+2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them).
+3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`)
+4. Copy the value of `install_url`
+5. To generate an install command, plug this `install_url` and your GitHub username into this template:
+
+    ```console
+    sh <(curl -L <install_url>) --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve
+    ```
+
+<!-- #### Manually generating test installers
+
+There's obviously a manual way to do this, and it's still the only way for
+platforms that lack GA runners.
+
+I did do this back in Fall 2020 (before the GA approach encouraged here). I'll
+sketch what I recall in case it encourages someone to fill in detail, but: I
+didn't know what I was doing at the time and had to fumble/ask around a lot--
+so I don't want to uphold any of it as "right". It may have been dumb or
+the _hard_ way from the getgo. Fundamentals may have changed since.
+
+Here's the build command I used to do this on and for x86_64-darwin:
+nix build --out-link /tmp/foo ".#checks.x86_64-darwin.binaryTarball"
+
+I used the stable out-link to make it easier to script the next steps:
+link=$(readlink /tmp/foo)
+cp $link/*-darwin.tar.xz ~/somewheres
+
+I've lost the last steps and am just going from memory:
+
+From here, I think I had to extract and modify the `install` script to point
+it at this tarball (which I scped to my own site, but it might make more sense
+to just share them locally). I extracted this script once and then just
+search/replaced in it for each new build.
+
+The installer now supports a `--tarball-url-prefix` flag which _may_ have
+solved this need?
+-->

doc/manual/src/glossary.md

@@ -7,10 +7,44 @@
     translated into low-level *store derivations* (implicitly by
     `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
 
+  - [content-addressed derivation]{#gloss-content-addressed-derivation}\
+    A derivation which has the
+    [`__contentAddressed`](language/advanced-attributes.md#adv-attr-__contentAddressed)
+    attribute set to `true`.
+
+  - [fixed-output derivation]{#gloss-fixed-output-derivation}\
+    A derivation which includes the
+    [`outputHash`](language/advanced-attributes.md#adv-attr-outputHash) attribute.
+
   - [store]{#gloss-store}\
     The location in the file system where store objects live. Typically
     `/nix/store`.
 
+    From   the  perspective   of   the  location   where  Nix   is
+    invoked, the  Nix store can be  referred to
+    as a "_local_" or a "_remote_" one:
+
+    + A *local  store* exists  on the filesystem of
+      the machine where Nix is  invoked. You can use other
+      local stores  by passing  the `--store` flag  to the
+      `nix` command.  Local stores can be used for building derivations.
+
+    + A  *remote store*  exists  anywhere  other than  the
+      local  filesystem. One  example is  the `/nix/store`
+      directory on another machine,  accessed via `ssh` or
+      served by the `nix-serve` Perl script.
+
+  - [chroot store]{#gloss-chroot-store}\
+    A local store whose canonical path is anything other than `/nix/store`.
+
+  - [binary cache]{#gloss-binary-cache}\
+    A *binary cache* is a Nix store which uses a different format: its
+    metadata and signatures are kept in `.narinfo` files rather than in a
+    Nix database.  This different format simplifies serving store objects
+    over the network, but cannot host builds.  Examples of binary caches
+    include S3 buckets and the [NixOS binary
+    cache](https://cache.nixos.org).
+
   - [store path]{#gloss-store-path}\
     The location in the file system of a store object, i.e., an
     immediate child of the Nix store directory.
@@ -22,6 +56,19 @@
     derivation outputs (objects produced by running a build action), or
     derivations (files describing a build action).
 
+  - [input-addressed store object]{#gloss-input-addressed-store-object}\
+    A store object produced by building a
+    non-[content-addressed](#gloss-content-addressed-derivation),
+    non-[fixed-output](#gloss-fixed-output-derivation)
+    derivation.
+
+  - [output-addressed store object]{#gloss-output-addressed-store-object}\
+    A store object whose store path hashes its content.  This
+    includes derivations, the outputs of
+    [content-addressed derivations](#gloss-content-addressed-derivation),
+    and the outputs of
+    [fixed-output derivations](#gloss-fixed-output-derivation).
+
   - [substitute]{#gloss-substitute}\
     A substitute is a command invocation stored in the Nix database that
     describes how to build a store object, bypassing the normal build
@@ -29,6 +76,11 @@
     store object by downloading a pre-built version of the store object
     from some server.
 
+  - [substituter]{#gloss-substituter}\
+    A *substituter* is an additional store from which Nix will
+    copy store objects it doesn't have.  For details, see the
+    [`substituters` option](command-ref/conf-file.html#conf-substituters).
+
   - [purity]{#gloss-purity}\
     The assumption that equal Nix derivations when run always produce
     the same output. This cannot be guaranteed in general (e.g., a

doc/manual/utils.nix

@@ -5,6 +5,32 @@ rec {
 
   concatStrings = concatStringsSep "";
 
+  replaceStringsRec = from: to: string:
+    # recursively replace occurrences of `from` with `to` within `string`
+    # example:
+    #     replaceStringRec "--" "-" "hello-----world"
+    #     => "hello-world"
+    let
+      replaced = replaceStrings [ from ] [ to ] string;
+    in
+      if replaced == string then string else replaceStringsRec from to replaced;
+
+  squash = replaceStringsRec "\n\n\n" "\n\n";
+
+  trim = string:
+    # trim trailing spaces and squash non-leading spaces
+    let
+      trimLine = line:
+        let
+          # separate leading spaces from the rest
+          parts = split "(^ *)" line;
+          spaces = head (elemAt parts 1);
+          rest = elemAt parts 2;
+          # drop trailing spaces
+          body = head (split " *$" rest);
+        in spaces + replaceStringsRec "  " " " body;
+    in concatStringsSep "\n" (map trimLine (splitLines string));
+
   # FIXME: O(n^2)
   unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) [];
 

src/libstore/globals.hh

@@ -619,6 +619,14 @@ public:
           are tried based on their Priority value, which each substituter can set
           independently. Lower value means higher priority.
           The default is `https://cache.nixos.org`, with a Priority of 40.
+
+          Nix will copy a store path from a remote store only if one
+          of the following is true:
+
+          - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys)
+          - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list
+          - the [`require-sigs`](#conf-require-sigs) option has been set to `false`
+          - the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object)
         )",
         {"binary-caches"}};
 

src/libstore/nar-accessor.cc

@@ -75,6 +75,9 @@ struct NarAccessor : public FSAccessor
             createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
         }
 
+        void closeRegularFile() override
+        { }
+
         void isExecutable() override
         {
             parents.top()->isExecutable = true;

src/libutil/archive.cc

@@ -230,6 +230,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
 
         else if (s == "contents" && type == tpRegular) {
             parseContents(sink, source, path);
+            sink.closeRegularFile();
         }
 
         else if (s == "executable" && type == tpRegular) {
@@ -320,6 +321,12 @@ struct RestoreSink : ParseSink
         if (!fd) throw SysError("creating file '%1%'", p);
     }
 
+    void closeRegularFile() override
+    {
+        /* Call close explicitly to make sure the error is checked */
+        fd.close();
+    }
+
     void isExecutable() override
     {
         struct stat st;

src/libutil/archive.hh

@@ -60,6 +60,7 @@ struct ParseSink
     virtual void createDirectory(const Path & path) { };
 
     virtual void createRegularFile(const Path & path) { };
+    virtual void closeRegularFile() { };
     virtual void isExecutable() { };
     virtual void preallocateContents(uint64_t size) { };
     virtual void receiveContents(std::string_view data) { };

tests/build-dry.sh

@@ -18,9 +18,6 @@ nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built"
 # Now new command:
 nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 
-# TODO: XXX: FIXME: #1793
-# Disable this part of the test until the problem is resolved:
-if [ -n "$ISSUE_1795_IS_FIXED" ]; then
 clearStore
 clearCache
 
@@ -28,7 +25,6 @@ clearCache
 nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built"
 # Now old command:
 nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built"
-fi
 
 ###################################################
 # Check --dry-run doesn't create links with --dry-run