Merge remote-tracking branch 'nixos/master'

.version

@@ -1 +1 @@
-2.10.0
+2.12.0

configure.ac

@@ -296,15 +296,6 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
 AC_CHECK_FUNCS([strsignal posix_fallocate sysconf])
 
 
-# This is needed if bzip2 is a static library, and the Nix libraries
-# are dynamic.
-case "${host_os}" in
-  darwin*)
-    LDFLAGS="-all_load $LDFLAGS"
-    ;;
-esac
-
-
 AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]),
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)

doc/manual/src/SUMMARY.md.in

@@ -73,6 +73,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
   - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md)
   - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md)
   - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md)
   - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md)

doc/manual/src/installation/installing-binary.md

@@ -13,7 +13,7 @@ for your platform:
 - multi-user on macOS
 
   > **Notes on read-only filesystem root in macOS 10.15 Catalina +**
-  > 
+  >
   > - It took some time to support this cleanly. You may see posts,
   >   examples, and tutorials using obsolete workarounds.
   > - Supporting it cleanly made macOS installs too complex to qualify
@@ -31,8 +31,8 @@ $ sh <(curl -L https://nixos.org/nix/install) --no-daemon
 ```
 
 This will perform a single-user installation of Nix, meaning that `/nix`
-is owned by the invoking user. You should run this under your usual user
+is owned by the invoking user. You can run this under your usual user
-account, *not* as root. The script will invoke `sudo` to create `/nix`
+account or root. The script will invoke `sudo` to create `/nix`
 if it doesn’t already exist. If you don’t have `sudo`, you should
 manually create `/nix` first as root, e.g.:
 
@@ -71,11 +71,11 @@ $ sh <(curl -L https://nixos.org/nix/install) --daemon
 
 The multi-user installation of Nix will create build users between the
 user IDs 30001 and 30032, and a group with the group ID 30000. You
-should run this under your usual user account, *not* as root. The script
+can run this under your usual user account or root. The script
 will invoke `sudo` as needed.
 
 > **Note**
-> 
+>
 > If you need Nix to use a different group ID or user ID set, you will
 > have to download the tarball manually and [edit the install
 > script](#installing-from-a-binary-tarball).
@@ -168,7 +168,7 @@ and `/etc/zshrc` which you may remove.
    removed next.
 
 7. Remove the Nix Store volume:
-   
+
    ```console
    sudo diskutil apfs deleteVolume /nix
    ```
@@ -189,7 +189,7 @@ and `/etc/zshrc` which you may remove.
    identifier.
 
 > **Note**
-> 
+>
 > After you complete the steps here, you will still have an empty `/nix`
 > directory. This is an expected sign of a successful uninstall. The empty
 > `/nix` directory will disappear the next time you reboot.

doc/manual/src/language/index.md

@@ -1,13 +1,33 @@
 # Nix Language
 
-The Nix language is a pure, lazy, functional language. Purity
+The Nix language is
-means that operations in the language don't have side-effects (for
-instance, there is no variable assignment). Laziness means that
-arguments to functions are evaluated only when they are needed.
-Functional means that functions are “normal” values that can be passed
-around and manipulated in interesting ways. The language is not a
-full-featured, general purpose language. Its main job is to describe
-packages, compositions of packages, and the variability within packages.
 
-This section presents the various features of the language.
+- *domain-specific*
+
+  It only exists for the Nix package manager:
+  to describe packages and configurations as well as their variants and compositions.
+  It is not intended for general purpose use.
+
+- *declarative*
+
+  There is no notion of executing sequential steps.
+  Dependencies between operations are established only through data.
+
+- *pure*
+
+  Values cannot change during computation.
+  Functions always produce the same output if their input does not change.
+
+- *functional*
+
+  Functions are like any other value.
+  Functions can be assigned to names, taken as arguments, or returned by functions.
+
+- *lazy*
+
+  Expressions are only evaluated when their value is needed.
+
+- *dynamically typed*
+
+  Type errors are only detected when expressions are evaluated.
 

doc/manual/src/release-notes/rl-2.11.md

@@ -0,0 +1,5 @@
+# Release 2.11 (2022-08-24)
+
+* `nix copy` now copies the store paths in parallel as much as possible (again).
+  This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything
+  in one batch to avoid latencies issues.

docker.nix

@@ -2,6 +2,7 @@
 , lib ? pkgs.lib
 , name ? "nix"
 , tag ? "latest"
+, bundleNixpkgs ? true
 , channelName ? "nixpkgs"
 , channelURL ? "https://nixos.org/channels/nixpkgs-unstable"
 , extraPkgs ? []
@@ -139,10 +140,12 @@ let
   baseSystem =
     let
       nixpkgs = pkgs.path;
-      channel = pkgs.runCommand "channel-nixos" { } ''
+      channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } ''
         mkdir $out
-        ln -s ${nixpkgs} $out/nixpkgs
+        if [ "$bundleNixpkgs" ]; then
-        echo "[]" > $out/manifest.nix
+          ln -s ${nixpkgs} $out/nixpkgs
+          echo "[]" > $out/manifest.nix
+        fi
       '';
       rootEnv = pkgs.buildPackages.buildEnv {
         name = "root-profile-env";

scripts/install-darwin-multi-user.sh

@@ -167,7 +167,7 @@ poly_user_shell_get() {
 }
 
 poly_user_shell_set() {
-    _sudo "in order to give $1 a safe home directory" \
+    _sudo "in order to give $1 a safe shell" \
           /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2"
 }
 

scripts/install-multi-user.sh

@@ -59,6 +59,30 @@ headless() {
     fi
 }
 
+is_root() {
+    if [ "$EUID" -eq 0 ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+is_os_linux() {
+    if [ "$(uname -s)" = "Linux" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+is_os_darwin() {
+    if [ "$(uname -s)" = "Darwin" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
 contact_us() {
     echo "You can open an issue at https://github.com/nixos/nix/issues"
     echo ""
@@ -313,14 +337,23 @@ __sudo() {
 _sudo() {
     local expl="$1"
     shift
-    if ! headless; then
+    if ! headless || is_root; then
         __sudo "$expl" "$*" >&2
     fi
-    sudo "$@"
+
+    if is_root; then
+        env "$@"
+    else
+        sudo "$@"
+    fi
 }
 
+# Ensure that $TMPDIR exists if defined.
+if [[ -n "${TMPDIR:-}" ]] && [[ ! -d "${TMPDIR:-}" ]]; then
+    mkdir -m 0700 -p "${TMPDIR:-}"
+fi
 
-readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX")
+readonly SCRATCH=$(mktemp -d)
 finish_cleanup() {
     rm -rf "$SCRATCH"
 }
@@ -423,7 +456,7 @@ EOF
         fi
     done
 
-    if [ "$(uname -s)" = "Linux" ] && [ ! -e /run/systemd/system ]; then
+    if is_os_linux && [ ! -e /run/systemd/system ]; then
         warning <<EOF
 We did not detect systemd on your system. With a multi-user install
 without systemd you will have to manually configure your init system to
@@ -640,7 +673,7 @@ place_channel_configuration() {
 
 check_selinux() {
     if command -v getenforce > /dev/null 2>&1; then
-        if ! [ "$(getenforce)" = "Disabled" ]; then
+        if [ "$(getenforce)" = "Enforcing" ]; then
             failure <<EOF
 Nix does not work with selinux enabled yet!
 see https://github.com/NixOS/nix/issues/2374
@@ -865,24 +898,14 @@ EOF
           install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf
 }
 
-main() {
-    # TODO: I've moved this out of validate_starting_assumptions so we
-    # can fail faster in this case. Sourcing install-darwin... now runs
-    # `touch /` to detect Read-only root, but it could update times on
-    # pre-Catalina macOS if run as root user.
-    if [ "$EUID" -eq 0 ]; then
-        failure <<EOF
-Please do not run this script with root privileges. I will call sudo
-when I need to.
-EOF
-    fi
 
+main() {
     check_selinux
 
-    if [ "$(uname -s)" = "Darwin" ]; then
+    if is_os_darwin; then
         # shellcheck source=./install-darwin-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
-    elif [ "$(uname -s)" = "Linux" ]; then
+    elif is_os_linux; then
         # shellcheck source=./install-systemd-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also
     else
@@ -890,7 +913,10 @@ EOF
     fi
 
     welcome_to_nix
-    chat_about_sudo
+
+    if ! is_root; then
+        chat_about_sudo
+    fi
 
     cure_artifacts
     # TODO: there's a tension between cure and validate. I moved the

src/libcmd/installables.cc

@@ -692,6 +692,8 @@ InstallableFlake::InstallableFlake(
 
 std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
 {
+    Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what()));
+
     auto attr = getCursor(*state);
 
     auto attrPath = attr->getAttrPathStr();

src/libcmd/repl.cc

@@ -35,6 +35,7 @@ extern "C" {
 #include "finally.hh"
 #include "markdown.hh"
 #include "local-fs-store.hh"
+#include "progress-bar.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
@@ -252,6 +253,10 @@ void NixRepl::mainLoop()
     rl_set_list_possib_func(listPossibleCallback);
 #endif
 
+    /* Stop the progress bar because it interferes with the display of
+       the repl. */
+    stopProgressBar();
+
     std::string input;
 
     while (true) {
@@ -1037,9 +1042,10 @@ void runRepl(
 
 struct CmdRepl : InstallablesCommand
 {
-    CmdRepl(){
+    CmdRepl() {
         evalSettings.pureEval = false;
     }
+
     void prepare()
     {
         if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) {
@@ -1053,12 +1059,15 @@ struct CmdRepl : InstallablesCommand
         }
         installables = InstallablesCommand::load();
     }
+
     std::vector<std::string> files;
+
     Strings getDefaultFlakeAttrPaths() override
     {
         return {""};
     }
-    virtual bool useDefaultInstallables() override
+
+    bool useDefaultInstallables() override
     {
         return file.has_value() or expr.has_value();
     }

src/libexpr/eval-cache.cc

@@ -507,11 +507,6 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
         return nullptr;
         //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
 
-    for (auto & attr : *v.attrs) {
-        if (root->db)
-            root->db->setPlaceholder({cachedValue->first, attr.name});
-    }
-
     auto attr = v.attrs->get(name);
 
     if (!attr) {

src/libexpr/value-to-json.cc

@@ -10,7 +10,7 @@
 namespace nix {
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context)
+    Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore)
 {
     checkInterrupt();
 
@@ -32,7 +32,10 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         case nPath:
-            out.write(state.copyPathToStore(context, v.path));
+            if (copyToStore)
+                out.write(state.copyPathToStore(context, v.path));
+            else
+                out.write(v.path);
             break;
 
         case nNull:
@@ -54,10 +57,10 @@ void printValueAsJSON(EvalState & state, bool strict,
                 for (auto & j : names) {
                     Attr & a(*v.attrs->find(state.symbols.create(j)));
                     auto placeholder(obj.placeholder(j));
-                    printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context);
+                    printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context, copyToStore);
                 }
             } else
-                printValueAsJSON(state, strict, *i->value, i->pos, out, context);
+                printValueAsJSON(state, strict, *i->value, i->pos, out, context, copyToStore);
             break;
         }
 
@@ -65,13 +68,13 @@ void printValueAsJSON(EvalState & state, bool strict,
             auto list(out.list());
             for (auto elem : v.listItems()) {
                 auto placeholder(list.placeholder());
-                printValueAsJSON(state, strict, *elem, pos, placeholder, context);
+                printValueAsJSON(state, strict, *elem, pos, placeholder, context, copyToStore);
             }
             break;
         }
 
         case nExternal:
-            v.external->printValueAsJSON(state, strict, out, context);
+            v.external->printValueAsJSON(state, strict, out, context, copyToStore);
             break;
 
         case nFloat:
@@ -91,14 +94,14 @@ void printValueAsJSON(EvalState & state, bool strict,
 }
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, std::ostream & str, PathSet & context)
+    Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore)
 {
     JSONPlaceholder out(str);
-    printValueAsJSON(state, strict, v, pos, out, context);
+    printValueAsJSON(state, strict, v, pos, out, context, copyToStore);
 }
 
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
-    JSONPlaceholder & out, PathSet & context) const
+    JSONPlaceholder & out, PathSet & context, bool copyToStore) const
 {
     state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType()));
 }

src/libexpr/value-to-json.hh

@@ -11,9 +11,9 @@ namespace nix {
 class JSONPlaceholder;
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context);
+    Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore = true);
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, const PosIdx pos, std::ostream & str, PathSet & context);
+    Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true);
 
 }

src/libexpr/value.hh

@@ -99,7 +99,7 @@ class ExternalValueBase
 
     /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */
     virtual void printValueAsJSON(EvalState & state, bool strict,
-        JSONPlaceholder & out, PathSet & context) const;
+        JSONPlaceholder & out, PathSet & context, bool copyToStore = true) const;
 
     /* Print the value as XML. Defaults to unevaluated */
     virtual void printValueAsXML(EvalState & state, bool strict, bool location,

src/libfetchers/git.cc

@@ -371,7 +371,7 @@ struct GitInputScheme : InputScheme
         auto gitDir = ".git";
 
         runProgram("git", true,
-            { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) });
+            { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) });
 
         if (commitMsg)
             runProgram("git", true,

src/libmain/loggers.cc

@@ -30,8 +30,11 @@ Logger * makeDefaultLogger() {
         return makeJSONLogger(*makeSimpleLogger(true));
     case LogFormat::bar:
         return makeProgressBar();
-    case LogFormat::barWithLogs:
+    case LogFormat::barWithLogs: {
-        return makeProgressBar(true);
+        auto logger = makeProgressBar();
+        logger->setPrintBuildLogs(true);
+        return logger;
+    }
     default:
         abort();
     }

src/libmain/progress-bar.cc

@@ -8,6 +8,7 @@
 #include <map>
 #include <thread>
 #include <iostream>
+#include <chrono>
 
 namespace nix {
 
@@ -48,6 +49,7 @@ private:
         bool visible = true;
         ActivityId parent;
         std::optional<std::string> name;
+        std::chrono::time_point<std::chrono::steady_clock> startTime;
     };
 
     struct ActivitiesByType
@@ -79,22 +81,22 @@ private:
 
     std::condition_variable quitCV, updateCV;
 
-    bool printBuildLogs;
+    bool printBuildLogs = false;
     bool isTTY;
 
 public:
 
-    ProgressBar(bool printBuildLogs, bool isTTY)
+    ProgressBar(bool isTTY)
-        : printBuildLogs(printBuildLogs)
+        : isTTY(isTTY)
-        , isTTY(isTTY)
     {
         state_.lock()->active = isTTY;
         updateThread = std::thread([&]() {
             auto state(state_.lock());
+            auto nextWakeup = std::chrono::milliseconds::max();
             while (state->active) {
                 if (!state->haveUpdate)
-                    state.wait(updateCV);
+                    state.wait_for(updateCV, nextWakeup);
-                draw(*state);
+                nextWakeup = draw(*state);
                 state.wait_for(quitCV, std::chrono::milliseconds(50));
             }
         });
@@ -118,7 +120,8 @@ public:
         updateThread.join();
     }
 
-    bool isVerbose() override {
+    bool isVerbose() override
+    {
         return printBuildLogs;
     }
 
@@ -159,11 +162,13 @@ public:
         if (lvl <= verbosity && !s.empty() && type != actBuildWaiting)
             log(*state, lvl, s + "...");
 
-        state->activities.emplace_back(ActInfo());
+        state->activities.emplace_back(ActInfo {
+            .s = s,
+            .type = type,
+            .parent = parent,
+            .startTime = std::chrono::steady_clock::now()
+        });
         auto i = std::prev(state->activities.end());
-        i->s = s;
-        i->type = type;
-        i->parent = parent;
         state->its.emplace(act, i);
         state->activitiesByType[type].its.emplace(act, i);
 
@@ -327,10 +332,12 @@ public:
         updateCV.notify_one();
     }
 
-    void draw(State & state)
+    std::chrono::milliseconds draw(State & state)
     {
+        auto nextWakeup = std::chrono::milliseconds::max();
+
         state.haveUpdate = false;
-        if (!state.active) return;
+        if (!state.active) return nextWakeup;
 
         std::string line;
 
@@ -341,12 +348,25 @@ public:
             line += "]";
         }
 
+        auto now = std::chrono::steady_clock::now();
+
         if (!state.activities.empty()) {
             if (!status.empty()) line += " ";
             auto i = state.activities.rbegin();
 
-            while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty())))
+            while (i != state.activities.rend()) {
+                if (i->visible && (!i->s.empty() || !i->lastLine.empty())) {
+                    /* Don't show activities until some time has
+                       passed, to avoid displaying very short
+                       activities. */
+                    auto delay = std::chrono::milliseconds(10);
+                    if (i->startTime + delay < now)
+                        break;
+                    else
+                        nextWakeup = std::min(nextWakeup, std::chrono::duration_cast<std::chrono::milliseconds>(delay - (now - i->startTime)));
+                }
                 ++i;
+            }
 
             if (i != state.activities.rend()) {
                 line += i->s;
@@ -366,6 +386,8 @@ public:
         if (width <= 0) width = std::numeric_limits<decltype(width)>::max();
 
         writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
+
+        return nextWakeup;
     }
 
     std::string getStatus(State & state)
@@ -480,19 +502,21 @@ public:
         draw(*state);
         return s[0];
     }
+
+    virtual void setPrintBuildLogs(bool printBuildLogs)
+    {
+        this->printBuildLogs = printBuildLogs;
+    }
 };
 
-Logger * makeProgressBar(bool printBuildLogs)
+Logger * makeProgressBar()
 {
-    return new ProgressBar(
+    return new ProgressBar(shouldANSI());
-        printBuildLogs,
-        shouldANSI()
-    );
 }
 
-void startProgressBar(bool printBuildLogs)
+void startProgressBar()
 {
-    logger = makeProgressBar(printBuildLogs);
+    logger = makeProgressBar();
 }
 
 void stopProgressBar()

src/libmain/progress-bar.hh

@@ -4,9 +4,9 @@
 
 namespace nix {
 
-Logger * makeProgressBar(bool printBuildLogs = false);
+Logger * makeProgressBar();
 
-void startProgressBar(bool printBuildLogs = false);
+void startProgressBar();
 
 void stopProgressBar();
 

src/libstore/build/derivation-goal.cc

@@ -705,8 +705,7 @@ static void movePath(const Path & src, const Path & dst)
     if (changePerm)
         chmod_(src, st.st_mode | S_IWUSR);
 
-    if (rename(src.c_str(), dst.c_str()))
+    renameFile(src, dst);
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
 
     if (changePerm)
         chmod_(dst, st.st_mode);
@@ -914,12 +913,6 @@ void DerivationGoal::buildDone()
             outputPaths
         );
 
-        if (buildMode == bmCheck) {
-            cleanupPostOutputsRegisteredModeCheck();
-            done(BuildResult::Built, std::move(builtOutputs));
-            return;
-        }
-
         cleanupPostOutputsRegisteredModeNonCheck();
 
         /* Repeat the build if necessary. */

src/libstore/build/local-derivation-goal.cc

@@ -223,8 +223,7 @@ static void movePath(const Path & src, const Path & dst)
     if (changePerm)
         chmod_(src, st.st_mode | S_IWUSR);
 
-    if (rename(src.c_str(), dst.c_str()))
+    renameFile(src, dst);
-        throw SysError("renaming '%1%' to '%2%'", src, dst);
 
     if (changePerm)
         chmod_(dst, st.st_mode);
@@ -311,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
             if (buildMode != bmCheck && status.known->isValid()) continue;
             auto p = worker.store.printStorePath(status.known->path);
             if (pathExists(chrootRootDir + p))
-                rename((chrootRootDir + p).c_str(), p.c_str());
+                renameFile((chrootRootDir + p), p);
         }
 
     return diskFull;
@@ -2375,10 +2374,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
             if (*scratchPath != finalPath) {
                 // Also rewrite the output path
                 auto source = sinkToSource([&](Sink & nextSink) {
-                    StringSink sink;
-                    dumpPath(actualPath, sink);
                     RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink);
-                    rsink2(sink.s);
+                    dumpPath(actualPath, rsink2);
                     rsink2.flush();
                 });
                 Path tmpPath = actualPath + ".tmp";
@@ -2625,8 +2622,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
             Path prev = path + checkSuffix;
             deletePath(prev);
             Path dst = path + checkSuffix;
-            if (rename(path.c_str(), dst.c_str()))
+            renameFile(path, dst);
-                throw SysError("renaming '%s' to '%s'", path, dst);
         }
     }
 

src/libstore/builtins/unpack-channel.cc

@@ -22,8 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv)
     auto entries = readDirectory(out);
     if (entries.size() != 1)
         throw Error("channel tarball '%s' contains more than one file", src);
-    if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1)
+    renameFile((out + "/" + entries[0].name), (out + "/" + channelName));
-        throw SysError("renaming channel directory");
 }
 
 }

src/libstore/filetransfer.cc

@@ -308,6 +308,9 @@ struct curlFileTransfer : public FileTransfer
 
             curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
 
+            if (settings.downloadSpeed.get() > 0)
+                curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
+
             if (request.head)
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 

src/libstore/gc.cc

@@ -39,9 +39,7 @@ static void makeSymlink(const Path & link, const Path & target)
     createSymlink(target, tempLink);
 
     /* Atomically replace the old one. */
-    if (rename(tempLink.c_str(), link.c_str()) == -1)
+    renameFile(tempLink, link);
-        throw SysError("cannot rename '%1%' to '%2%'",
-            tempLink , link);
 }
 
 

src/libstore/globals.hh

@@ -746,6 +746,13 @@ public:
               /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
         )"};
 
+    Setting<unsigned int> downloadSpeed {
+        this, 0, "download-speed",
+        R"(
+          Specify the maximum transfer rate in kilobytes per second you want
+          Nix to use for downloads.
+        )"};
+
     Setting<std::string> netrcFile{
         this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
         R"(

src/libstore/local-binary-cache-store.cc

@@ -57,8 +57,7 @@ protected:
         AutoDelete del(tmp, false);
         StreamToSourceAdapter source(istream);
         writeFile(tmp, source);
-        if (rename(tmp.c_str(), path2.c_str()))
+        renameFile(tmp, path2);
-            throw SysError("renaming '%1%' to '%2%'", tmp, path2);
         del.cancel();
     }
 

src/libstore/local-store.cc

@@ -1430,8 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
                     writeFile(realPath, dumpSource);
             } else {
                 /* Move the temporary path we restored above. */
-                if (rename(tempPath.c_str(), realPath.c_str()))
+                moveFile(tempPath, realPath);
-                    throw Error("renaming '%s' to '%s'", tempPath, realPath);
             }
 
             /* For computing the nar hash. In recursive SHA-256 mode, this
@@ -1942,8 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
 
     writeFile(tmpFile, compress("bzip2", log));
 
-    if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
+    renameFile(tmpFile, logPath);
-        throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
 }
 
 std::optional<std::string> LocalStore::getVersion()

src/libstore/optimise-store.cc

@@ -229,7 +229,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     }
 
     /* Atomically replace the old file with the new hard link. */
-    if (rename(tempLink.c_str(), path.c_str()) == -1) {
+    try {
+        renameFile(tempLink, path);
+    } catch (SysError & e) {
         if (unlink(tempLink.c_str()) == -1)
             printError("unable to unlink '%1%'", tempLink);
         if (errno == EMLINK) {
@@ -240,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
             debug("'%s' has reached maximum number of links", linkPath);
             return;
         }
-        throw SysError("cannot rename '%1%' to '%2%'", tempLink, path);
+        throw;
     }
 
     stats.filesLinked++;

src/libstore/remote-store.cc

@@ -580,7 +580,6 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 
                 try {
                     conn->to.written = 0;
-                    conn->to.warn = true;
                     connections->incCapacity();
                     {
                         Finally cleanup([&]() { connections->decCapacity(); });
@@ -591,7 +590,6 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                             dumpString(contents, conn->to);
                         }
                     }
-                    conn->to.warn = false;
                     conn.processStderr();
                 } catch (SysError & e) {
                     /* Daemon closed while we were sending the path. Probably OOM
@@ -673,6 +671,23 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 }
 
 
+void RemoteStore::addMultipleToStore(
+    PathsSource & pathsToCopy,
+    Activity & act,
+    RepairFlag repair,
+    CheckSigsFlag checkSigs)
+{
+    auto source = sinkToSource([&](Sink & sink) {
+        sink << pathsToCopy.size();
+        for (auto & [pathInfo, pathSource] : pathsToCopy) {
+            pathInfo.write(sink, *this, 16);
+            pathSource->drainInto(sink);
+        }
+    });
+
+    addMultipleToStore(*source, repair, checkSigs);
+}
+
 void RemoteStore::addMultipleToStore(
     Source & source,
     RepairFlag repair,

src/libstore/remote-store.hh

@@ -88,6 +88,12 @@ public:
         RepairFlag repair,
         CheckSigsFlag checkSigs) override;
 
+    void addMultipleToStore(
+        PathsSource & pathsToCopy,
+        Activity & act,
+        RepairFlag repair,
+        CheckSigsFlag checkSigs) override;
+
     StorePath addTextToStore(
         std::string_view name,
         std::string_view s,

src/libstore/store-api.cc

@@ -258,6 +258,84 @@ StorePath Store::addToStore(
     return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
 }
 
+void Store::addMultipleToStore(
+    PathsSource & pathsToCopy,
+    Activity & act,
+    RepairFlag repair,
+    CheckSigsFlag checkSigs)
+{
+    std::atomic<size_t> nrDone{0};
+    std::atomic<size_t> nrFailed{0};
+    std::atomic<uint64_t> bytesExpected{0};
+    std::atomic<uint64_t> nrRunning{0};
+
+    using PathWithInfo = std::pair<ValidPathInfo, std::unique_ptr<Source>>;
+
+    std::map<StorePath, PathWithInfo *> infosMap;
+    StorePathSet storePathsToAdd;
+    for (auto & thingToAdd : pathsToCopy) {
+        infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd);
+        storePathsToAdd.insert(thingToAdd.first.path);
+    }
+
+    auto showProgress = [&]() {
+        act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed);
+    };
+
+    ThreadPool pool;
+
+    processGraph<StorePath>(pool,
+        storePathsToAdd,
+
+        [&](const StorePath & path) {
+
+            auto & [info, _] = *infosMap.at(path);
+
+            if (isValidPath(info.path)) {
+                nrDone++;
+                showProgress();
+                return StorePathSet();
+            }
+
+            bytesExpected += info.narSize;
+            act.setExpected(actCopyPath, bytesExpected);
+
+            return info.references;
+        },
+
+        [&](const StorePath & path) {
+            checkInterrupt();
+
+            auto & [info_, source_] = *infosMap.at(path);
+            auto info = info_;
+            info.ultimate = false;
+
+            /* Make sure that the Source object is destroyed when
+               we're done. In particular, a SinkToSource object must
+               be destroyed to ensure that the destructors on its
+               stack frame are run; this includes
+               LegacySSHStore::narFromPath()'s connection lock. */
+            auto source = std::move(source_);
+
+            if (!isValidPath(info.path)) {
+                MaintainCount<decltype(nrRunning)> mc(nrRunning);
+                showProgress();
+                try {
+                    addToStore(info, *source, repair, checkSigs);
+                } catch (Error & e) {
+                    nrFailed++;
+                    if (!settings.keepGoing)
+                        throw e;
+                    printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what());
+                    showProgress();
+                    return;
+                }
+            }
+
+            nrDone++;
+            showProgress();
+        });
+}
 
 void Store::addMultipleToStore(
     Source & source,
@@ -992,113 +1070,61 @@ std::map<StorePath, StorePath> copyPaths(
     for (auto & path : storePaths)
         if (!valid.count(path)) missing.insert(path);
 
+    Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
+
+    // In the general case, `addMultipleToStore` requires a sorted list of
+    // store paths to add, so sort them right now
+    auto sortedMissing = srcStore.topoSortPaths(missing);
+    std::reverse(sortedMissing.begin(), sortedMissing.end());
+
     std::map<StorePath, StorePath> pathsMap;
     for (auto & path : storePaths)
         pathsMap.insert_or_assign(path, path);
 
-    Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
+    Store::PathsSource pathsToCopy;
 
-    auto sorted = srcStore.topoSortPaths(missing);
+    auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath {
-    std::reverse(sorted.begin(), sorted.end());
+        auto storePathForSrc = currentPathInfo.path;
+        auto storePathForDst = storePathForSrc;
+        if (currentPathInfo.ca && currentPathInfo.references.empty()) {
+            storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca);
+            if (dstStore.storeDir == srcStore.storeDir)
+                assert(storePathForDst == storePathForSrc);
+            if (storePathForDst != storePathForSrc)
+                debug("replaced path '%s' to '%s' for substituter '%s'",
+                        srcStore.printStorePath(storePathForSrc),
+                        dstStore.printStorePath(storePathForDst),
+                        dstStore.getUri());
+        }
+        return storePathForDst;
+    };
 
-    auto source = sinkToSource([&](Sink & sink) {
+    for (auto & missingPath : sortedMissing) {
-        sink << sorted.size();
+        auto info = srcStore.queryPathInfo(missingPath);
-        for (auto & storePath : sorted) {
+
+        auto storePathForDst = computeStorePathForDst(*info);
+        pathsMap.insert_or_assign(missingPath, storePathForDst);
+
+        ValidPathInfo infoForDst = *info;
+        infoForDst.path = storePathForDst;
+
+        auto source = sinkToSource([&](Sink & sink) {
+            // We can reasonably assume that the copy will happen whenever we
+            // read the path, so log something about that at that point
             auto srcUri = srcStore.getUri();
             auto dstUri = dstStore.getUri();
-            auto storePathS = srcStore.printStorePath(storePath);
+            auto storePathS = srcStore.printStorePath(missingPath);
             Activity act(*logger, lvlInfo, actCopyPath,
                 makeCopyPathMessage(srcUri, dstUri, storePathS),
                 {storePathS, srcUri, dstUri});
             PushActivity pact(act.id);
 
-            auto info = srcStore.queryPathInfo(storePath);
+            srcStore.narFromPath(missingPath, sink);
-            info->write(sink, srcStore, 16);
-            srcStore.narFromPath(storePath, sink);
-        }
-    });
-
-    dstStore.addMultipleToStore(*source, repair, checkSigs);
-
-    #if 0
-    std::atomic<size_t> nrDone{0};
-    std::atomic<size_t> nrFailed{0};
-    std::atomic<uint64_t> bytesExpected{0};
-    std::atomic<uint64_t> nrRunning{0};
-
-    auto showProgress = [&]() {
-        act.progress(nrDone, missing.size(), nrRunning, nrFailed);
-    };
-
-    ThreadPool pool;
-
-    processGraph<StorePath>(pool,
-        StorePathSet(missing.begin(), missing.end()),
-
-        [&](const StorePath & storePath) {
-            auto info = srcStore.queryPathInfo(storePath);
-            auto storePathForDst = storePath;
-            if (info->ca && info->references.empty()) {
-                storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
-                if (dstStore.storeDir == srcStore.storeDir)
-                    assert(storePathForDst == storePath);
-                if (storePathForDst != storePath)
-                    debug("replaced path '%s' to '%s' for substituter '%s'",
-                        srcStore.printStorePath(storePath),
-                        dstStore.printStorePath(storePathForDst),
-                        dstStore.getUri());
-            }
-            pathsMap.insert_or_assign(storePath, storePathForDst);
-
-            if (dstStore.isValidPath(storePath)) {
-                nrDone++;
-                showProgress();
-                return StorePathSet();
-            }
-
-            bytesExpected += info->narSize;
-            act.setExpected(actCopyPath, bytesExpected);
-
-            return info->references;
-        },
-
-        [&](const StorePath & storePath) {
-            checkInterrupt();
-
-            auto info = srcStore.queryPathInfo(storePath);
-
-            auto storePathForDst = storePath;
-            if (info->ca && info->references.empty()) {
-                storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
-                if (dstStore.storeDir == srcStore.storeDir)
-                    assert(storePathForDst == storePath);
-                if (storePathForDst != storePath)
-                    debug("replaced path '%s' to '%s' for substituter '%s'",
-                        srcStore.printStorePath(storePath),
-                        dstStore.printStorePath(storePathForDst),
-                        dstStore.getUri());
-            }
-            pathsMap.insert_or_assign(storePath, storePathForDst);
-
-            if (!dstStore.isValidPath(storePathForDst)) {
-                MaintainCount<decltype(nrRunning)> mc(nrRunning);
-                showProgress();
-                try {
-                    copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
-                } catch (Error &e) {
-                    nrFailed++;
-                    if (!settings.keepGoing)
-                        throw e;
-                    printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what());
-                    showProgress();
-                    return;
-                }
-            }
-
-            nrDone++;
-            showProgress();
         });
-    #endif
+        pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
+    }
+
+    dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
 
     return pathsMap;
 }

src/libstore/store-api.hh

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "nar-info.hh"
 #include "realisation.hh"
 #include "path.hh"
 #include "derived-path.hh"
@@ -359,12 +360,22 @@ public:
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
 
+    // A list of paths infos along with a source providing the content of the
+    // associated store path
+    using PathsSource = std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>>;
+
     /* Import multiple paths into the store. */
     virtual void addMultipleToStore(
         Source & source,
         RepairFlag repair = NoRepair,
         CheckSigsFlag checkSigs = CheckSigs);
 
+    virtual void addMultipleToStore(
+        PathsSource & pathsToCopy,
+        Activity & act,
+        RepairFlag repair = NoRepair,
+        CheckSigsFlag checkSigs = CheckSigs);
+
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
        The function object `filter' can be used to exclude files (see

src/libutil/filesystem.cc

@@ -0,0 +1,172 @@
+#include <sys/time.h>
+#include <filesystem>
+
+#include "finally.hh"
+#include "util.hh"
+#include "types.hh"
+
+namespace fs = std::filesystem;
+
+namespace nix {
+
+static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
+    int & counter)
+{
+    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
+    if (includePid)
+        return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
+    else
+        return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str();
+}
+
+Path createTempDir(const Path & tmpRoot, const Path & prefix,
+    bool includePid, bool useGlobalCounter, mode_t mode)
+{
+    static int globalCounter = 0;
+    int localCounter = 0;
+    int & counter(useGlobalCounter ? globalCounter : localCounter);
+
+    while (1) {
+        checkInterrupt();
+        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
+        if (mkdir(tmpDir.c_str(), mode) == 0) {
+#if __FreeBSD__
+            /* Explicitly set the group of the directory.  This is to
+               work around around problems caused by BSD's group
+               ownership semantics (directories inherit the group of
+               the parent).  For instance, the group of /tmp on
+               FreeBSD is "wheel", so all directories created in /tmp
+               will be owned by "wheel"; but if the user is not in
+               "wheel", then "tar" will fail to unpack archives that
+               have the setgid bit set on directories. */
+            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
+                throw SysError("setting group of directory '%1%'", tmpDir);
+#endif
+            return tmpDir;
+        }
+        if (errno != EEXIST)
+            throw SysError("creating directory '%1%'", tmpDir);
+    }
+}
+
+
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
+{
+    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    // Strictly speaking, this is UB, but who cares...
+    // FIXME: use O_TMPFILE.
+    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
+    if (!fd)
+        throw SysError("creating temporary file '%s'", tmpl);
+    closeOnExec(fd.get());
+    return {std::move(fd), tmpl};
+}
+
+void createSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
+{
+    if (symlink(target.c_str(), link.c_str()))
+        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
+    if (mtime) {
+        struct timeval times[2];
+        times[0].tv_sec = *mtime;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = *mtime;
+        times[1].tv_usec = 0;
+        if (lutimes(link.c_str(), times))
+            throw SysError("setting time of symlink '%s'", link);
+    }
+}
+
+void replaceSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
+{
+    for (unsigned int n = 0; true; n++) {
+        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
+
+        try {
+            createSymlink(target, tmp, mtime);
+        } catch (SysError & e) {
+            if (e.errNo == EEXIST) continue;
+            throw;
+        }
+
+        renameFile(tmp, link);
+
+        break;
+    }
+}
+
+void setWriteTime(const fs::path & p, const struct stat & st)
+{
+    struct timeval times[2];
+    times[0] = {
+        .tv_sec = st.st_atime,
+        .tv_usec = 0,
+    };
+    times[1] = {
+        .tv_sec = st.st_mtime,
+        .tv_usec = 0,
+    };
+    if (lutimes(p.c_str(), times) != 0)
+        throw SysError("changing modification time of '%s'", p);
+}
+
+void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
+{
+    // TODO: Rewrite the `is_*` to use `symlink_status()`
+    auto statOfFrom = lstat(from.path().c_str());
+    auto fromStatus = from.symlink_status();
+
+    // Mark the directory as writable so that we can delete its children
+    if (andDelete && fs::is_directory(fromStatus)) {
+        fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+    }
+
+
+    if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
+        fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
+    } else if (fs::is_directory(fromStatus)) {
+        fs::create_directory(to);
+        for (auto & entry : fs::directory_iterator(from.path())) {
+            copy(entry, to / entry.path().filename(), andDelete);
+        }
+    } else {
+        throw Error("file '%s' has an unsupported type", from.path());
+    }
+
+    setWriteTime(to, statOfFrom);
+    if (andDelete) {
+        if (!fs::is_symlink(fromStatus))
+            fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+        fs::remove(from.path());
+    }
+}
+
+void renameFile(const Path & oldName, const Path & newName)
+{
+    fs::rename(oldName, newName);
+}
+
+void moveFile(const Path & oldName, const Path & newName)
+{
+    try {
+        renameFile(oldName, newName);
+    } catch (fs::filesystem_error & e) {
+        auto oldPath = fs::path(oldName);
+        auto newPath = fs::path(newName);
+        // For the move to be as atomic as possible, copy to a temporary
+        // directory
+        fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
+        Finally removeTemp = [&]() { fs::remove(temp); };
+        auto tempCopyTarget = temp / "copy-target";
+        if (e.code().value() == EXDEV) {
+            fs::remove(newPath);
+            warn("Can’t rename %s as %s, copying instead", oldName, newName);
+            copy(fs::directory_entry(oldPath), tempCopyTarget, true);
+            renameFile(tempCopyTarget, newPath);
+        }
+    }
+}
+
+}

src/libutil/json.cc

@@ -193,7 +193,11 @@ JSONObject JSONPlaceholder::object()
 
 JSONPlaceholder::~JSONPlaceholder()
 {
-    assert(!first || std::uncaught_exceptions());
+    if (first) {
+        assert(std::uncaught_exceptions());
+        if (state->stack != 0)
+            write(nullptr);
+    }
 }
 
 }

src/libutil/logging.hh

@@ -111,6 +111,9 @@ public:
 
     virtual std::optional<char> ask(std::string_view s)
     { return {}; }
+
+    virtual void setPrintBuildLogs(bool printBuildLogs)
+    { }
 };
 
 ActivityId getCurActivity();

src/libutil/serialise.cc

@@ -48,24 +48,9 @@ FdSink::~FdSink()
 }
 
 
-size_t threshold = 256 * 1024 * 1024;
-
-static void warnLargeDump()
-{
-    warn("dumping very large path (> 256 MiB); this may run out of memory");
-}
-
-
 void FdSink::write(std::string_view data)
 {
     written += data.size();
-    static bool warned = false;
-    if (warn && !warned) {
-        if (written > threshold) {
-            warnLargeDump();
-            warned = true;
-        }
-    }
     try {
         writeFull(fd, data);
     } catch (SysError & e) {
@@ -448,11 +433,6 @@ Error readError(Source & source)
 
 void StringSink::operator () (std::string_view data)
 {
-    static bool warned = false;
-    if (!warned && s.size() > threshold) {
-        warnLargeDump();
-        warned = true;
-    }
     s.append(data);
 }
 

src/libutil/serialise.hh

@@ -97,19 +97,17 @@ protected:
 struct FdSink : BufferedSink
 {
     int fd;
-    bool warn = false;
     size_t written = 0;
 
     FdSink() : fd(-1) { }
     FdSink(int fd) : fd(fd) { }
     FdSink(FdSink&&) = default;
 
-    FdSink& operator=(FdSink && s)
+    FdSink & operator=(FdSink && s)
     {
         flush();
         fd = s.fd;
         s.fd = -1;
-        warn = s.warn;
         written = s.written;
         return *this;
     }

src/libutil/util.cc

@@ -508,61 +508,6 @@ void deletePath(const Path & path, uint64_t & bytesFreed)
 }
 
 
-static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
-    int & counter)
-{
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
-    if (includePid)
-        return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
-    else
-        return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str();
-}
-
-
-Path createTempDir(const Path & tmpRoot, const Path & prefix,
-    bool includePid, bool useGlobalCounter, mode_t mode)
-{
-    static int globalCounter = 0;
-    int localCounter = 0;
-    int & counter(useGlobalCounter ? globalCounter : localCounter);
-
-    while (1) {
-        checkInterrupt();
-        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
-        if (mkdir(tmpDir.c_str(), mode) == 0) {
-#if __FreeBSD__
-            /* Explicitly set the group of the directory.  This is to
-               work around around problems caused by BSD's group
-               ownership semantics (directories inherit the group of
-               the parent).  For instance, the group of /tmp on
-               FreeBSD is "wheel", so all directories created in /tmp
-               will be owned by "wheel"; but if the user is not in
-               "wheel", then "tar" will fail to unpack archives that
-               have the setgid bit set on directories. */
-            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
-                throw SysError("setting group of directory '%1%'", tmpDir);
-#endif
-            return tmpDir;
-        }
-        if (errno != EEXIST)
-            throw SysError("creating directory '%1%'", tmpDir);
-    }
-}
-
-
-std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
-{
-    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
-    // Strictly speaking, this is UB, but who cares...
-    // FIXME: use O_TMPFILE.
-    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
-    if (!fd)
-        throw SysError("creating temporary file '%s'", tmpl);
-    closeOnExec(fd.get());
-    return {std::move(fd), tmpl};
-}
-
-
 std::string getUserName()
 {
     auto pw = getpwuid(geteuid());
@@ -577,6 +522,7 @@ Path getHome()
 {
     static Path homeDir = []()
     {
+        std::optional<std::string> unownedUserHomeDir = {};
         auto homeDir = getEnv("HOME");
         if (homeDir) {
             // Only use $HOME if doesn't exist or is owned by the current user.
@@ -588,8 +534,7 @@ Path getHome()
                     homeDir.reset();
                 }
             } else if (st.st_uid != geteuid()) {
-                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir);
+                unownedUserHomeDir.swap(homeDir);
-                homeDir.reset();
             }
         }
         if (!homeDir) {
@@ -600,6 +545,9 @@ Path getHome()
                 || !pw || !pw->pw_dir || !pw->pw_dir[0])
                 throw Error("cannot determine user's home directory");
             homeDir = pw->pw_dir;
+            if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
+                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
+            }
         }
         return *homeDir;
     }();
@@ -681,44 +629,6 @@ Paths createDirs(const Path & path)
 }
 
 
-void createSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime)
-{
-    if (symlink(target.c_str(), link.c_str()))
-        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
-    if (mtime) {
-        struct timeval times[2];
-        times[0].tv_sec = *mtime;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = *mtime;
-        times[1].tv_usec = 0;
-        if (lutimes(link.c_str(), times))
-            throw SysError("setting time of symlink '%s'", link);
-    }
-}
-
-
-void replaceSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime)
-{
-    for (unsigned int n = 0; true; n++) {
-        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
-
-        try {
-            createSymlink(target, tmp, mtime);
-        } catch (SysError & e) {
-            if (e.errNo == EEXIST) continue;
-            throw;
-        }
-
-        if (rename(tmp.c_str(), link.c_str()) != 0)
-            throw SysError("renaming '%1%' to '%2%'", tmp, link);
-
-        break;
-    }
-}
-
-
 void readFull(int fd, char * buf, size_t count)
 {
     while (count) {

src/libutil/util.hh

@@ -168,6 +168,17 @@ void createSymlink(const Path & target, const Path & link,
 void replaceSymlink(const Path & target, const Path & link,
     std::optional<time_t> mtime = {});
 
+void renameFile(const Path & src, const Path & dst);
+
+/**
+ * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
+ * are on a different filesystem.
+ *
+ * Beware that this might not be atomic because of the copy that happens behind
+ * the scenes
+ */
+void moveFile(const Path & src, const Path & dst);
+
 
 /* Wrappers arount read()/write() that read/write exactly the
    requested number of bytes. */

src/nix-build/nix-build.cc

@@ -401,7 +401,7 @@ static void main_nix_build(int argc, char * * argv)
                 auto bashDrv = drv->requireDrvPath();
                 pathsToBuild.push_back(DerivedPath::Built {
                     .drvPath = bashDrv,
-                    .outputs = {},
+                    .outputs = {"out"},
                 });
                 pathsToCopy.insert(bashDrv);
                 shellDrv = bashDrv;

src/nix-env/nix-env.cc

@@ -940,12 +940,12 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
                 JSONObject metaObj = pkgObj.object("meta");
                 StringSet metaNames = i.queryMetaNames();
                 for (auto & j : metaNames) {
-                    auto placeholder = metaObj.placeholder(j);
                     Value * v = i.queryMeta(j);
                     if (!v) {
                         printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
-                        placeholder.write(nullptr);
+                        metaObj.attr(j, nullptr);
                     } else {
+                        auto placeholder = metaObj.placeholder(j);
                         PathSet context;
                         printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context);
                     }

src/nix-instantiate/nix-instantiate.cc

@@ -52,9 +52,10 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 state.autoCallFunction(autoArgs, v, vRes);
             if (output == okXML)
                 printValueAsXML(state, strict, location, vRes, std::cout, context, noPos);
-            else if (output == okJSON)
+            else if (output == okJSON) {
                 printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context);
-            else {
+                std::cout << std::endl;
+            } else {
                 if (strict) state.forceValueDeep(vRes);
                 vRes.print(state.symbols, std::cout);
                 std::cout << std::endl;

src/nix-store/nix-store.cc

@@ -922,7 +922,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
                 if (GET_PROTOCOL_MINOR(clientVersion) >= 3)
                     out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime;
-                if (GET_PROTOCOL_MINOR(clientVersion >= 6)) {
+                if (GET_PROTOCOL_MINOR(clientVersion) >= 6) {
                     worker_proto::write(*store, out, status.builtOutputs);
                 }
 

src/nix/bundle.md

@@ -44,7 +44,7 @@ flake output attributes:
 
 * `bundlers.<system>.default`
 
-If an attribute *name* is given, `nix run` tries the following flake
+If an attribute *name* is given, `nix bundle` tries the following flake
 output attributes:
 
 * `bundlers.<system>.<name>`

src/nix/eval.cc

@@ -116,7 +116,8 @@ struct CmdEval : MixJSON, InstallableCommand
 
         else if (json) {
             JSONPlaceholder jsonOut(std::cout);
-            printValueAsJSON(*state, true, *v, pos, jsonOut, context);
+            printValueAsJSON(*state, true, *v, pos, jsonOut, context, false);
+            std::cout << std::endl;
         }
 
         else {

src/nix/flake.cc

@@ -212,7 +212,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
                     ANSI_BOLD "Last modified:" ANSI_NORMAL " %s",
                     std::put_time(std::localtime(&*lastModified), "%F %T"));
 
-            logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL);
+            if (!lockedFlake.lockFile.root->inputs.empty())
+                logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL);
 
             std::unordered_set<std::shared_ptr<Node>> visited;
 

src/nix/get-env.sh

@@ -43,6 +43,7 @@ __dumpEnv() {
         local __var_name="${BASH_REMATCH[2]}"
 
         if [[ $__var_name =~ ^BASH_ || \
+              $__var_name =~ ^COMP_ || \
               $__var_name = _ || \
               $__var_name = DIRSTACK || \
               $__var_name = EUID || \
@@ -54,7 +55,9 @@ __dumpEnv() {
               $__var_name = PWD || \
               $__var_name = RANDOM || \
               $__var_name = SHLVL || \
-              $__var_name = SECONDS \
+              $__var_name = SECONDS || \
+              $__var_name = EPOCHREALTIME || \
+              $__var_name = EPOCHSECONDS \
             ]]; then continue; fi
 
         if [[ -z $__first ]]; then printf ',\n'; else __first=; fi

src/nix/main.cc

@@ -82,7 +82,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .shortName = 'L',
             .description = "Print full build logs on standard error.",
             .category = loggingCategory,
-            .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }},
+            .handler = {[&]() { logger->setPrintBuildLogs(true); }},
         });
 
         addFlag({

src/nix/profile.md

@@ -11,7 +11,7 @@ them to be rolled back easily.
 
 The default profile used by `nix profile` is `$HOME/.nix-profile`,
 which, if it does not exist, is created as a symlink to
-`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the
+`/nix/var/nix/profiles/default` if Nix is invoked by the
 `root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise.
 
 You can specify another profile location using `--profile` *path*.

tests/check.sh

@@ -40,6 +40,14 @@ nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
 if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi
 checkBuildTempDirRemoved $TEST_ROOT/log
 
+nix build -f check.nix deterministic --rebuild --repeat 1 \
+    --argstr checkBuildId $checkBuildId --keep-failed --no-link \
+    2> $TEST_ROOT/log
+if grep -q 'checking is not possible' $TEST_ROOT/log; then false; fi
+# Repeat is set to 1, ie. nix should build deterministic twice.
+if [ "$(grep "checking outputs" $TEST_ROOT/log | wc -l)" -ne 2 ]; then false; fi
+checkBuildTempDirRemoved $TEST_ROOT/log
+
 nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
     --no-out-link 2> $TEST_ROOT/log
 checkBuildTempDirRemoved $TEST_ROOT/log
@@ -50,6 +58,12 @@ grep 'may not be deterministic' $TEST_ROOT/log
 [ "$status" = "104" ]
 checkBuildTempDirRemoved $TEST_ROOT/log
 
+nix build -f check.nix nondeterministic --rebuild --repeat 1 \
+    --argstr checkBuildId $checkBuildId --keep-failed --no-link \
+    2> $TEST_ROOT/log || status=$?
+grep 'may not be deterministic' $TEST_ROOT/log
+checkBuildTempDirRemoved $TEST_ROOT/log
+
 nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
     --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$?
 grep 'may not be deterministic' $TEST_ROOT/log

tests/common.sh.in

@@ -193,7 +193,7 @@ fi
 onError() {
     set +x
     echo "$0: test failed at:" >&2
-    for ((i = 1; i < 16; i++)); do
+    for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do
         if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi
         echo "  ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2
     done