Merge remote-tracking branch 'origin/master' into lazy-trees

Makefile.config.in

@@ -1,4 +1,3 @@
-HOST_OS = @host_os@
 AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
 BOOST_LDFLAGS = @BOOST_LDFLAGS@
@@ -13,13 +12,14 @@ ENABLE_S3 = @ENABLE_S3@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@
+HOST_OS = @host_os@
 LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
+LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 LOWDOWN_LIBS = @LOWDOWN_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
-LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
 SHELL = @bash@
@@ -31,6 +31,7 @@ datadir = @datadir@
 datarootdir = @datarootdir@
 doc_generate = @doc_generate@
 docdir = @docdir@
+embedded_sandbox_shell = @embedded_sandbox_shell@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
 libdir = @libdir@

configure.ac

@@ -323,6 +323,14 @@ if test ${cross_compiling:-no} = no && ! test -z ${sandbox_shell+x}; then
   fi
 fi
 
+AC_ARG_ENABLE(embedded-sandbox-shell, AS_HELP_STRING([--enable-embedded-sandbox-shell],[include the sandbox shell in the Nix binary [default=no]]),
+  embedded_sandbox_shell=$enableval, embedded_sandbox_shell=no)
+AC_SUBST(embedded_sandbox_shell)
+if test "$embedded_sandbox_shell" = yes; then
+  AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.])
+fi
+
+
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'

flake.nix

@@ -572,14 +572,23 @@
           nativeBuildInputs = nativeBuildDeps;
           buildInputs = buildDeps ++ propagatedDeps;
 
-          configureFlags = [ "--sysconfdir=/etc" ];
+          # Work around pkgsStatic disabling all tests.
+          preHook =
+            ''
+              doCheck=1
+              doInstallCheck=1
+            '';
+
+          configureFlags =
+            configureFlags ++
+            [ "--sysconfdir=/etc"
+              "--enable-embedded-sandbox-shell"
+            ];
 
           enableParallelBuilding = true;
 
           makeFlags = "profiledir=$(out)/etc/profile.d";
 
-          doCheck = true;
-
           installFlags = "sysconfdir=$(out)/etc";
 
           postInstall = ''
@@ -589,7 +598,6 @@
             echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
           '';
 
-          doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
 
           stripAllList = ["bin"];
@@ -598,6 +606,7 @@
 
           hardeningDisable = [ "pie" ];
         };
+
         dockerImage =
           let
             pkgs = nixpkgsFor.${system};

mk/libraries.mk

@@ -125,7 +125,7 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	+$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
+	+$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$^
 	$$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)

src/libstore/build/hook-instance.cc

@@ -7,6 +7,22 @@ HookInstance::HookInstance()
 {
     debug("starting build hook '%s'", settings.buildHook);
 
+    auto buildHookArgs = tokenizeString<std::list<std::string>>(settings.buildHook.get());
+
+    if (buildHookArgs.empty())
+        throw Error("'build-hook' setting is empty");
+
+    auto buildHook = buildHookArgs.front();
+    buildHookArgs.pop_front();
+
+    Strings args;
+
+    for (auto & arg : buildHookArgs)
+        args.push_back(arg);
+
+    args.push_back(std::string(baseNameOf(settings.buildHook.get())));
+    args.push_back(std::to_string(verbosity));
+
     /* Create a pipe to get the output of the child. */
     fromHook.create();
 
@@ -36,14 +52,9 @@ HookInstance::HookInstance()
         if (dup2(builderOut.readSide.get(), 5) == -1)
             throw SysError("dupping builder's stdout/stderr");
 
-        Strings args = {
-            std::string(baseNameOf(settings.buildHook.get())),
-            std::to_string(verbosity),
-        };
+        execv(buildHook.c_str(), stringsToCharPtrs(args).data());
 
-        execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("executing '%s'", settings.buildHook);
+        throw SysError("executing '%s'", buildHook);
     });
 
     pid.setSeparatePG(true);

src/libstore/build/local-derivation-goal.cc

@@ -1717,7 +1717,19 @@ void LocalDerivationGoal::runChild()
 
             for (auto & i : dirsInChroot) {
                 if (i.second.source == "/proc") continue; // backwards compatibility
-                doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
+
+                #if HAVE_EMBEDDED_SANDBOX_SHELL
+                if (i.second.source == "__embedded_sandbox_shell__") {
+                    static unsigned char sh[] = {
+                        #include "embedded-sandbox-shell.gen.hh"
+                    };
+                    auto dst = chrootRootDir + i.first;
+                    createDirs(dirOf(dst));
+                    writeFile(dst, std::string_view((const char *) sh, sizeof(sh)));
+                    chmod_(dst, 0555);
+                } else
+                #endif
+                    doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
             }
 
             /* Bind a new instance of procfs on /proc. */

src/libstore/build/substitution-goal.cc

@@ -154,7 +154,7 @@ void PathSubstitutionGoal::tryNext()
        only after we've downloaded the path. */
     if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
     {
-        warn("the substitute for '%s' from '%s' is not signed by any of the keys in 'trusted-public-keys'",
+        warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'",
             worker.store.printStorePath(storePath), sub->getUri());
         tryNext();
         return;

src/libstore/globals.cc

@@ -36,7 +36,6 @@ Settings::Settings()
     , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
     , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
     , nixUserConfFiles(getUserConfigFiles())
-    , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
     , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
@@ -67,12 +66,13 @@ Settings::Settings()
     sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
 #endif
 
-
-/* chroot-like behavior from Apple's sandbox */
+    /* chroot-like behavior from Apple's sandbox */
 #if __APPLE__
     sandboxPaths = tokenizeString<StringSet>("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib");
     allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
 #endif
+
+    buildHook = getSelfExe().value_or("nix") + " __build-remote";
 }
 
 void loadConfFile()

src/libstore/globals.hh

@@ -79,9 +79,6 @@ public:
     /* A list of user configuration files to load. */
     std::vector<Path> nixUserConfFiles;
 
-    /* The directory where internal helper programs are stored. */
-    Path nixLibexecDir;
-
     /* The directory where the main programs are stored. */
     Path nixBinDir;
 
@@ -195,7 +192,7 @@ public:
         )",
         {"build-timeout"}};
 
-    PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook",
+    PathSetting buildHook{this, true, "", "build-hook",
         "The path of the helper program that executes builds to remote machines."};
 
     Setting<std::string> builders{

src/libstore/local.mk

@@ -39,14 +39,23 @@ libstore_CXXFLAGS += \
  -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \
  -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \
  -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
- -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
  -DNIX_MAN_DIR=\"$(mandir)\" \
  -DLSOF=\"$(lsof)\"
 
+ifeq ($(embedded_sandbox_shell),yes)
+libstore_CXXFLAGS += -DSANDBOX_SHELL=\"__embedded_sandbox_shell__\"
+
+$(d)/build/local-derivation-goal.cc: $(d)/embedded-sandbox-shell.gen.hh
+
+$(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell)
+	$(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp
+	@mv $@.tmp $@
+else
 ifneq ($(sandbox_shell),)
 libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
 endif
+endif
 
 $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 

src/libutil/util.cc

@@ -29,6 +29,7 @@
 
 #ifdef __APPLE__
 #include <sys/syscall.h>
+#include <mach-o/dyld.h>
 #endif
 
 #ifdef __linux__
@@ -574,6 +575,20 @@ Path getHome()
     static Path homeDir = []()
     {
         auto homeDir = getEnv("HOME");
+        if (homeDir) {
+            // Only use $HOME if doesn't exist or is owned by the current user.
+            struct stat st;
+            int result = stat(homeDir->c_str(), &st);
+            if (result != 0) {
+                if (errno != ENOENT) {
+                    warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno);
+                    homeDir.reset();
+                }
+            } else if (st.st_uid != geteuid()) {
+                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir);
+                homeDir.reset();
+            }
+        }
         if (!homeDir) {
             std::vector<char> buf(16384);
             struct passwd pwbuf;
@@ -619,6 +634,27 @@ Path getDataDir()
 }
 
 
+std::optional<Path> getSelfExe()
+{
+    static auto cached = []() -> std::optional<Path>
+    {
+        #if __linux__
+        return readLink("/proc/self/exe");
+        #elif __APPLE__
+        char buf[1024];
+        uint32_t size = sizeof(buf);
+        if (_NSGetExecutablePath(buf, &size) == 0)
+            return buf;
+        else
+            return std::nullopt;
+        #else
+        return std::nullopt;
+        #endif
+    }();
+    return cached;
+}
+
+
 Paths createDirs(const Path & path)
 {
     Paths created;

src/libutil/util.hh

@@ -149,10 +149,14 @@ std::vector<Path> getConfigDirs();
 /* Return $XDG_DATA_HOME or $HOME/.local/share. */
 Path getDataDir();
 
+/* Return the path of the current executable. */
+std::optional<Path> getSelfExe();
+
 /* Create a directory and all its parents, if necessary.  Returns the
    list of created directories, in order of creation. */
 Paths createDirs(const Path & path);
-inline Paths createDirs(PathView path) {
+inline Paths createDirs(PathView path)
+{
     return createDirs(Path(path));
 }
 

src/nix/main.cc

@@ -263,6 +263,11 @@ void mainWrapped(int argc, char * * argv)
     programPath = argv[0];
     auto programName = std::string(baseNameOf(programPath));
 
+    if (argc > 0 && std::string_view(argv[0]) == "__build-remote") {
+        programName = "build-remote";
+        argv++; argc--;
+    }
+
     {
         auto legacy = (*RegisterLegacyCommand::commands)[programName];
         if (legacy) return legacy(argc, argv);

src/nix/run.cc

@@ -47,7 +47,7 @@ void runProgramInStore(ref<Store> store,
         Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program };
         for (auto & arg : args) helperArgs.push_back(arg);
 
-        execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
+        execv(getSelfExe().value_or("nix").c_str(), stringsToCharPtrs(helperArgs).data());
 
         throw SysError("could not execute chroot helper");
     }

src/nix/search.cc

@@ -34,7 +34,9 @@ struct CmdSearch : InstallableCommand, MixJSON
             .shortName = 'e',
             .description = "Hide packages whose attribute path, name or description contain *regex*.",
             .labels = {"regex"},
-            .handler = Handler(&excludeRes),
+            .handler = {[this](std::string s) {
+                excludeRes.push_back(s);
+            }},
         });
     }
 

tests/ca/content-addressed.nix

@@ -75,7 +75,7 @@ rec {
     buildCommand = ''
       mkdir -p $out/bin
       echo ${rootCA} # Just to make it depend on it
-      echo "" > $out/bin/${name}
+      echo "#! ${shell}" > $out/bin/${name}
       chmod +x $out/bin/${name}
     '';
   };

tests/common.sh.in

@@ -50,6 +50,8 @@ export busybox="@sandbox_shell@"
 export version=@PACKAGE_VERSION@
 export system=@system@
 
+export BUILD_SHARED_LIBS=@BUILD_SHARED_LIBS@
+
 export IMPURE_VAR1=foo
 export IMPURE_VAR2=bar
 

tests/fmt.sh

@@ -18,7 +18,12 @@ cat << EOF > flake.nix
       with import ./config.nix;
       mkDerivation {
         name = "formatter";
-        buildCommand = "mkdir -p \$out/bin; cp \${./fmt.simple.sh} \$out/bin/formatter";
+        buildCommand = ''
+          mkdir -p \$out/bin
+          echo "#! ${shell}" > \$out/bin/formatter
+          cat \${./fmt.simple.sh} >> \$out/bin/formatter
+          chmod +x \$out/bin/formatter
+        '';
       };
   };
 }

tests/local.mk

@@ -114,4 +114,8 @@ tests-environment = NIX_REMOTE= $(bash) -e
 
 clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix
 
-test-deps += tests/common.sh tests/config.nix tests/ca/config.nix tests/plugins/libplugintest.$(SO_EXT)
+test-deps += tests/common.sh tests/config.nix tests/ca/config.nix
+
+ifeq ($(BUILD_SHARED_LIBS), 1)
+  test-deps += tests/plugins/libplugintest.$(SO_EXT)
+endif

tests/plugins.sh

@@ -2,6 +2,11 @@ source common.sh
 
 set -o pipefail
 
+if [[ $BUILD_SHARED_LIBS != 1 ]]; then
+    echo "plugins are not supported"
+    exit 99
+fi
+
 res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull)
 
 [ "$res"x = "nullx" ]

tests/search.sh

@@ -43,3 +43,4 @@ e=$'\x1b' # grep doesn't support \e, \033 or even \x1b
 
 (( $(nix search -f search.nix foo --exclude 'foo|bar' | grep -Ec 'foo|bar') == 0 ))
 (( $(nix search -f search.nix foo -e foo --exclude bar | grep -Ec 'foo|bar') == 0 ))
+[[ $(nix search -f search.nix -e bar --json | jq -c 'keys') == '["foo","hello"]' ]]