nix-super/src/libutil/util.cc

1958 lines
49 KiB
C++
Raw Normal View History

#include "util.hh"
#include "sync.hh"
#include "finally.hh"
#include "serialise.hh"
2021-09-08 13:20:08 +03:00
#include <array>
#include <cctype>
2003-09-11 11:31:29 +03:00
#include <cerrno>
#include <climits>
2003-09-11 11:31:29 +03:00
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <future>
#include <iostream>
#include <mutex>
#include <sstream>
#include <thread>
#include <fcntl.h>
2019-05-11 23:35:53 +03:00
#include <grp.h>
#include <pwd.h>
2017-08-25 16:57:49 +03:00
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/wait.h>
2019-05-28 23:35:41 +03:00
#include <sys/time.h>
#include <sys/un.h>
#include <unistd.h>
2006-09-28 00:04:07 +03:00
#ifdef __APPLE__
#include <sys/syscall.h>
2022-06-23 02:32:17 +03:00
#include <mach-o/dyld.h>
#endif
#ifdef __linux__
#include <sys/prctl.h>
#include <sys/resource.h>
#include <mntent.h>
#include <cmath>
#endif
2003-05-26 16:45:00 +03:00
extern char * * environ __attribute__((weak));
2006-12-07 18:40:41 +02:00
namespace nix {
std::optional<std::string> getEnv(const std::string & key)
{
char * value = getenv(key.c_str());
if (!value) return {};
return std::string(value);
}
std::map<std::string, std::string> getEnv()
{
std::map<std::string, std::string> env;
for (size_t i = 0; environ[i]; ++i) {
auto s = environ[i];
auto eq = strchr(s, '=');
if (!eq)
// invalid env, just keep going
continue;
env.emplace(std::string(s, eq), std::string(eq + 1));
}
return env;
}
void clearEnv()
{
for (auto & name : getEnv())
unsetenv(name.first.c_str());
}
2022-03-31 11:39:53 +03:00
void replaceEnv(const std::map<std::string, std::string> & newEnv)
Add a post-build-hook Passing `--post-build-hook /foo/bar` to a nix-* command will cause `/foo/bar` to be executed after each build with the following environment variables set: DRV_PATH=/nix/store/drv-that-has-been-built.drv OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev This can be useful in particular to upload all the builded artifacts to the cache (including the ones that don't appear in the runtime closure of the final derivation or are built because of IFD). This new feature prints the stderr/stdout output to the `nix-build` and `nix build` client, and the output is printed in a Nix 2 compatible format: [nix]$ ./inst/bin/nix-build ./test.nix these derivations will be built: /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'... hello! bye! running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... post-build-hook: + sleep 1 post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + printf 'very important stuff' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix my-example-derivation> hello! my-example-derivation> bye! my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + printf 'very important stuff' [1 built, 0.0 MiB DL] Co-authored-by: Graham Christensen <graham@grahamc.com> Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
2019-07-11 21:23:03 +03:00
{
clearEnv();
2022-03-31 11:39:53 +03:00
for (auto & newEnvVar : newEnv)
Add a post-build-hook Passing `--post-build-hook /foo/bar` to a nix-* command will cause `/foo/bar` to be executed after each build with the following environment variables set: DRV_PATH=/nix/store/drv-that-has-been-built.drv OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev This can be useful in particular to upload all the builded artifacts to the cache (including the ones that don't appear in the runtime closure of the final derivation or are built because of IFD). This new feature prints the stderr/stdout output to the `nix-build` and `nix build` client, and the output is printed in a Nix 2 compatible format: [nix]$ ./inst/bin/nix-build ./test.nix these derivations will be built: /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'... hello! bye! running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... post-build-hook: + sleep 1 post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + printf 'very important stuff' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix my-example-derivation> hello! my-example-derivation> bye! my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + printf 'very important stuff' [1 built, 0.0 MiB DL] Co-authored-by: Graham Christensen <graham@grahamc.com> Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
2019-07-11 21:23:03 +03:00
setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
}
Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
2003-05-26 16:45:00 +03:00
{
if (path[0] != '/') {
if (!dir) {
2010-02-10 17:55:50 +02:00
#ifdef __GNU__
/* GNU (aka. GNU/Hurd) doesn't have any limitation on path
lengths and doesn't define `PATH_MAX'. */
char *buf = getcwd(NULL, 0);
if (buf == NULL)
#else
2003-05-26 16:45:00 +03:00
char buf[PATH_MAX];
if (!getcwd(buf, sizeof(buf)))
2010-02-10 17:55:50 +02:00
#endif
throw SysError("cannot get cwd");
path = concatStrings(buf, "/", path);
2010-02-10 17:55:50 +02:00
#ifdef __GNU__
free(buf);
#endif
} else
path = concatStrings(*dir, "/", path);
2003-05-26 16:45:00 +03:00
}
return canonPath(path, resolveSymlinks);
}
Path canonPath(PathView path, bool resolveSymlinks)
{
2017-04-13 16:32:43 +03:00
assert(path != "");
std::string s;
s.reserve(256);
if (path[0] != '/')
throw Error("not an absolute path: '%1%'", path);
std::string temp;
/* Count the number of times we follow a symlink and stop at some
arbitrary (but high) limit to prevent infinite loops. */
unsigned int followCount = 0, maxFollow = 1024;
while (1) {
/* Skip slashes. */
while (!path.empty() && path[0] == '/') path.remove_prefix(1);
if (path.empty()) break;
/* Ignore `.'. */
if (path == "." || path.substr(0, 2) == "./")
path.remove_prefix(1);
/* If `..', delete the last component. */
else if (path == ".." || path.substr(0, 3) == "../")
{
if (!s.empty()) s.erase(s.rfind('/'));
path.remove_prefix(2);
}
/* Normal component; copy it. */
else {
s += '/';
if (const auto slash = path.find('/'); slash == std::string::npos) {
s += path;
path = {};
} else {
s += path.substr(0, slash);
path = path.substr(slash);
}
2021-03-31 05:20:41 +03:00
/* If s points to a symlink, resolve it and continue from there */
if (resolveSymlinks && isLink(s)) {
if (++followCount >= maxFollow)
throw Error("infinite symlink recursion in path '%1%'", path);
temp = concatStrings(readLink(s), path);
path = temp;
2021-03-31 05:20:41 +03:00
if (!temp.empty() && temp[0] == '/') {
s.clear(); /* restart for symlinks pointing to absolute path */
} else {
s = dirOf(s);
if (s == "/") { // we dont want trailing slashes here, which dirOf only produces if s = /
s.clear();
}
2021-03-31 05:20:41 +03:00
}
}
}
}
return s.empty() ? "/" : std::move(s);
}
Path dirOf(const PathView path)
{
Path::size_type pos = path.rfind('/');
if (pos == std::string::npos)
2018-08-13 12:27:35 +03:00
return ".";
return pos == 0 ? "/" : Path(path, 0, pos);
2003-05-26 16:45:00 +03:00
}
std::string_view baseNameOf(std::string_view path)
2003-05-26 16:45:00 +03:00
{
if (path.empty())
2016-01-27 18:18:31 +02:00
return "";
auto last = path.size() - 1;
if (path[last] == '/' && last > 0)
last -= 1;
auto pos = path.rfind('/', last);
if (pos == std::string::npos)
pos = 0;
else
pos += 1;
2016-01-27 18:18:31 +02:00
return path.substr(pos, last - pos + 1);
2003-05-26 16:45:00 +03:00
}
std::string expandTilde(std::string_view path)
{
// TODO: expand ~user ?
auto tilde = path.substr(0, 2);
if (tilde == "~/" || tilde == "~")
return getHome() + std::string(path.substr(1));
else
return std::string(path);
}
bool isInDir(std::string_view path, std::string_view dir)
{
return path.substr(0, 1) == "/"
&& path.substr(0, dir.size()) == dir
2015-10-22 00:40:35 +03:00
&& path.size() >= dir.size() + 2
&& path[dir.size()] == '/';
}
bool isDirOrInDir(std::string_view path, std::string_view dir)
{
return path == dir || isInDir(path, dir);
}
struct stat stat(const Path & path)
{
struct stat st;
if (stat(path.c_str(), &st))
throw SysError("getting status of '%1%'", path);
return st;
}
struct stat lstat(const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError("getting status of '%1%'", path);
return st;
}
2003-10-07 17:37:41 +03:00
bool pathExists(const Path & path)
{
int res;
struct stat st;
res = lstat(path.c_str(), &st);
if (!res) return true;
if (errno != ENOENT && errno != ENOTDIR)
throw SysError("getting status of %1%", path);
return false;
}
Path readLink(const Path & path)
{
checkInterrupt();
std::vector<char> buf;
for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) {
buf.resize(bufSize);
ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize);
if (rlSize == -1)
if (errno == EINVAL)
2017-11-20 18:32:58 +02:00
throw Error("'%1%' is not a symlink", path);
else
2017-11-20 18:32:58 +02:00
throw SysError("reading symbolic link '%1%'", path);
else if (rlSize < bufSize)
return std::string(buf.data(), rlSize);
}
}
bool isLink(const Path & path)
{
struct stat st = lstat(path);
return S_ISLNK(st.st_mode);
}
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
DirEntries readDirectory(DIR *dir, const Path & path)
{
DirEntries entries;
entries.reserve(64);
struct dirent * dirent;
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
while (errno = 0, dirent = readdir(dir)) { /* sic */
checkInterrupt();
std::string name = dirent->d_name;
if (name == "." || name == "..") continue;
2016-01-05 15:05:11 +02:00
entries.emplace_back(name, dirent->d_ino,
#ifdef HAVE_STRUCT_DIRENT_D_TYPE
dirent->d_type
#else
DT_UNKNOWN
#endif
);
}
if (errno) throw SysError("reading directory '%1%'", path);
return entries;
}
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
DirEntries readDirectory(const Path & path)
{
AutoCloseDir dir(opendir(path.c_str()));
2020-05-12 00:52:15 +03:00
if (!dir) throw SysError("opening directory '%1%'", path);
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
return readDirectory(dir.get(), path);
}
2014-10-03 23:37:51 +03:00
unsigned char getFileType(const Path & path)
{
struct stat st = lstat(path);
if (S_ISDIR(st.st_mode)) return DT_DIR;
if (S_ISLNK(st.st_mode)) return DT_LNK;
if (S_ISREG(st.st_mode)) return DT_REG;
return DT_UNKNOWN;
}
std::string readFile(int fd)
{
struct stat st;
if (fstat(fd, &st) == -1)
throw SysError("statting file");
return drainFD(fd, true, st.st_size);
}
std::string readFile(const Path & path)
{
2016-06-09 17:15:58 +03:00
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
2016-07-11 22:44:44 +03:00
if (!fd)
2020-05-12 00:52:15 +03:00
throw SysError("opening file '%1%'", path);
return readFile(fd.get());
}
void readFile(const Path & path, Sink & sink)
{
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
2020-06-15 15:12:39 +03:00
if (!fd)
2020-05-06 23:07:20 +03:00
throw SysError("opening file '%s'", path);
drainFD(fd.get(), sink);
}
void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync)
{
AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
2016-07-11 22:44:44 +03:00
if (!fd)
throw SysError("opening file '%1%'", path);
try {
writeFull(fd.get(), s);
} catch (Error & e) {
e.addTrace({}, "writing file '%1%'", path);
throw;
}
if (sync)
fd.fsync();
// Explicitly close to make sure exceptions are propagated.
fd.close();
if (sync)
syncParent(path);
}
void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
{
AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
if (!fd)
throw SysError("opening file '%1%'", path);
std::vector<char> buf(64 * 1024);
try {
while (true) {
try {
auto n = source.read(buf.data(), buf.size());
writeFull(fd.get(), {buf.data(), n});
} catch (EndOfFile &) { break; }
}
} catch (Error & e) {
e.addTrace({}, "writing file '%1%'", path);
throw;
}
if (sync)
fd.fsync();
// Explicitly close to make sure exceptions are propagated.
fd.close();
if (sync)
syncParent(path);
}
void syncParent(const Path & path)
{
AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0);
if (!fd)
throw SysError("opening file '%1%'", path);
fd.fsync();
}
std::string readLine(int fd)
{
std::string s;
while (1) {
checkInterrupt();
char ch;
// FIXME: inefficient
ssize_t rd = read(fd, &ch, 1);
if (rd == -1) {
if (errno != EINTR)
throw SysError("reading a line");
} else if (rd == 0)
throw EndOfFile("unexpected EOF reading a line");
else {
if (ch == '\n') return s;
s += ch;
}
}
}
void writeLine(int fd, std::string s)
{
s += '\n';
writeFull(fd, s);
}
2020-07-30 14:10:49 +03:00
static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
2003-06-23 17:40:49 +03:00
{
checkInterrupt();
std::string name(baseNameOf(path));
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
struct stat st;
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
if (errno == ENOENT) return;
throw SysError("getting status of '%1%'", path);
}
2003-06-23 17:40:49 +03:00
if (!S_ISDIR(st.st_mode)) {
/* We are about to delete a file. Will it likely free space? */
switch (st.st_nlink) {
/* Yes: last link. */
case 1:
bytesFreed += st.st_size;
break;
/* Maybe: yes, if 'auto-optimise-store' or manual optimisation
was performed. Instead of checking for real let's assume
it's an optimised file and space will be freed.
In worst case we will double count on freed space for files
with exactly two hardlinks for unoptimised packages.
*/
case 2:
bytesFreed += st.st_size;
break;
/* No: 3+ links. */
default:
break;
}
}
2003-06-23 17:40:49 +03:00
if (S_ISDIR(st.st_mode)) {
/* Make the directory accessible. */
const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
if ((st.st_mode & PERM_MASK) != PERM_MASK) {
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
2020-05-12 00:52:15 +03:00
throw SysError("chmod '%1%'", path);
2013-01-03 14:00:46 +02:00
}
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
int fd = openat(parentfd, path.c_str(), O_RDONLY);
if (fd == -1)
2020-05-12 00:52:15 +03:00
throw SysError("opening directory '%1%'", path);
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
AutoCloseDir dir(fdopendir(fd));
if (!dir)
2020-05-12 00:52:15 +03:00
throw SysError("opening directory '%1%'", path);
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
for (auto & i : readDirectory(dir.get(), path))
_deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed);
2003-06-23 17:40:49 +03:00
}
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
if (unlinkat(parentfd, name.c_str(), flags) == -1) {
if (errno == ENOENT) return;
throw SysError("cannot unlink '%1%'", path);
}
}
2020-07-30 14:10:49 +03:00
static void _deletePath(const Path & path, uint64_t & bytesFreed)
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
{
Path dir = dirOf(path);
if (dir == "")
dir = "/";
AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)};
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
if (!dirfd) {
if (errno == ENOENT) return;
2020-05-12 00:52:15 +03:00
throw SysError("opening directory '%1%'", path);
Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross <hi@alyssa.is> Thanks-to: Puck Meerburg <puck@puckipedia.com> Tested-by: Puck Meerburg <puck@puckipedia.com> Reviewed-by: Puck Meerburg <puck@puckipedia.com>
2020-04-27 17:15:15 +03:00
}
_deletePath(dirfd.get(), path, bytesFreed);
}
2004-03-22 23:42:28 +02:00
void deletePath(const Path & path)
{
2020-07-30 14:10:49 +03:00
uint64_t dummy;
deletePath(path, dummy);
}
2020-07-30 14:10:49 +03:00
void deletePath(const Path & path, uint64_t & bytesFreed)
2004-03-22 23:42:28 +02:00
{
//Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path);
bytesFreed = 0;
_deletePath(path, bytesFreed);
2004-03-22 23:42:28 +02:00
}
std::string getUserName()
{
auto pw = getpwuid(geteuid());
std::string name = pw ? pw->pw_name : getEnv("USER").value_or("");
if (name.empty())
throw Error("cannot figure out user name");
return name;
}
Path getHome()
{
static Path homeDir = []()
{
std::optional<std::string> unownedUserHomeDir = {};
auto homeDir = getEnv("HOME");
if (homeDir) {
// Only use $HOME if doesn't exist or is owned by the current user.
struct stat st;
int result = stat(homeDir->c_str(), &st);
if (result != 0) {
if (errno != ENOENT) {
2022-06-22 16:35:52 +03:00
warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno);
homeDir.reset();
}
} else if (st.st_uid != geteuid()) {
unownedUserHomeDir.swap(homeDir);
}
}
if (!homeDir) {
std::vector<char> buf(16384);
struct passwd pwbuf;
struct passwd * pw;
if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0
|| !pw || !pw->pw_dir || !pw->pw_dir[0])
throw Error("cannot determine user's home directory");
homeDir = pw->pw_dir;
if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
}
}
return *homeDir;
}();
return homeDir;
}
Path getCacheDir()
{
auto cacheDir = getEnv("XDG_CACHE_HOME");
return cacheDir ? *cacheDir : getHome() + "/.cache";
}
Path getConfigDir()
{
auto configDir = getEnv("XDG_CONFIG_HOME");
return configDir ? *configDir : getHome() + "/.config";
}
std::vector<Path> getConfigDirs()
{
Path configHome = getConfigDir();
auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg");
std::vector<Path> result = tokenizeString<std::vector<std::string>>(configDirs, ":");
result.insert(result.begin(), configHome);
return result;
}
Path getDataDir()
{
auto dataDir = getEnv("XDG_DATA_HOME");
return dataDir ? *dataDir : getHome() + "/.local/share";
}
std::optional<Path> getSelfExe()
{
2022-06-23 02:32:17 +03:00
static auto cached = []() -> std::optional<Path>
{
#if __linux__
return readLink("/proc/self/exe");
2022-06-23 02:32:17 +03:00
#elif __APPLE__
char buf[1024];
uint32_t size = sizeof(buf);
if (_NSGetExecutablePath(buf, &size) == 0)
return buf;
else
return std::nullopt;
#else
return std::nullopt;
#endif
}();
return cached;
}
Paths createDirs(const Path & path)
{
Paths created;
if (path == "/") return created;
struct stat st;
if (lstat(path.c_str(), &st) == -1) {
created = createDirs(dirOf(path));
if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST)
throw SysError("creating directory '%1%'", path);
st = lstat(path);
created.push_back(path);
}
if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1)
throw SysError("statting symlink '%1%'", path);
if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path);
2013-01-03 14:00:46 +02:00
return created;
}
void readFull(int fd, char * buf, size_t count)
{
while (count) {
checkInterrupt();
ssize_t res = read(fd, buf, count);
if (res == -1) {
if (errno == EINTR) continue;
throw SysError("reading from file");
}
if (res == 0) throw EndOfFile("unexpected end-of-file");
count -= res;
buf += res;
}
}
2020-12-02 15:00:43 +02:00
void writeFull(int fd, std::string_view s, bool allowInterrupts)
{
2020-12-02 15:00:43 +02:00
while (!s.empty()) {
if (allowInterrupts) checkInterrupt();
2020-12-02 15:00:43 +02:00
ssize_t res = write(fd, s.data(), s.size());
if (res == -1 && errno != EINTR)
throw SysError("writing to file");
2020-12-02 15:00:43 +02:00
if (res > 0)
s.remove_prefix(res);
}
}
std::string drainFD(int fd, bool block, const size_t reserveSize)
{
avoid copies of parser input data when given a string yacc will copy the entire input to a newly allocated location so that it can add a second terminating NUL byte. since the parser is a very internal thing to EvalState we can ensure that having two terminating NUL bytes is always possible without copying, and have the parser itself merely check that the expected NULs are present. # before Benchmark 1: nix search --offline nixpkgs hello Time (mean ± σ): 572.4 ms ± 2.3 ms [User: 563.4 ms, System: 8.6 ms] Range (min … max): 566.9 ms … 579.1 ms 50 runs Benchmark 2: nix eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix Time (mean ± σ): 381.7 ms ± 1.0 ms [User: 348.3 ms, System: 33.1 ms] Range (min … max): 380.2 ms … 387.7 ms 50 runs Benchmark 3: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system' Time (mean ± σ): 2.936 s ± 0.005 s [User: 2.715 s, System: 0.221 s] Range (min … max): 2.923 s … 2.946 s 50 runs # after Benchmark 1: nix search --offline nixpkgs hello Time (mean ± σ): 571.7 ms ± 2.4 ms [User: 563.3 ms, System: 8.0 ms] Range (min … max): 566.7 ms … 579.7 ms 50 runs Benchmark 2: nix eval -f ../nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix Time (mean ± σ): 376.6 ms ± 1.0 ms [User: 345.8 ms, System: 30.5 ms] Range (min … max): 374.5 ms … 379.1 ms 50 runs Benchmark 3: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system' Time (mean ± σ): 2.922 s ± 0.006 s [User: 2.707 s, System: 0.215 s] Range (min … max): 2.906 s … 2.934 s 50 runs
2021-12-21 14:56:57 +02:00
// the parser needs two extra bytes to append terminating characters, other users will
// not care very much about the extra memory.
StringSink sink(reserveSize + 2);
drainFD(fd, sink, block);
return std::move(sink.s);
}
void drainFD(int fd, Sink & sink, bool block)
{
// silence GCC maybe-uninitialized warning in finally
int saved = 0;
if (!block) {
saved = fcntl(fd, F_GETFL);
if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
throw SysError("making file descriptor non-blocking");
}
Finally finally([&]() {
if (!block) {
if (fcntl(fd, F_SETFL, saved) == -1)
throw SysError("making file descriptor blocking");
}
});
std::vector<unsigned char> buf(64 * 1024);
while (1) {
checkInterrupt();
ssize_t rd = read(fd, buf.data(), buf.size());
if (rd == -1) {
if (!block && (errno == EAGAIN || errno == EWOULDBLOCK))
break;
if (errno != EINTR)
throw SysError("reading from file");
}
else if (rd == 0) break;
2020-12-02 15:00:43 +02:00
else sink({(char *) buf.data(), (size_t) rd});
}
}
//////////////////////////////////////////////////////////////////////
unsigned int getMaxCPU()
{
#if __linux__
try {
FILE *fp = fopen("/proc/mounts", "r");
if (!fp)
return 0;
Strings cgPathParts;
struct mntent *ent;
while ((ent = getmntent(fp))) {
std::string mountType, mountPath;
mountType = ent->mnt_type;
mountPath = ent->mnt_dir;
if (mountType == "cgroup2") {
cgPathParts.push_back(mountPath);
break;
}
}
fclose(fp);
if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) {
std::string currentCgroup = readFile("/proc/self/cgroup");
Strings cgValues = tokenizeString<Strings>(currentCgroup, ":");
cgPathParts.push_back(trim(cgValues.back(), "\n"));
cgPathParts.push_back("cpu.max");
std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts));
if (pathExists(fullCgPath)) {
std::string cpuMax = readFile(fullCgPath);
std::vector<std::string> cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " ");
std::string quota = cpuMaxParts[0];
std::string period = trim(cpuMaxParts[1], "\n");
if (quota != "max")
return std::ceil(std::stoi(quota) / std::stof(period));
}
}
} catch (Error &) { ignoreException(); }
#endif
return 0;
}
2004-06-22 12:51:44 +03:00
//////////////////////////////////////////////////////////////////////
AutoDelete::AutoDelete() : del{false} {}
AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p)
{
del = true;
this->recursive = recursive;
}
AutoDelete::~AutoDelete()
{
try {
if (del) {
if (recursive)
deletePath(path);
else {
if (remove(path.c_str()) == -1)
throw SysError("cannot unlink '%1%'", path);
}
}
} catch (...) {
ignoreException();
}
}
void AutoDelete::cancel()
{
del = false;
}
void AutoDelete::reset(const Path & p, bool recursive) {
2015-11-16 12:54:34 +02:00
path = p;
this->recursive = recursive;
del = true;
}
2004-06-22 12:51:44 +03:00
//////////////////////////////////////////////////////////////////////
2016-07-11 22:44:44 +03:00
AutoCloseFD::AutoCloseFD() : fd{-1} {}
AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
2004-06-22 12:51:44 +03:00
AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
{
2016-07-11 22:44:44 +03:00
that.fd = -1;
}
2004-06-22 12:51:44 +03:00
AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
{
2016-07-11 22:44:44 +03:00
close();
fd = that.fd;
that.fd = -1;
return *this;
}
AutoCloseFD::~AutoCloseFD()
{
2004-06-15 16:49:42 +03:00
try {
close();
} catch (...) {
ignoreException();
2004-06-15 16:49:42 +03:00
}
}
2004-06-22 12:51:44 +03:00
2016-07-11 22:44:44 +03:00
int AutoCloseFD::get() const
{
return fd;
}
2004-06-22 12:51:44 +03:00
2004-06-15 16:49:42 +03:00
void AutoCloseFD::close()
{
if (fd != -1) {
if (::close(fd) == -1)
/* This should never happen. */
throw SysError("closing file descriptor %1%", fd);
fd = -1;
2004-06-15 16:49:42 +03:00
}
}
void AutoCloseFD::fsync()
{
if (fd != -1) {
int result;
#if __APPLE__
result = ::fcntl(fd, F_FULLFSYNC);
#else
result = ::fsync(fd);
#endif
if (result == -1)
throw SysError("fsync file descriptor %1%", fd);
}
}
2004-06-22 12:51:44 +03:00
2016-07-11 22:44:44 +03:00
AutoCloseFD::operator bool() const
2004-06-15 16:49:42 +03:00
{
return fd != -1;
}
2016-07-11 22:44:44 +03:00
int AutoCloseFD::release()
{
int oldFD = fd;
fd = -1;
return oldFD;
}
2004-06-15 16:49:42 +03:00
void Pipe::create()
{
int fds[2];
2016-06-09 17:15:58 +03:00
#if HAVE_PIPE2
if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe");
#else
2004-06-15 16:49:42 +03:00
if (pipe(fds) != 0) throw SysError("creating pipe");
2016-06-09 17:15:58 +03:00
closeOnExec(fds[0]);
closeOnExec(fds[1]);
#endif
2004-06-15 16:49:42 +03:00
readSide = fds[0];
writeSide = fds[1];
}
void Pipe::close()
{
readSide.close();
writeSide.close();
}
2004-06-22 12:51:44 +03:00
//////////////////////////////////////////////////////////////////////
Pid::Pid()
{
}
Pid::Pid(pid_t pid)
: pid(pid)
{
}
2004-06-22 12:51:44 +03:00
Pid::~Pid()
{
if (pid != -1) kill();
2004-06-22 12:51:44 +03:00
}
void Pid::operator =(pid_t pid)
{
if (this->pid != -1 && this->pid != pid) kill();
2004-06-22 12:51:44 +03:00
this->pid = pid;
killSignal = SIGKILL; // reset signal to default
2004-06-22 12:51:44 +03:00
}
Pid::operator pid_t()
{
return pid;
}
int Pid::kill()
2004-06-22 12:51:44 +03:00
{
assert(pid != -1);
2020-05-12 00:52:15 +03:00
debug("killing process %1%", pid);
2004-06-22 12:51:44 +03:00
/* Send the requested signal to the child. If it has its own
process group, send the signal to every process in the child
process group (which hopefully includes *all* its children). */
if (::kill(separatePG ? -pid : pid, killSignal) != 0) {
/* On BSDs, killing a process group will return EPERM if all
processes in the group are zombies (or something like
that). So try to detect and ignore that situation. */
#if __FreeBSD__ || __APPLE__
if (errno != EPERM || ::kill(pid, 0) != 0)
#endif
2020-05-13 18:52:36 +03:00
logError(SysError("killing process %d", pid).info());
}
return wait();
2004-06-22 12:51:44 +03:00
}
int Pid::wait()
2004-06-22 12:51:44 +03:00
{
assert(pid != -1);
2004-06-22 12:51:44 +03:00
while (1) {
int status;
int res = waitpid(pid, &status, 0);
2004-06-22 12:51:44 +03:00
if (res == pid) {
pid = -1;
return status;
}
if (errno != EINTR)
2021-10-05 13:26:04 +03:00
throw SysError("cannot get exit status of PID %d", pid);
checkInterrupt();
2004-06-22 12:51:44 +03:00
}
}
void Pid::setSeparatePG(bool separatePG)
{
this->separatePG = separatePG;
}
void Pid::setKillSignal(int signal)
{
this->killSignal = signal;
}
2016-10-12 16:49:37 +03:00
pid_t Pid::release()
{
pid_t p = pid;
pid = -1;
return p;
}
void killUser(uid_t uid)
{
2020-05-12 00:52:15 +03:00
debug("killing all processes running under uid '%1%'", uid);
assert(uid != 0); /* just to be safe... */
/* The system call kill(-1, sig) sends the signal `sig' to all
users to which the current process can send signals. So we
fork a process, switch to uid, and send a mass kill. */
Pid pid = startProcess([&]() {
if (setuid(uid) == -1)
throw SysError("setting uid");
while (true) {
#ifdef __APPLE__
/* OSX's kill syscall takes a third parameter that, among
other things, determines if kill(-1, signo) affects the
calling process. In the OSX libc, it's set to true,
which means "follow POSIX", which we don't want here
*/
if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
#else
if (kill(-1, SIGKILL) == 0) break;
#endif
if (errno == ESRCH || errno == EPERM) break; /* no more processes */
if (errno != EINTR)
throw SysError("cannot kill processes for uid '%1%'", uid);
}
2012-11-09 17:42:10 +02:00
_exit(0);
});
2013-01-03 14:00:46 +02:00
int status = pid.wait();
if (status != 0)
throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status));
/* !!! We should really do some check to make sure that there are
no processes left running under `uid', but there is no portable
way to do so (I think). The most reliable way may be `ps -eo
uid | grep -q $uid'. */
}
2004-06-22 12:51:44 +03:00
//////////////////////////////////////////////////////////////////////
2014-12-10 17:35:42 +02:00
/* Wrapper around vfork to prevent the child process from clobbering
the caller's stack frame in the parent. */
2019-05-11 23:35:53 +03:00
static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
static pid_t doFork(bool allowVfork, std::function<void()> fun)
{
2014-12-10 17:35:42 +02:00
#ifdef __linux__
pid_t pid = allowVfork ? vfork() : fork();
#else
pid_t pid = fork();
2014-12-10 17:35:42 +02:00
#endif
if (pid != 0) return pid;
fun();
abort();
}
2014-12-10 17:35:42 +02:00
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
{
auto wrapper = [&]() {
if (!options.allowVfork)
logger = makeSimpleLogger();
try {
#if __linux__
2014-12-10 17:35:42 +02:00
if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1)
throw SysError("setting death signal");
#endif
fun();
} catch (std::exception & e) {
try {
2014-12-10 17:35:42 +02:00
std::cerr << options.errorPrefix << e.what() << "\n";
} catch (...) { }
} catch (...) { }
2014-12-10 17:35:42 +02:00
if (options.runExitHandlers)
exit(1);
else
_exit(1);
2014-12-10 17:35:42 +02:00
};
pid_t pid = doFork(options.allowVfork, wrapper);
if (pid == -1) throw SysError("unable to fork");
return pid;
}
2015-06-09 11:50:55 +03:00
std::vector<char *> stringsToCharPtrs(const Strings & ss)
2014-12-12 16:01:16 +02:00
{
2015-06-09 11:50:55 +03:00
std::vector<char *> res;
for (auto & s : ss) res.push_back((char *) s.c_str());
2014-12-12 16:01:16 +02:00
res.push_back(0);
return res;
}
std::string runProgram(Path program, bool searchPath, const Strings & args,
2019-02-12 14:43:32 +02:00
const std::optional<std::string> & input)
{
2021-09-14 09:19:41 +03:00
auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
if (!statusOk(res.first))
throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
return res.second;
}
2019-03-21 10:30:16 +02:00
// Output = error code + "standard out" output stream
std::pair<int, std::string> runProgram(RunOptions && options)
{
StringSink sink;
options.standardOut = &sink;
int status = 0;
try {
runProgram2(options);
} catch (ExecError & e) {
status = e.status;
}
return {status, std::move(sink.s)};
}
void runProgram2(const RunOptions & options)
{
checkInterrupt();
assert(!(options.standardIn && options.input));
std::unique_ptr<Source> source_;
Source * source = options.standardIn;
if (options.input) {
source_ = std::make_unique<StringSource>(*options.input);
source = source_.get();
}
/* Create a pipe. */
Pipe out, in;
if (options.standardOut) out.create();
if (source) in.create();
Add a post-build-hook Passing `--post-build-hook /foo/bar` to a nix-* command will cause `/foo/bar` to be executed after each build with the following environment variables set: DRV_PATH=/nix/store/drv-that-has-been-built.drv OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev This can be useful in particular to upload all the builded artifacts to the cache (including the ones that don't appear in the runtime closure of the final derivation or are built because of IFD). This new feature prints the stderr/stdout output to the `nix-build` and `nix build` client, and the output is printed in a Nix 2 compatible format: [nix]$ ./inst/bin/nix-build ./test.nix these derivations will be built: /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'... hello! bye! running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... post-build-hook: + sleep 1 post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + printf 'very important stuff' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix my-example-derivation> hello! my-example-derivation> bye! my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + printf 'very important stuff' [1 built, 0.0 MiB DL] Co-authored-by: Graham Christensen <graham@grahamc.com> Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
2019-07-11 21:23:03 +03:00
ProcessOptions processOptions;
// vfork implies that the environment of the main process and the fork will
// be shared (technically this is undefined, but in practice that's the
// case), so we can't use it if we alter the environment
processOptions.allowVfork = !options.environment;
Add a post-build-hook Passing `--post-build-hook /foo/bar` to a nix-* command will cause `/foo/bar` to be executed after each build with the following environment variables set: DRV_PATH=/nix/store/drv-that-has-been-built.drv OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev This can be useful in particular to upload all the builded artifacts to the cache (including the ones that don't appear in the runtime closure of the final derivation or are built because of IFD). This new feature prints the stderr/stdout output to the `nix-build` and `nix build` client, and the output is printed in a Nix 2 compatible format: [nix]$ ./inst/bin/nix-build ./test.nix these derivations will be built: /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'... hello! bye! running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... post-build-hook: + sleep 1 post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + printf 'very important stuff' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix my-example-derivation> hello! my-example-derivation> bye! my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + printf 'very important stuff' [1 built, 0.0 MiB DL] Co-authored-by: Graham Christensen <graham@grahamc.com> Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
2019-07-11 21:23:03 +03:00
/* Fork. */
Pid pid = startProcess([&]() {
Add a post-build-hook Passing `--post-build-hook /foo/bar` to a nix-* command will cause `/foo/bar` to be executed after each build with the following environment variables set: DRV_PATH=/nix/store/drv-that-has-been-built.drv OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev This can be useful in particular to upload all the builded artifacts to the cache (including the ones that don't appear in the runtime closure of the final derivation or are built because of IFD). This new feature prints the stderr/stdout output to the `nix-build` and `nix build` client, and the output is printed in a Nix 2 compatible format: [nix]$ ./inst/bin/nix-build ./test.nix these derivations will be built: /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'... hello! bye! running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... post-build-hook: + sleep 1 post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + printf 'very important stuff' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix my-example-derivation> hello! my-example-derivation> bye! my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + printf 'very important stuff' [1 built, 0.0 MiB DL] Co-authored-by: Graham Christensen <graham@grahamc.com> Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
2019-07-11 21:23:03 +03:00
if (options.environment)
replaceEnv(*options.environment);
if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
throw SysError("dupping stdout");
Add a post-build-hook Passing `--post-build-hook /foo/bar` to a nix-* command will cause `/foo/bar` to be executed after each build with the following environment variables set: DRV_PATH=/nix/store/drv-that-has-been-built.drv OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev This can be useful in particular to upload all the builded artifacts to the cache (including the ones that don't appear in the runtime closure of the final derivation or are built because of IFD). This new feature prints the stderr/stdout output to the `nix-build` and `nix build` client, and the output is printed in a Nix 2 compatible format: [nix]$ ./inst/bin/nix-build ./test.nix these derivations will be built: /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'... hello! bye! running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... post-build-hook: + sleep 1 post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + printf 'very important stuff' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix my-example-derivation> hello! my-example-derivation> bye! my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + printf 'very important stuff' [1 built, 0.0 MiB DL] Co-authored-by: Graham Christensen <graham@grahamc.com> Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
2019-07-11 21:23:03 +03:00
if (options.mergeStderrToStdout)
if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
throw SysError("cannot dup stdout into stderr");
if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1)
throw SysError("dupping stdin");
2019-05-13 00:03:01 +03:00
if (options.chdir && chdir((*options.chdir).c_str()) == -1)
throw SysError("chdir failed");
2019-05-11 23:35:53 +03:00
if (options.gid && setgid(*options.gid) == -1)
throw SysError("setgid failed");
/* Drop all other groups if we're setgid. */
if (options.gid && setgroups(0, 0) == -1)
throw SysError("setgroups failed");
if (options.uid && setuid(*options.uid) == -1)
throw SysError("setuid failed");
Strings args_(options.args);
args_.push_front(options.program);
2014-12-12 16:01:16 +02:00
restoreProcessContext();
if (options.searchPath)
execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
2019-03-21 10:30:16 +02:00
// This allows you to refer to a program with a pathname relative
// to the PATH variable.
else
execv(options.program.c_str(), stringsToCharPtrs(args_).data());
throw SysError("executing '%1%'", options.program);
Add a post-build-hook Passing `--post-build-hook /foo/bar` to a nix-* command will cause `/foo/bar` to be executed after each build with the following environment variables set: DRV_PATH=/nix/store/drv-that-has-been-built.drv OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev This can be useful in particular to upload all the builded artifacts to the cache (including the ones that don't appear in the runtime closure of the final derivation or are built because of IFD). This new feature prints the stderr/stdout output to the `nix-build` and `nix build` client, and the output is printed in a Nix 2 compatible format: [nix]$ ./inst/bin/nix-build ./test.nix these derivations will be built: /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'... hello! bye! running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... post-build-hook: + sleep 1 post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + printf 'very important stuff' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix my-example-derivation> hello! my-example-derivation> bye! my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + printf 'very important stuff' [1 built, 0.0 MiB DL] Co-authored-by: Graham Christensen <graham@grahamc.com> Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
2019-07-11 21:23:03 +03:00
}, processOptions);
out.writeSide.close();
std::thread writerThread;
std::promise<void> promise;
Finally doJoin([&]() {
if (writerThread.joinable())
writerThread.join();
});
if (source) {
in.readSide.close();
writerThread = std::thread([&]() {
try {
std::vector<char> buf(8 * 1024);
while (true) {
size_t n;
try {
n = source->read(buf.data(), buf.size());
} catch (EndOfFile &) {
break;
}
writeFull(in.writeSide.get(), {buf.data(), n});
}
promise.set_value();
} catch (...) {
promise.set_exception(std::current_exception());
}
in.writeSide.close();
});
}
if (options.standardOut)
drainFD(out.readSide.get(), *options.standardOut);
/* Wait for the child to finish. */
int status = pid.wait();
/* Wait for the writer thread to finish. */
if (source) promise.get_future().get();
if (status)
throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status));
}
2022-02-21 17:28:23 +02:00
void closeMostFDs(const std::set<int> & exceptions)
{
#if __linux__
try {
for (auto & s : readDirectory("/proc/self/fd")) {
auto fd = std::stoi(s.name);
if (!exceptions.count(fd)) {
debug("closing leaked FD %d", fd);
close(fd);
}
}
return;
} catch (SysError &) {
}
#endif
int maxFD = 0;
maxFD = sysconf(_SC_OPEN_MAX);
for (int fd = 0; fd < maxFD; ++fd)
if (!exceptions.count(fd))
close(fd); /* ignore result */
}
void closeOnExec(int fd)
{
int prev;
if ((prev = fcntl(fd, F_GETFD, 0)) == -1 ||
fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1)
throw SysError("setting close-on-exec flag");
}
2004-06-22 12:51:44 +03:00
//////////////////////////////////////////////////////////////////////
std::atomic<bool> _isInterrupted = false;
static thread_local bool interruptThrown = false;
thread_local std::function<bool()> interruptCheck;
void setInterruptThrown()
{
interruptThrown = true;
}
void _interrupted()
{
/* Block user interrupts while an exception is being handled.
Throwing an exception while another exception is being handled
kills the program! */
if (!interruptThrown && !std::uncaught_exceptions()) {
interruptThrown = true;
throw Interrupted("interrupted by the user");
}
}
2004-06-20 16:37:51 +03:00
2004-06-22 12:51:44 +03:00
//////////////////////////////////////////////////////////////////////
template<class C> C tokenizeString(std::string_view s, std::string_view separators)
2005-09-22 18:43:22 +03:00
{
2012-09-19 22:43:23 +03:00
C result;
auto pos = s.find_first_not_of(separators, 0);
while (pos != std::string_view::npos) {
auto end = s.find_first_of(separators, pos + 1);
if (end == std::string_view::npos) end = s.size();
result.insert(result.end(), std::string(s, pos, end - pos));
2005-09-22 18:43:22 +03:00
pos = s.find_first_not_of(separators, end);
}
return result;
}
template Strings tokenizeString(std::string_view s, std::string_view separators);
template StringSet tokenizeString(std::string_view s, std::string_view separators);
template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);
2012-09-19 22:43:23 +03:00
2005-09-22 18:43:22 +03:00
std::string chomp(std::string_view s)
{
size_t i = s.find_last_not_of(" \n\r\t");
return i == std::string_view::npos ? "" : std::string(s, 0, i + 1);
}
std::string trim(std::string_view s, std::string_view whitespace)
{
auto i = s.find_first_not_of(whitespace);
if (i == s.npos) return "";
auto j = s.find_last_not_of(whitespace);
return std::string(s, i, j == s.npos ? j : j - i + 1);
}
std::string replaceStrings(
std::string res,
std::string_view from,
std::string_view to)
{
2020-11-10 15:59:03 +02:00
if (from.empty()) return res;
size_t pos = 0;
while ((pos = res.find(from, pos)) != std::string::npos) {
res.replace(pos, from.size(), to);
pos += to.size();
}
return res;
}
std::string rewriteStrings(std::string s, const StringMap & rewrites)
Allow content-addressable paths to have references This adds a command 'nix make-content-addressable' that rewrites the specified store paths into content-addressable paths. The advantage of such paths is that 1) they can be imported without signatures; 2) they can enable deduplication in cases where derivation changes do not cause output changes (apart from store path hashes). For example, $ nix make-content-addressable -r nixpkgs.cowsay rewrote '/nix/store/g1g31ah55xdia1jdqabv1imf6mcw0nb1-glibc-2.25-49' to '/nix/store/48jfj7bg78a8n4f2nhg269rgw1936vj4-glibc-2.25-49' ... rewrote '/nix/store/qbi6rzpk0bxjw8lw6azn2mc7ynnn455q-cowsay-3.03+dfsg1-16' to '/nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16' We can then copy the resulting closure to another store without signatures: $ nix copy --trusted-public-keys '' ---to ~/my-nix /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 In order to support self-references in content-addressable paths, these paths are hashed "modulo" self-references, meaning that self-references are zeroed out during hashing. Somewhat annoyingly, this means that the NAR hash stored in the Nix database is no longer necessarily equal to the output of "nix hash-path"; for content-addressable paths, you need to pass the --modulo flag: $ nix path-info --json /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 | jq -r .[].narHash sha256:0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 1ggznh07khq0hz6id09pqws3a8q9pn03ya3c03nwck1kwq8rclzs $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 --modulo iq6g2x4q62xp7y7493bibx0qn5w7xz67 0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw
2018-03-30 01:56:13 +03:00
{
for (auto & i : rewrites) {
if (i.first == i.second) continue;
size_t j = 0;
while ((j = s.find(i.first, j)) != std::string::npos)
Allow content-addressable paths to have references This adds a command 'nix make-content-addressable' that rewrites the specified store paths into content-addressable paths. The advantage of such paths is that 1) they can be imported without signatures; 2) they can enable deduplication in cases where derivation changes do not cause output changes (apart from store path hashes). For example, $ nix make-content-addressable -r nixpkgs.cowsay rewrote '/nix/store/g1g31ah55xdia1jdqabv1imf6mcw0nb1-glibc-2.25-49' to '/nix/store/48jfj7bg78a8n4f2nhg269rgw1936vj4-glibc-2.25-49' ... rewrote '/nix/store/qbi6rzpk0bxjw8lw6azn2mc7ynnn455q-cowsay-3.03+dfsg1-16' to '/nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16' We can then copy the resulting closure to another store without signatures: $ nix copy --trusted-public-keys '' ---to ~/my-nix /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 In order to support self-references in content-addressable paths, these paths are hashed "modulo" self-references, meaning that self-references are zeroed out during hashing. Somewhat annoyingly, this means that the NAR hash stored in the Nix database is no longer necessarily equal to the output of "nix hash-path"; for content-addressable paths, you need to pass the --modulo flag: $ nix path-info --json /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 | jq -r .[].narHash sha256:0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 1ggznh07khq0hz6id09pqws3a8q9pn03ya3c03nwck1kwq8rclzs $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 --modulo iq6g2x4q62xp7y7493bibx0qn5w7xz67 0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw
2018-03-30 01:56:13 +03:00
s.replace(j, i.first.size(), i.second);
}
return s;
}
std::string statusToString(int status)
2004-06-22 11:50:25 +03:00
{
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
if (WIFEXITED(status))
2004-06-22 20:04:10 +03:00
return (format("failed with exit code %1%") % WEXITSTATUS(status)).str();
else if (WIFSIGNALED(status)) {
2013-01-03 14:00:46 +02:00
int sig = WTERMSIG(status);
#if HAVE_STRSIGNAL
const char * description = strsignal(sig);
return (format("failed due to signal %1% (%2%)") % sig % description).str();
#else
return (format("failed due to signal %1%") % sig).str();
#endif
2013-01-03 14:00:46 +02:00
}
2004-06-22 11:50:25 +03:00
else
return "died abnormally";
} else return "succeeded";
}
bool statusOk(int status)
{
return WIFEXITED(status) && WEXITSTATUS(status) == 0;
}
bool hasPrefix(std::string_view s, std::string_view prefix)
{
2017-05-01 18:28:19 +03:00
return s.compare(0, prefix.size(), prefix) == 0;
}
bool hasSuffix(std::string_view s, std::string_view suffix)
{
return s.size() >= suffix.size()
&& s.substr(s.size() - suffix.size()) == suffix;
}
2016-09-14 15:42:15 +03:00
std::string toLower(const std::string & s)
{
std::string r(s);
for (auto & c : r)
c = std::tolower(c);
return r;
}
std::string shellEscape(const std::string_view s)
{
std::string r;
r.reserve(s.size() + 2);
r += "'";
for (auto & i : s)
if (i == '\'') r += "'\\''"; else r += i;
r += '\'';
return r;
}
void ignoreException()
{
/* Make sure no exceptions leave this function.
printError() also throws when remote is closed. */
try {
try {
throw;
} catch (std::exception & e) {
printError("error (ignored): %1%", e.what());
}
} catch (...) { }
}
2021-07-02 03:19:01 +03:00
bool shouldANSI()
{
return isatty(STDERR_FILENO)
&& getEnv("TERM").value_or("dumb") != "dumb"
&& !getEnv("NO_COLOR").has_value();
2021-07-02 03:19:01 +03:00
}
std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width)
{
std::string t, e;
size_t w = 0;
auto i = s.begin();
while (w < (size_t) width && i != s.end()) {
if (*i == '\e') {
std::string e;
e += *i++;
char last = 0;
if (i != s.end() && *i == '[') {
e += *i++;
// eat parameter bytes
while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++;
// eat intermediate bytes
while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
// eat final byte
if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
} else {
if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
2014-08-20 17:01:16 +03:00
}
if (!filterAll && last == 'm')
t += e;
}
else if (*i == '\t') {
i++; t += ' '; w++;
while (w < (size_t) width && w % 8) {
t += ' '; w++;
2014-08-20 17:01:16 +03:00
}
}
else if (*i == '\r' || *i == '\a')
// do nothing for now
i++;
else {
w++;
// Copy one UTF-8 character.
if ((*i & 0xe0) == 0xc0) {
t += *i++;
if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
} else if ((*i & 0xf0) == 0xe0) {
t += *i++;
if (i != s.end() && ((*i & 0xc0) == 0x80)) {
t += *i++;
if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
}
} else if ((*i & 0xf8) == 0xf0) {
t += *i++;
if (i != s.end() && ((*i & 0xc0) == 0x80)) {
t += *i++;
if (i != s.end() && ((*i & 0xc0) == 0x80)) {
t += *i++;
if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
}
}
} else
t += *i++;
}
2014-08-20 17:01:16 +03:00
}
2014-08-20 17:01:16 +03:00
return t;
}
constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
2015-02-09 16:09:39 +02:00
std::string base64Encode(std::string_view s)
2015-02-09 16:09:39 +02:00
{
std::string res;
res.reserve((s.size() + 2) / 3 * 4);
2015-02-09 16:09:39 +02:00
int data = 0, nbits = 0;
for (char c : s) {
data = data << 8 | (unsigned char) c;
nbits += 8;
while (nbits >= 6) {
nbits -= 6;
res.push_back(base64Chars[data >> nbits & 0x3f]);
}
}
if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]);
while (res.size() % 4) res.push_back('=');
return res;
}
std::string base64Decode(std::string_view s)
2015-02-09 16:09:39 +02:00
{
constexpr char npos = -1;
constexpr std::array<char, 256> base64DecodeChars = [&]() {
std::array<char, 256> result{};
for (auto& c : result)
c = npos;
2015-02-09 16:09:39 +02:00
for (int i = 0; i < 64; i++)
result[base64Chars[i]] = i;
return result;
}();
2015-02-09 16:09:39 +02:00
std::string res;
// Some sequences are missing the padding consisting of up to two '='.
// vvv
res.reserve((s.size() + 2) / 4 * 3);
2015-02-09 16:09:39 +02:00
unsigned int d = 0, bits = 0;
for (char c : s) {
if (c == '=') break;
if (c == '\n') continue;
char digit = base64DecodeChars[(unsigned char) c];
if (digit == npos)
2020-07-02 00:32:06 +03:00
throw Error("invalid character in Base64 string: '%c'", c);
2015-02-09 16:09:39 +02:00
bits += 6;
d = d << 6 | digit;
if (bits >= 8) {
res.push_back(d >> (bits - 8) & 0xff);
bits -= 8;
}
}
return res;
}
std::string stripIndentation(std::string_view s)
{
size_t minIndent = 10000;
size_t curIndent = 0;
bool atStartOfLine = true;
for (auto & c : s) {
if (atStartOfLine && c == ' ')
curIndent++;
else if (c == '\n') {
if (atStartOfLine)
minIndent = std::max(minIndent, curIndent);
curIndent = 0;
atStartOfLine = true;
} else {
if (atStartOfLine) {
minIndent = std::min(minIndent, curIndent);
atStartOfLine = false;
}
}
}
std::string res;
size_t pos = 0;
while (pos < s.size()) {
auto eol = s.find('\n', pos);
if (eol == s.npos) eol = s.size();
if (eol - pos > minIndent)
res.append(s.substr(pos + minIndent, eol - pos - minIndent));
res.push_back('\n');
pos = eol + 1;
}
return res;
}
//////////////////////////////////////////////////////////////////////
2017-08-25 16:57:49 +03:00
static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
static void updateWindowSize()
{
struct winsize ws;
if (ioctl(2, TIOCGWINSZ, &ws) == 0) {
2017-08-25 16:57:49 +03:00
auto windowSize_(windowSize.lock());
windowSize_->first = ws.ws_row;
windowSize_->second = ws.ws_col;
}
}
std::pair<unsigned short, unsigned short> getWindowSize()
{
return *windowSize.lock();
}
/* We keep track of interrupt callbacks using integer tokens, so we can iterate
safely without having to lock the data structure while executing arbitrary
functions.
*/
struct InterruptCallbacks {
typedef int64_t Token;
/* We use unique tokens so that we can't accidentally delete the wrong
handler because of an erroneous double delete. */
Token nextToken = 0;
/* Used as a list, see InterruptCallbacks comment. */
std::map<Token, std::function<void()>> callbacks;
};
static Sync<InterruptCallbacks> _interruptCallbacks;
static void signalHandlerThread(sigset_t set)
{
while (true) {
int signal = 0;
sigwait(&set, &signal);
2017-01-25 14:37:02 +02:00
if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP)
triggerInterrupt();
2017-08-25 16:57:49 +03:00
else if (signal == SIGWINCH) {
updateWindowSize();
}
2017-01-25 14:37:02 +02:00
}
}
void triggerInterrupt()
{
_isInterrupted = true;
2017-01-25 14:37:02 +02:00
{
InterruptCallbacks::Token i = 0;
while (true) {
std::function<void()> callback;
{
auto interruptCallbacks(_interruptCallbacks.lock());
auto lb = interruptCallbacks->callbacks.lower_bound(i);
if (lb == interruptCallbacks->callbacks.end())
break;
callback = lb->second;
i = lb->first + 1;
}
2017-01-25 14:37:02 +02:00
try {
callback();
} catch (...) {
ignoreException();
}
}
}
}
static sigset_t savedSignalMask;
void startSignalHandlerThread()
{
2017-08-25 16:57:49 +03:00
updateWindowSize();
if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
2021-03-26 17:14:38 +02:00
throw SysError("querying signal mask");
sigset_t set;
sigemptyset(&set);
sigaddset(&set, SIGINT);
sigaddset(&set, SIGTERM);
sigaddset(&set, SIGHUP);
sigaddset(&set, SIGPIPE);
2017-08-25 16:57:49 +03:00
sigaddset(&set, SIGWINCH);
if (pthread_sigmask(SIG_BLOCK, &set, nullptr))
throw SysError("blocking signals");
std::thread(signalHandlerThread, set).detach();
}
static void restoreSignals()
{
if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
throw SysError("restoring signals");
}
#if __linux__
rlim_t savedStackSize = 0;
#endif
void setStackSize(size_t stackSize)
{
#if __linux__
struct rlimit limit;
if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
savedStackSize = limit.rlim_cur;
limit.rlim_cur = stackSize;
setrlimit(RLIMIT_STACK, &limit);
}
#endif
}
2022-04-01 19:23:43 +03:00
#if __linux__
static AutoCloseFD fdSavedMountNamespace;
2022-04-01 19:23:43 +03:00
#endif
void saveMountNamespace()
{
#if __linux__
static std::once_flag done;
std::call_once(done, []() {
AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY);
if (!fd)
throw SysError("saving parent mount namespace");
fdSavedMountNamespace = std::move(fd);
});
#endif
}
void restoreMountNamespace()
{
#if __linux__
try {
auto savedCwd = absPath(".");
if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
throw SysError("restoring parent mount namespace");
if (chdir(savedCwd.c_str()) == -1) {
throw SysError("restoring cwd");
}
} catch (Error & e) {
debug(e.msg());
}
#endif
}
void unshareFilesystem()
{
#ifdef __linux__
if (unshare(CLONE_FS) != 0 && errno != EPERM)
throw SysError("unsharing filesystem state in download thread");
#endif
}
void restoreProcessContext(bool restoreMounts)
{
restoreSignals();
if (restoreMounts) {
restoreMountNamespace();
}
#if __linux__
if (savedStackSize) {
struct rlimit limit;
if (getrlimit(RLIMIT_STACK, &limit) == 0) {
limit.rlim_cur = savedStackSize;
setrlimit(RLIMIT_STACK, &limit);
}
}
#endif
}
/* RAII helper to automatically deregister a callback. */
struct InterruptCallbackImpl : InterruptCallback
{
InterruptCallbacks::Token token;
~InterruptCallbackImpl() override
{
auto interruptCallbacks(_interruptCallbacks.lock());
interruptCallbacks->callbacks.erase(token);
}
};
std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> callback)
{
auto interruptCallbacks(_interruptCallbacks.lock());
auto token = interruptCallbacks->nextToken++;
interruptCallbacks->callbacks.emplace(token, callback);
auto res = std::make_unique<InterruptCallbackImpl>();
res->token = token;
return std::unique_ptr<InterruptCallback>(res.release());
}
AutoCloseFD createUnixDomainSocket()
{
AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM
#ifdef SOCK_CLOEXEC
| SOCK_CLOEXEC
#endif
, 0);
if (!fdSocket)
throw SysError("cannot create Unix domain socket");
closeOnExec(fdSocket.get());
return fdSocket;
}
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
{
auto fdSocket = nix::createUnixDomainSocket();
bind(fdSocket.get(), path);
if (chmod(path.c_str(), mode) == -1)
throw SysError("changing permissions on '%1%'", path);
if (listen(fdSocket.get(), 100) == -1)
throw SysError("cannot listen on socket '%1%'", path);
return fdSocket;
}
2020-10-06 11:40:49 +03:00
void bind(int fd, const std::string & path)
{
unlink(path.c_str());
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
Path dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot bind to socket '%s'", path);
} else {
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
}
}
void connect(int fd, const std::string & path)
{
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
Path dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
2021-08-25 14:28:36 +03:00
throw Error("cannot connect to socket at '%s'", path);
} else {
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
}
}
std::string showBytes(uint64_t bytes)
2020-10-06 11:40:49 +03:00
{
return fmt("%.2f MiB", bytes / (1024.0 * 1024.0));
}
// FIXME: move to libstore/build
2020-10-11 19:38:46 +03:00
void commonChildInit(Pipe & logPipe)
{
logger = makeSimpleLogger();
const static std::string pathNullDevice = "/dev/null";
restoreProcessContext(false);
2020-10-11 19:38:46 +03:00
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning
that e.g. ssh cannot open /dev/tty) and it doesn't receive
terminal signals. */
if (setsid() == -1)
throw SysError("creating a new session");
/* Dup the write side of the logger pipe into stderr. */
if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1)
throw SysError("cannot pipe standard error into log file");
/* Dup stderr to stdout. */
if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
throw SysError("cannot dup stderr into stdout");
/* Reroute stdin to /dev/null. */
int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
if (fdDevNull == -1)
throw SysError("cannot open '%1%'", pathNullDevice);
if (dup2(fdDevNull, STDIN_FILENO) == -1)
throw SysError("cannot dup null device into stdin");
close(fdDevNull);
}
}