Build the local store on Windows

Fixes #10558

Co-Authored-By: Eugene Butler <eugene@eugene4.com>
Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
This commit is contained in:
John Ericson 2024-04-18 17:49:17 -04:00
parent 0998a3ac01
commit e0ff8da9d5
20 changed files with 228 additions and 155 deletions

View file

@ -241,17 +241,17 @@
''^src/libstore/unix/build/worker\.hh$'' ''^src/libstore/unix/build/worker\.hh$''
''^src/libstore/unix/builtins/fetchurl\.cc$'' ''^src/libstore/unix/builtins/fetchurl\.cc$''
''^src/libstore/unix/builtins/unpack-channel\.cc$'' ''^src/libstore/unix/builtins/unpack-channel\.cc$''
''^src/libstore/unix/gc\.cc$'' ''^src/libstore/gc\.cc$''
''^src/libstore/unix/local-overlay-store\.cc$'' ''^src/libstore/unix/local-overlay-store\.cc$''
''^src/libstore/unix/local-overlay-store\.hh$'' ''^src/libstore/unix/local-overlay-store\.hh$''
''^src/libstore/unix/local-store\.cc$'' ''^src/libstore/local-store\.cc$''
''^src/libstore/unix/local-store\.hh$'' ''^src/libstore/local-store\.hh$''
''^src/libstore/unix/lock\.cc$'' ''^src/libstore/unix/lock\.cc$''
''^src/libstore/unix/lock\.hh$'' ''^src/libstore/unix/lock\.hh$''
''^src/libstore/unix/optimise-store\.cc$'' ''^src/libstore/optimise-store\.cc$''
''^src/libstore/unix/pathlocks\.cc$'' ''^src/libstore/unix/pathlocks\.cc$''
''^src/libstore/unix/posix-fs-canonicalise\.cc$'' ''^src/libstore/posix-fs-canonicalise\.cc$''
''^src/libstore/unix/posix-fs-canonicalise\.hh$'' ''^src/libstore/posix-fs-canonicalise\.hh$''
''^src/libstore/uds-remote-store\.cc$'' ''^src/libstore/uds-remote-store\.cc$''
''^src/libstore/uds-remote-store\.hh$'' ''^src/libstore/uds-remote-store\.hh$''
''^src/libstore/windows/build\.cc$'' ''^src/libstore/windows/build\.cc$''

View file

@ -109,7 +109,7 @@ struct CacheImpl : Cache
Key key, Key key,
Store & store, Store & store,
Attrs value, Attrs value,
const StorePath & storePath) const StorePath & storePath) override
{ {
/* Add the store prefix to the cache key to handle multiple /* Add the store prefix to the cache key to handle multiple
store prefixes. */ store prefixes. */

View file

@ -19,18 +19,20 @@
#include <climits> #include <climits>
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <sys/stat.h>
#if HAVE_STATVFS
# include <sys/statvfs.h>
#endif
#ifndef _WIN32
# include <poll.h> # include <poll.h>
# include <sys/socket.h> # include <sys/socket.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/types.h>
# include <sys/un.h> # include <sys/un.h>
#endif
#include <sys/types.h>
#include <unistd.h> #include <unistd.h>
namespace nix { namespace nix {
using namespace nix::unix;
static std::string gcSocketPath = "/gc-socket/socket"; static std::string gcSocketPath = "/gc-socket/socket";
static std::string gcRootsDir = "gcroots"; static std::string gcRootsDir = "gcroots";
@ -64,7 +66,7 @@ void LocalStore::createTempRootsFile()
/* Check whether the garbage collector didn't get in our /* Check whether the garbage collector didn't get in our
way. */ way. */
struct stat st; struct stat st;
if (fstat(fdTempRoots->get(), &st) == -1) if (fstat(fromDescriptorReadOnly(fdTempRoots->get()), &st) == -1)
throw SysError("statting '%1%'", fnTempRoots); throw SysError("statting '%1%'", fnTempRoots);
if (st.st_size == 0) break; if (st.st_size == 0) break;
@ -108,7 +110,7 @@ void LocalStore::addTempRoot(const StorePath & path)
debug("connecting to '%s'", socketPath); debug("connecting to '%s'", socketPath);
*fdRootsSocket = createUnixDomainSocket(); *fdRootsSocket = createUnixDomainSocket();
try { try {
nix::connect(fdRootsSocket->get(), socketPath); nix::connect(toSocket(fdRootsSocket->get()), socketPath);
} catch (SysError & e) { } catch (SysError & e) {
/* The garbage collector may have exited or not /* The garbage collector may have exited or not
created the socket yet, so we need to restart. */ created the socket yet, so we need to restart. */
@ -166,12 +168,16 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
// those to keep the directory alive. // those to keep the directory alive.
continue; continue;
} }
Path path = i.path(); Path path = i.path().string();
pid_t pid = std::stoi(name); pid_t pid = std::stoi(name);
debug("reading temporary root file '%1%'", path); debug("reading temporary root file '%1%'", path);
AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); AutoCloseFD fd(toDescriptor(open(path.c_str(),
#ifndef _WIN32
O_CLOEXEC |
#endif
O_RDWR, 0666)));
if (!fd) { if (!fd) {
/* It's okay if the file has disappeared. */ /* It's okay if the file has disappeared. */
if (errno == ENOENT) continue; if (errno == ENOENT) continue;
@ -240,8 +246,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
unlink(path.c_str()); unlink(path.c_str());
} }
} else { } else {
struct stat st2 = lstat(target); if (!std::filesystem::is_symlink(target)) return;
if (!S_ISLNK(st2.st_mode)) return;
Path target2 = readLink(target); Path target2 = readLink(target);
if (isInStore(target2)) foundRoot(target, target2); if (isInStore(target2)) foundRoot(target, target2);
} }
@ -297,24 +302,25 @@ Roots LocalStore::findRoots(bool censor)
return roots; return roots;
} }
typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots; /**
* Key is a mere string because cannot has path with macOS's libc++
*/
typedef std::unordered_map<std::string, std::unordered_set<std::string>> UncheckedRoots;
static void readProcLink(const std::string & file, UncheckedRoots & roots) static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots)
{ {
constexpr auto bufsiz = PATH_MAX; std::filesystem::path buf;
char buf[bufsiz]; try {
auto res = readlink(file.c_str(), buf, bufsiz); buf = std::filesystem::read_symlink(file);
if (res == -1) { } catch (std::filesystem::filesystem_error & e) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH) if (e.code() == std::errc::no_such_file_or_directory
|| e.code() == std::errc::permission_denied
|| e.code() == std::errc::no_such_process)
return; return;
throw SysError("reading symlink"); throw;
} }
if (res == bufsiz) { if (buf.is_absolute())
throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz)); roots[buf.string()].emplace(file.string());
}
if (res > 0 && buf[0] == '/')
roots[std::string(static_cast<char *>(buf), res)]
.emplace(file);
} }
static std::string quoteRegexChars(const std::string & raw) static std::string quoteRegexChars(const std::string & raw)
@ -371,12 +377,12 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
} }
fdDir.reset(); fdDir.reset();
auto mapFile = fmt("/proc/%s/maps", ent->d_name); std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n"); auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile.string()), "\n");
for (const auto & line : mapLines) { for (const auto & line : mapLines) {
auto match = std::smatch{}; auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex)) if (std::regex_match(line, match, mapRegex))
unchecked[match[1]].emplace(mapFile); unchecked[match[1]].emplace(mapFile.string());
} }
auto envFile = fmt("/proc/%s/environ", ent->d_name); auto envFile = fmt("/proc/%s/environ", ent->d_name);
@ -407,7 +413,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
for (const auto & line : lsofLines) { for (const auto & line : lsofLines) {
std::smatch match; std::smatch match;
if (std::regex_match(line, match, lsofRegex)) if (std::regex_match(line, match, lsofRegex))
unchecked[match[1]].emplace("{lsof}"); unchecked[match[1].str()].emplace("{lsof}");
} }
} catch (ExecError & e) { } catch (ExecError & e) {
/* lsof not installed, lsof failed */ /* lsof not installed, lsof failed */
@ -490,6 +496,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
createDirs(dirOf(socketPath)); createDirs(dirOf(socketPath));
auto fdServer = createUnixDomainSocket(socketPath, 0666); auto fdServer = createUnixDomainSocket(socketPath, 0666);
// TODO nonblocking socket on windows?
#ifdef _WIN32
throw UnimplementedError("External GC client not implemented yet");
#else
if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1) if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1)
throw SysError("making socket '%1%' non-blocking", socketPath); throw SysError("making socket '%1%' non-blocking", socketPath);
@ -590,6 +600,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
if (serverThread.joinable()) serverThread.join(); if (serverThread.joinable()) serverThread.join();
}); });
#endif
/* Find the roots. Since we've grabbed the GC lock, the set of /* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */ permanent roots cannot increase now. */
printInfo("finding garbage collector roots..."); printInfo("finding garbage collector roots...");
@ -623,8 +635,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
by another process. We need to be sure that we can acquire an by another process. We need to be sure that we can acquire an
exclusive lock before deleting them. */ exclusive lock before deleting them. */
if (baseName.find("tmp-", 0) == 0) { if (baseName.find("tmp-", 0) == 0) {
AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); AutoCloseFD tmpDirFd = openDirectory(realPath);
if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) { if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) {
debug("skipping locked tempdir '%s'", realPath); debug("skipping locked tempdir '%s'", realPath);
return; return;
} }
@ -857,7 +869,13 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
struct stat st; struct stat st;
if (stat(linksDir.c_str(), &st) == -1) if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir); throw SysError("statting '%1%'", linksDir);
int64_t overhead = st.st_blocks * 512ULL; int64_t overhead =
#ifdef _WIN32
0
#else
st.st_blocks * 512ULL
#endif
;
printInfo("note: currently hard linking saves %.2f MiB", printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
@ -870,6 +888,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
void LocalStore::autoGC(bool sync) void LocalStore::autoGC(bool sync)
{ {
#ifdef HAVE_STATVFS
static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE");
auto getAvail = [this]() -> uint64_t { auto getAvail = [this]() -> uint64_t {
@ -946,6 +965,7 @@ void LocalStore::autoGC(bool sync)
sync: sync:
// Wait for the future outside of the state lock. // Wait for the future outside of the state lock.
if (sync) future.get(); if (sync) future.get();
#endif
} }

View file

@ -424,8 +424,10 @@ public:
Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal",
"Whether SQLite should use WAL mode."}; "Whether SQLite should use WAL mode."};
#ifndef _WIN32
Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering", Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
"Whether to call `sync()` before registering a path as valid."}; "Whether to call `sync()` before registering a path as valid."};
#endif
Setting<bool> useSubstitutes{ Setting<bool> useSubstitutes{
this, true, "substitute", this, true, "substitute",

View file

@ -26,7 +26,6 @@
#include <new> #include <new>
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/select.h>
#include <sys/time.h> #include <sys/time.h>
#include <unistd.h> #include <unistd.h>
#include <utime.h> #include <utime.h>
@ -34,7 +33,10 @@
#include <errno.h> #include <errno.h>
#include <stdio.h> #include <stdio.h>
#include <time.h> #include <time.h>
#ifndef _WIN32
# include <grp.h> # include <grp.h>
#endif
#if __linux__ #if __linux__
# include <sched.h> # include <sched.h>
@ -52,8 +54,6 @@
namespace nix { namespace nix {
using namespace nix::unix;
std::string LocalStoreConfig::doc() std::string LocalStoreConfig::doc()
{ {
return return
@ -224,6 +224,7 @@ LocalStore::LocalStore(const Params & params)
} }
} }
#ifndef _WIN32
/* Optionally, create directories and set permissions for a /* Optionally, create directories and set permissions for a
multi-user install. */ multi-user install. */
if (isRootUser() && settings.buildUsersGroup != "") { if (isRootUser() && settings.buildUsersGroup != "") {
@ -245,6 +246,7 @@ LocalStore::LocalStore(const Params & params)
} }
} }
} }
#endif
/* Ensure that the store and its parents are not symlinks. */ /* Ensure that the store and its parents are not symlinks. */
if (!settings.allowSymlinkedStore) { if (!settings.allowSymlinkedStore) {
@ -270,14 +272,25 @@ LocalStore::LocalStore(const Params & params)
if (stat(reservedPath.c_str(), &st) == -1 || if (stat(reservedPath.c_str(), &st) == -1 ||
st.st_size != settings.reservedSize) st.st_size != settings.reservedSize)
{ {
AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600); AutoCloseFD fd = toDescriptor(open(reservedPath.c_str(), O_WRONLY | O_CREAT
#ifndef _WIN32
| O_CLOEXEC
#endif
, 0600));
int res = -1; int res = -1;
#if HAVE_POSIX_FALLOCATE #if HAVE_POSIX_FALLOCATE
res = posix_fallocate(fd.get(), 0, settings.reservedSize); res = posix_fallocate(fd.get(), 0, settings.reservedSize);
#endif #endif
if (res == -1) { if (res == -1) {
writeFull(fd.get(), std::string(settings.reservedSize, 'X')); writeFull(fd.get(), std::string(settings.reservedSize, 'X'));
[[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); [[gnu::unused]] auto res2 =
#ifdef _WIN32
SetEndOfFile(fd.get())
#else
ftruncate(fd.get(), settings.reservedSize)
#endif
;
} }
} }
} catch (SystemError & e) { /* don't care about errors */ } catch (SystemError & e) { /* don't care about errors */
@ -460,10 +473,14 @@ LocalStore::LocalStore(std::string scheme, std::string path, const Params & para
AutoCloseFD LocalStore::openGCLock() AutoCloseFD LocalStore::openGCLock()
{ {
Path fnGCLock = stateDir + "/gc.lock"; Path fnGCLock = stateDir + "/gc.lock";
auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT
#ifndef _WIN32
| O_CLOEXEC
#endif
, 0600);
if (!fdGCLock) if (!fdGCLock)
throw SysError("opening global GC lock '%1%'", fnGCLock); throw SysError("opening global GC lock '%1%'", fnGCLock);
return fdGCLock; return toDescriptor(fdGCLock);
} }
@ -491,7 +508,7 @@ LocalStore::~LocalStore()
try { try {
auto fdTempRoots(_fdTempRoots.lock()); auto fdTempRoots(_fdTempRoots.lock());
if (*fdTempRoots) { if (*fdTempRoots) {
*fdTempRoots = -1; fdTempRoots->close();
unlink(fnTempRoots.c_str()); unlink(fnTempRoots.c_str());
} }
} catch (...) { } catch (...) {
@ -969,11 +986,13 @@ void LocalStore::registerValidPath(const ValidPathInfo & info)
void LocalStore::registerValidPaths(const ValidPathInfos & infos) void LocalStore::registerValidPaths(const ValidPathInfos & infos)
{ {
#ifndef _WIN32
/* SQLite will fsync by default, but the new valid paths may not /* SQLite will fsync by default, but the new valid paths may not
be fsync-ed. So some may want to fsync them before registering be fsync-ed. So some may want to fsync them before registering
the validity, at the expense of some speed of the path the validity, at the expense of some speed of the path
registering operation. */ registering operation. */
if (settings.syncBeforeRegistering) sync(); if (settings.syncBeforeRegistering) sync();
#endif
return retrySQLite<void>([&]() { return retrySQLite<void>([&]() {
auto state(_state.lock()); auto state(_state.lock());
@ -1155,7 +1174,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
autoGC(); autoGC();
canonicalisePathMetaData(realPath, {}); canonicalisePathMetaData(realPath);
optimisePath(realPath, repair); // FIXME: combine with hashPath() optimisePath(realPath, repair); // FIXME: combine with hashPath()
@ -1307,7 +1326,7 @@ StorePath LocalStore::addToStoreFromDump(
narHash = narSink.finish(); narHash = narSink.finish();
} }
canonicalisePathMetaData(realPath, {}); // FIXME: merge into restorePath canonicalisePathMetaData(realPath); // FIXME: merge into restorePath
optimisePath(realPath, repair); optimisePath(realPath, repair);
@ -1340,8 +1359,8 @@ std::pair<std::filesystem::path, AutoCloseFD> LocalStore::createTempDirInStore()
the GC between createTempDir() and when we acquire a lock on it. the GC between createTempDir() and when we acquire a lock on it.
We'll repeat until 'tmpDir' exists and we've locked it. */ We'll repeat until 'tmpDir' exists and we've locked it. */
tmpDirFn = createTempDir(realStoreDir, "tmp"); tmpDirFn = createTempDir(realStoreDir, "tmp");
tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY); tmpDirFd = openDirectory(tmpDirFn);
if (tmpDirFd.get() < 0) { if (!tmpDirFd) {
continue; continue;
} }
lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true);
@ -1390,19 +1409,16 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
for (auto & link : readDirectory(linksDir)) { for (auto & link : readDirectory(linksDir)) {
auto name = link.path().filename(); auto name = link.path().filename();
printMsg(lvlTalkative, "checking contents of '%s'", name); printMsg(lvlTalkative, "checking contents of '%s'", name);
Path linkPath = linksDir / name;
PosixSourceAccessor accessor; PosixSourceAccessor accessor;
std::string hash = hashPath( std::string hash = hashPath(
{getFSSourceAccessor(), CanonPath(linkPath)}, PosixSourceAccessor::createAtRoot(link.path()),
FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false); FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false);
if (hash != name.string()) { if (hash != name.string()) {
printError("link '%s' was modified! expected hash '%s', got '%s'", printError("link '%s' was modified! expected hash '%s', got '%s'",
linkPath, name, hash); link.path(), name, hash);
if (repair) { if (repair) {
if (unlink(linkPath.c_str()) == 0) std::filesystem::remove(link.path());
printInfo("removed link '%s'", linkPath); printInfo("removed link '%s'", link.path());
else
throw SysError("removing corrupt link '%s'", linkPath);
} else { } else {
errors = true; errors = true;
} }
@ -1583,8 +1599,12 @@ static void makeMutable(const Path & path)
/* The O_NOFOLLOW is important to prevent us from changing the /* The O_NOFOLLOW is important to prevent us from changing the
mutable bit on the target of a symlink (which would be a mutable bit on the target of a symlink (which would be a
security hole). */ security hole). */
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW
if (fd == -1) { #ifndef _WIN32
| O_CLOEXEC
#endif
);
if (fd == INVALID_DESCRIPTOR) {
if (errno == ELOOP) return; // it's a symlink if (errno == ELOOP) return; // it's a symlink
throw SysError("opening file '%1%'", path); throw SysError("opening file '%1%'", path);
} }

View file

@ -89,11 +89,11 @@ else
endif endif
endif endif
$(d)/unix/local-store.cc: $(d)/unix/schema.sql.gen.hh $(d)/unix/ca-specific-schema.sql.gen.hh $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(d)/unix/build.cc: $(d)/unix/build.cc:
clean-files += $(d)/unix/schema.sql.gen.hh $(d)/unix/ca-specific-schema.sql.gen.hh clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) $(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))

View file

@ -155,55 +155,53 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
/* Check if this is a known hash. */ /* Check if this is a known hash. */
Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Nix32, false); std::filesystem::path linkPath = std::filesystem::path{linksDir} / hash.to_string(HashFormat::Nix32, false);
/* Maybe delete the link, if it has been corrupted. */ /* Maybe delete the link, if it has been corrupted. */
if (pathExists(linkPath)) { if (std::filesystem::exists(std::filesystem::symlink_status(linkPath))) {
auto stLink = lstat(linkPath); auto stLink = lstat(linkPath.string());
if (st.st_size != stLink.st_size if (st.st_size != stLink.st_size
|| (repair && hash != ({ || (repair && hash != ({
hashPath( hashPath(
{make_ref<PosixSourceAccessor>(), CanonPath(linkPath)}, PosixSourceAccessor::createAtRoot(linkPath),
FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first;
}))) })))
{ {
// XXX: Consider overwriting linkPath with our valid version. // XXX: Consider overwriting linkPath with our valid version.
warn("removing corrupted link '%s'", linkPath); warn("removing corrupted link %s", linkPath);
warn("There may be more corrupted paths." warn("There may be more corrupted paths."
"\nYou should run `nix-store --verify --check-contents --repair` to fix them all"); "\nYou should run `nix-store --verify --check-contents --repair` to fix them all");
unlink(linkPath.c_str()); std::filesystem::remove(linkPath);
} }
} }
if (!pathExists(linkPath)) { if (!std::filesystem::exists(std::filesystem::symlink_status(linkPath))) {
/* Nope, create a hard link in the links directory. */ /* Nope, create a hard link in the links directory. */
if (link(path.c_str(), linkPath.c_str()) == 0) { try {
std::filesystem::create_hard_link(path, linkPath);
inodeHash.insert(st.st_ino); inodeHash.insert(st.st_ino);
return; } catch (std::filesystem::filesystem_error & e) {
} if (e.code() == std::errc::file_exists) {
switch (errno) {
case EEXIST:
/* Fall through if another process created linkPath before /* Fall through if another process created linkPath before
we did. */ we did. */
break; }
case ENOSPC: else if (e.code() == std::errc::no_space_on_device) {
/* On ext4, that probably means the directory index is /* On ext4, that probably means the directory index is
full. When that happens, it's fine to ignore it: we full. When that happens, it's fine to ignore it: we
just effectively disable deduplication of this just effectively disable deduplication of this
file. */ file. */
printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno));
return; return;
}
default: else throw;
throw SysError("cannot link '%1%' to '%2%'", linkPath, path);
} }
} }
/* Yes! We've seen a file with the same contents. Replace the /* Yes! We've seen a file with the same contents. Replace the
current file with a hard link to that file. */ current file with a hard link to that file. */
auto stLink = lstat(linkPath); auto stLink = lstat(linkPath.string());
if (st.st_ino == stLink.st_ino) { if (st.st_ino == stLink.st_ino) {
debug("'%1%' is already linked to '%2%'", path, linkPath); debug("'%1%' is already linked to '%2%'", path, linkPath);
@ -223,10 +221,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
its timestamp back to 0. */ its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : "");
Path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), random()); std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand());
if (link(linkPath.c_str(), tempLink.c_str()) == -1) { try {
if (errno == EMLINK) { std::filesystem::create_hard_link(linkPath, tempLink);
inodeHash.insert(st.st_ino);
} catch (std::filesystem::filesystem_error & e) {
if (e.code() == std::errc::too_many_links) {
/* Too many links to the same file (>= 32000 on most file /* Too many links to the same file (>= 32000 on most file
systems). This is likely to happen with empty files. systems). This is likely to happen with empty files.
Just shrug and ignore. */ Just shrug and ignore. */
@ -234,16 +235,16 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
printInfo("'%1%' has maximum number of links", linkPath); printInfo("'%1%' has maximum number of links", linkPath);
return; return;
} }
throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); throw;
} }
/* Atomically replace the old file with the new hard link. */ /* Atomically replace the old file with the new hard link. */
try { try {
renameFile(tempLink, path); std::filesystem::rename(tempLink, path);
} catch (SystemError & e) { } catch (std::filesystem::filesystem_error & e) {
if (unlink(tempLink.c_str()) == -1) std::filesystem::remove(tempLink);
printError("unable to unlink '%1%'", tempLink); printError("unable to unlink '%1%'", tempLink);
if (errno == EMLINK) { if (e.code() == std::errc::too_many_links) {
/* Some filesystems generate too many links on the rename, /* Some filesystems generate too many links on the rename,
rather than on the original link. (Probably it rather than on the original link. (Probably it
temporarily increases the st_nlink field before temporarily increases the st_nlink field before
@ -258,7 +259,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
stats.bytesFreed += st.st_size; stats.bytesFreed += st.st_size;
if (act) if (act)
act->result(resFileLinked, st.st_size, st.st_blocks); act->result(resFileLinked, st.st_size
#ifndef _WIN32
, st.st_blocks
#endif
);
} }

View file

@ -31,6 +31,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
} }
#ifndef _WIN32 // TODO implement
if (st.st_mtime != mtimeStore) { if (st.st_mtime != mtimeStore) {
struct timeval times[2]; struct timeval times[2];
times[0].tv_sec = st.st_atime; times[0].tv_sec = st.st_atime;
@ -46,6 +47,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
#endif #endif
throw SysError("changing modification time of '%1%'", path); throw SysError("changing modification time of '%1%'", path);
} }
#endif
} }
@ -57,7 +59,9 @@ void canonicaliseTimestampAndPermissions(const Path & path)
static void canonicalisePathMetaData_( static void canonicalisePathMetaData_(
const Path & path, const Path & path,
#ifndef _WIN32
std::optional<std::pair<uid_t, uid_t>> uidRange, std::optional<std::pair<uid_t, uid_t>> uidRange,
#endif
InodesSeen & inodesSeen) InodesSeen & inodesSeen)
{ {
checkInterrupt(); checkInterrupt();
@ -99,6 +103,7 @@ static void canonicalisePathMetaData_(
} }
#endif #endif
#ifndef _WIN32
/* Fail if the file is not owned by the build user. This prevents /* Fail if the file is not owned by the build user. This prevents
us from messing up the ownership/permissions of files us from messing up the ownership/permissions of files
hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
@ -112,11 +117,13 @@ static void canonicalisePathMetaData_(
assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
return; return;
} }
#endif
inodesSeen.insert(Inode(st.st_dev, st.st_ino)); inodesSeen.insert(Inode(st.st_dev, st.st_ino));
canonicaliseTimestampAndPermissions(path, st); canonicaliseTimestampAndPermissions(path, st);
#ifndef _WIN32
/* Change ownership to the current uid. If it's a symlink, use /* Change ownership to the current uid. If it's a symlink, use
lchown if available, otherwise don't bother. Wrong ownership lchown if available, otherwise don't bother. Wrong ownership
of a symlink doesn't matter, since the owning user can't change of a symlink doesn't matter, since the owning user can't change
@ -134,22 +141,36 @@ static void canonicalisePathMetaData_(
throw SysError("changing owner of '%1%' to %2%", throw SysError("changing owner of '%1%' to %2%",
path, geteuid()); path, geteuid());
} }
#endif
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
std::vector<std::filesystem::directory_entry> entries = readDirectory(path); std::vector<std::filesystem::directory_entry> entries = readDirectory(path);
for (auto & i : entries) for (auto & i : entries)
canonicalisePathMetaData_(i.path().string(), uidRange, inodesSeen); canonicalisePathMetaData_(
i.path().string(),
#ifndef _WIN32
uidRange,
#endif
inodesSeen);
} }
} }
void canonicalisePathMetaData( void canonicalisePathMetaData(
const Path & path, const Path & path,
#ifndef _WIN32
std::optional<std::pair<uid_t, uid_t>> uidRange, std::optional<std::pair<uid_t, uid_t>> uidRange,
#endif
InodesSeen & inodesSeen) InodesSeen & inodesSeen)
{ {
canonicalisePathMetaData_(path, uidRange, inodesSeen); canonicalisePathMetaData_(
path,
#ifndef _WIN32
uidRange,
#endif
inodesSeen);
#ifndef _WIN32
/* On platforms that don't have lchown(), the top-level path can't /* On platforms that don't have lchown(), the top-level path can't
be a symlink, since we can't change its ownership. */ be a symlink, since we can't change its ownership. */
auto st = lstat(path); auto st = lstat(path);
@ -158,14 +179,23 @@ void canonicalisePathMetaData(
assert(S_ISLNK(st.st_mode)); assert(S_ISLNK(st.st_mode));
throw Error("wrong ownership of top-level store path '%1%'", path); throw Error("wrong ownership of top-level store path '%1%'", path);
} }
#endif
} }
void canonicalisePathMetaData(const Path & path, void canonicalisePathMetaData(const Path & path
std::optional<std::pair<uid_t, uid_t>> uidRange) #ifndef _WIN32
, std::optional<std::pair<uid_t, uid_t>> uidRange
#endif
)
{ {
InodesSeen inodesSeen; InodesSeen inodesSeen;
canonicalisePathMetaData(path, uidRange, inodesSeen); canonicalisePathMetaData_(
path,
#ifndef _WIN32
uidRange,
#endif
inodesSeen);
} }
} }

View file

@ -24,7 +24,7 @@ typedef std::set<Inode> InodesSeen;
* without execute permission; setuid bits etc. are cleared) * without execute permission; setuid bits etc. are cleared)
* *
* - the owner and group are set to the Nix user and group, if we're * - the owner and group are set to the Nix user and group, if we're
* running as root. * running as root. (Unix only.)
* *
* If uidRange is not empty, this function will throw an error if it * If uidRange is not empty, this function will throw an error if it
* encounters files owned by a user outside of the closed interval * encounters files owned by a user outside of the closed interval
@ -32,11 +32,17 @@ typedef std::set<Inode> InodesSeen;
*/ */
void canonicalisePathMetaData( void canonicalisePathMetaData(
const Path & path, const Path & path,
#ifndef _WIN32
std::optional<std::pair<uid_t, uid_t>> uidRange, std::optional<std::pair<uid_t, uid_t>> uidRange,
#endif
InodesSeen & inodesSeen); InodesSeen & inodesSeen);
void canonicalisePathMetaData( void canonicalisePathMetaData(
const Path & path, const Path & path
std::optional<std::pair<uid_t, uid_t>> uidRange); #ifndef _WIN32
, std::optional<std::pair<uid_t, uid_t>> uidRange = std::nullopt
#endif
);
void canonicaliseTimestampAndPermissions(const Path & path); void canonicaliseTimestampAndPermissions(const Path & path);

View file

@ -20,10 +20,6 @@
#include "signals.hh" #include "signals.hh"
#include "users.hh" #include "users.hh"
#ifndef _WIN32
# include "remote-store.hh"
#endif
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <regex> #include <regex>
@ -1265,10 +1261,9 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath)
} }
#ifndef _WIN32
#include "local-store.hh" #include "local-store.hh"
#include "uds-remote-store.hh" #include "uds-remote-store.hh"
#endif
namespace nix { namespace nix {
@ -1286,9 +1281,6 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
return {uri, params}; return {uri, params};
} }
#ifdef _WIN32 // Unused on Windows because the next `#ifndef`
[[maybe_unused]]
#endif
static bool isNonUriPath(const std::string & spec) static bool isNonUriPath(const std::string & spec)
{ {
return return
@ -1303,7 +1295,6 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
{ {
// TODO reenable on Windows once we have `LocalStore` and // TODO reenable on Windows once we have `LocalStore` and
// `UDSRemoteStore`. // `UDSRemoteStore`.
#ifndef _WIN32
if (uri == "" || uri == "auto") { if (uri == "" || uri == "auto") {
auto stateDir = getOr(params, "state", settings.nixStateDir); auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0) if (access(stateDir.c_str(), R_OK | W_OK) == 0)
@ -1348,9 +1339,6 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
} else { } else {
return nullptr; return nullptr;
} }
#else
return nullptr;
#endif
} }
// The `parseURL` function supports both IPv6 URIs as defined in // The `parseURL` function supports both IPv6 URIs as defined in

View file

@ -30,32 +30,6 @@
#include <sys/utsname.h> #include <sys/utsname.h>
#include <sys/resource.h> #include <sys/resource.h>
#if HAVE_STATVFS
#include <sys/statvfs.h>
#endif
/* Includes required for chroot support. */
#if __linux__
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <netinet/ip.h>
#include <sys/mman.h>
#include <sched.h>
#include <sys/param.h>
#include <sys/mount.h>
#include <sys/syscall.h>
#if HAVE_SECCOMP
#include <seccomp.h>
#endif
#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
#endif
#if __APPLE__
#include <spawn.h>
#include <sys/sysctl.h>
#endif
#include <pwd.h> #include <pwd.h>
#include <grp.h> #include <grp.h>

View file

@ -94,7 +94,7 @@ Path canonPath(PathView path, bool resolveSymlinks)
path, path,
[&followCount, &temp, maxFollow, resolveSymlinks] [&followCount, &temp, maxFollow, resolveSymlinks]
(std::string & result, std::string_view & remaining) { (std::string & result, std::string_view & remaining) {
if (resolveSymlinks && std::filesystem::is_symlink(result)) { if (resolveSymlinks && fs::is_symlink(result)) {
if (++followCount >= maxFollow) if (++followCount >= maxFollow)
throw Error("infinite symlink recursion in path '%0%'", remaining); throw Error("infinite symlink recursion in path '%0%'", remaining);
remaining = (temp = concatStrings(readLink(result), remaining)); remaining = (temp = concatStrings(readLink(result), remaining));

View file

@ -117,6 +117,11 @@ bool pathAccessible(const Path & path);
*/ */
Path readLink(const Path & path); Path readLink(const Path & path);
/**
* Open a `Descriptor` with read-only access to the given directory.
*/
Descriptor openDirectory(const std::filesystem::path & path);
/** /**
* Read the contents of a directory. The entries `.` and `..` are * Read the contents of a directory. The entries `.` and `..` are
* removed. * removed.

View file

@ -39,6 +39,7 @@ using Socket =
* Windows gives this a different name * Windows gives this a different name
*/ */
# define SHUT_WR SD_SEND # define SHUT_WR SD_SEND
# define SHUT_RDWR SD_BOTH
#endif #endif
/** /**

View file

@ -0,0 +1,10 @@
#include "file-system.hh"
namespace nix {
Descriptor openDirectory(const std::filesystem::path & path)
{
return open(path.c_str(), O_RDONLY | O_DIRECTORY);
}
}

View file

@ -0,0 +1,12 @@
#include "file-system.hh"
namespace nix {
Descriptor openDirectory(const std::filesystem::path & path)
{
return CreateFileW(
path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, NULL);
}
}