max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2025-02-16 15:17:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'nixos/master'

Browse source
This commit is contained in:
Max Headroom 2023-11-22 22:10:12 +01:00
commit 4f4d3a538e
40 changed files with 575 additions and 215 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
doc/manual/src/release-notes/rl-next.md
View file

@ -1,2 +1,6 @@
# Release X.Y (202?-??-??)
- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.

2
scripts/nix-profile-daemon.sh.in
View file

@ -31,7 +31,7 @@ fi
export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
# Populate bash completions, .desktop files, etc
if [ -z "$XDG_DATA_DIRS" ]; then
if [ -z "${XDG_DATA_DIRS-}" ]; then
# According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
else

2
scripts/nix-profile.sh.in
View file

@ -33,7 +33,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
# Populate bash completions, .desktop files, etc
if [ -z "$XDG_DATA_DIRS" ]; then
if [ -z "${XDG_DATA_DIRS-}" ]; then
# According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
else

3
src/libexpr/eval.cc
View file

@ -532,6 +532,9 @@ EvalState::EvalState(
, baseEnv(allocEnv(128))
, staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)}
{
corepkgsFS->setPathDisplay("<nix", ">");
internalFS->setPathDisplay("«nix-internal»", "");
countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
assert(gcInitialised);

2
src/libexpr/tests/local.mk
View file

@ -20,4 +20,4 @@ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/l
libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock
libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock

5
src/libfetchers/fetchers.cc
View file

@ -108,6 +108,11 @@ Input Input::fromAttrs(Attrs && attrs)
return std::move(*res);
}
std::optional<std::string> Input::getFingerprint(ref<Store> store) const
{
return scheme ? scheme->getFingerprint(store, *this) : std::nullopt;
}
ParsedURL Input::toURL() const
{
if (!scheme)

9
src/libfetchers/fetchers.hh
View file

@ -113,6 +113,12 @@ public:
std::optional<Hash> getRev() const;
std::optional<uint64_t> getRevCount() const;
std::optional<time_t> getLastModified() const;
/**
* For locked inputs, return a string that uniquely specifies the
* content of the input (typically a commit hash or content hash).
*/
std::optional<std::string> getFingerprint(ref<Store> store) const;
};
@ -180,6 +186,9 @@ struct InputScheme
virtual bool isDirect(const Input & input) const
{ return true; }
virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
{ return std::nullopt; }
};
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

1
src/libfetchers/fs-input-accessor.cc
View file

@ -18,6 +18,7 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
, allowedPaths(std::move(allowedPaths))
, makeNotAllowedError(std::move(makeNotAllowedError))
{
displayPrefix = root.isRoot() ? "" : root.abs();
}
void readFile(

4
src/libfetchers/git-utils.cc
View file

@ -381,7 +381,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
};
git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
opts.depth = shallow ? 1 : GIT_FETCH_DEPTH_FULL;
// FIXME: for some reason, shallow fetching over ssh barfs
// with "could not read from remote repository".
opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL;
opts.callbacks.payload = &act;
opts.callbacks.sideband_progress = sidebandProgressCallback;
opts.callbacks.transfer_progress = transferProgressCallback;

21
src/libfetchers/git.cc
View file

@ -519,8 +519,11 @@ struct GitInputScheme : InputScheme
if (doFetch) {
try {
auto fetchRef = getAllRefsAttr(input)
auto fetchRef =
getAllRefsAttr(input)
? "refs/*"
: input.getRev()
? input.getRev()->gitRev()
: ref.compare(0, 5, "refs/") == 0
? ref
: ref == "HEAD"
@ -583,6 +586,8 @@ struct GitInputScheme : InputScheme
auto accessor = repo->getAccessor(rev);
accessor->setPathDisplay("«" + input.to_string() + "»");
/* If the repo has submodules, fetch them and return a mounted
input accessor consisting of the accessor for the top-level
repo and the accessors for the submodules. */
@ -701,10 +706,22 @@ struct GitInputScheme : InputScheme
auto repoInfo = getRepoInfo(input);
return
auto [accessor, final] =
input.getRef() || input.getRev() || !repoInfo.isLocal
? getAccessorFromCommit(store, repoInfo, std::move(input))
: getAccessorFromWorkdir(store, repoInfo, std::move(input));
accessor->fingerprint = final.getFingerprint(store);
return {accessor, std::move(final)};
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())
return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "");
else
return std::nullopt;
}
};

8
src/libfetchers/github.cc
View file

@ -229,6 +229,14 @@ struct GitArchiveInputScheme : InputScheme
{
return Xp::Flakes;
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())
return rev->gitRev();
else
return std::nullopt;
}
};
struct GitHubInputScheme : GitArchiveInputScheme

25
src/libfetchers/input-accessor.cc
View file

@ -1,5 +1,6 @@
#include "input-accessor.hh"
#include "store-api.hh"
#include "cache.hh"
namespace nix {
@ -11,6 +12,27 @@ StorePath InputAccessor::fetchToStore(
PathFilter * filter,
RepairFlag repair)
{
// FIXME: add an optimisation for the case where the accessor is
// an FSInputAccessor pointing to a store path.
std::optional<fetchers::Attrs> cacheKey;
if (!filter && fingerprint) {
cacheKey = fetchers::Attrs{
{"_what", "fetchToStore"},
{"store", store->storeDir},
{"name", std::string(name)},
{"fingerprint", *fingerprint},
{"method", (uint8_t) method},
{"path", path.abs()}
};
if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
debug("store path cache hit for '%s'", showPath(path));
return res->second;
}
} else
debug("source path '%s' is uncacheable", showPath(path));
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
auto source = sinkToSource([&](Sink & sink) {
@ -25,6 +47,9 @@ StorePath InputAccessor::fetchToStore(
? store->computeStorePathFromDump(*source, name, method, htSHA256).first
: store->addToStoreFromDump(*source, name, method, htSHA256, repair);
if (cacheKey)
fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
return storePath;
}

2
src/libfetchers/input-accessor.hh
View file

@ -18,6 +18,8 @@ class Store;
struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
{
std::optional<std::string> fingerprint;
/**
* Return the maximum last-modified time of the files in this
* tree, if available.

8
src/libfetchers/mercurial.cc
View file

@ -339,6 +339,14 @@ struct MercurialInputScheme : InputScheme
return makeResult(infoAttrs, std::move(storePath));
}
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{
if (auto rev = input.getRev())
return rev->gitRev();
else
return std::nullopt;
}
};
static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });

1
src/libstore/build/local-derivation-goal.cc
View file

@ -19,6 +19,7 @@
#include "namespaces.hh"
#include "child.hh"
#include "unix-domain-socket.hh"
#include "posix-fs-canonicalise.hh"
#include <regex>
#include <queue>

4
src/libstore/content-address.hh
View file

@ -39,12 +39,12 @@ enum struct FileIngestionMethod : uint8_t {
/**
* Flat-file hashing. Directly ingest the contents of a single file
*/
Flat = false,
Flat = 0,
/**
* Recursive (or NAR) hashing. Serializes the file-system object in Nix
* Archive format and ingest that
*/
Recursive = true
Recursive = 1
};
/**

15
src/libstore/daemon.cc
View file

@ -657,6 +657,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
case WorkerProto::Op::AddPermRoot: {
if (!trusted)
throw Error(
"you are not privileged to create perm roots\n\n"
"hint: you can just do this client-side without special privileges, and probably want to do that instead.");
auto storePath = WorkerProto::Serialise<StorePath>::read(*store, rconn);
Path gcRoot = absPath(readString(from));
logger->startWork();
auto & localFSStore = require<LocalFSStore>(*store);
localFSStore.addPermRoot(storePath, gcRoot);
logger->stopWork();
to << gcRoot;
break;
}
case WorkerProto::Op::AddIndirectRoot: {
Path path = absPath(readString(from));

24
src/libstore/indirect-root-store.hh
View file

@ -11,6 +11,30 @@ namespace nix {
* reference.
*
* See methods for details on the operations it represents.
*
* @note
* To understand the purpose of this class, it might help to do some
* "closed-world" rather than "open-world" reasoning, and consider the
* problem it solved for us. This class was factored out from
* `LocalFSStore` in order to support the following table, which
* contains 4 concrete store types (non-abstract classes, exposed to the
* user), and how they implemented the two GC root methods:
*
* @note
* | | `addPermRoot()` | `addIndirectRoot()` |
* |-------------------|-----------------|---------------------|
* | `LocalStore` | local | local |
* | `UDSRemoteStore` | local | remote |
* | `SSHStore` | doesn't have | doesn't have |
* | `MountedSSHStore` | remote | doesn't have |
*
* @note
* Note how only the local implementations of `addPermRoot()` need
* `addIndirectRoot()`; that is what this class enforces. Without it,
* and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we
* would accidentally be allowing for a combinatorial explosion of
* possible implementations many of which make no sense. Having this and
* that invariant enforced cuts down that space.
*/
struct IndirectRootStore : public virtual LocalFSStore
{

159
src/libstore/local-store.cc
View file

@ -11,6 +11,7 @@
#include "finally.hh"
#include "compression.hh"
#include "signals.hh"
#include "posix-fs-canonicalise.hh"
#include <iostream>
#include <algorithm>
@ -581,164 +582,6 @@ void LocalStore::makeStoreWritable()
}
const time_t mtimeStore = 1; /* 1 second into the epoch */
static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
{
if (!S_ISLNK(st.st_mode)) {
/* Mask out all type related bits. */
mode_t mode = st.st_mode & ~S_IFMT;
if (mode != 0444 && mode != 0555) {
mode = (st.st_mode & S_IFMT)
| 0444
| (st.st_mode & S_IXUSR ? 0111 : 0);
if (chmod(path.c_str(), mode) == -1)
throw SysError("changing mode of '%1%' to %2$o", path, mode);
}
}
if (st.st_mtime != mtimeStore) {
struct timeval times[2];
times[0].tv_sec = st.st_atime;
times[0].tv_usec = 0;
times[1].tv_sec = mtimeStore;
times[1].tv_usec = 0;
#if HAVE_LUTIMES
if (lutimes(path.c_str(), times) == -1)
if (errno != ENOSYS ||
(!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
#else
if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
#endif
throw SysError("changing modification time of '%1%'", path);
}
}
void canonicaliseTimestampAndPermissions(const Path & path)
{
canonicaliseTimestampAndPermissions(path, lstat(path));
}
static void canonicalisePathMetaData_(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,
InodesSeen & inodesSeen)
{
checkInterrupt();
#if __APPLE__
/* Remove flags, in particular UF_IMMUTABLE which would prevent
the file from being garbage-collected. FIXME: Use
setattrlist() to remove other attributes as well. */
if (lchflags(path.c_str(), 0)) {
if (errno != ENOTSUP)
throw SysError("clearing flags of path '%1%'", path);
}
#endif
auto st = lstat(path);
/* Really make sure that the path is of a supported type. */
if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
throw Error("file '%1%' has an unsupported type", path);
#if __linux__
/* Remove extended attributes / ACLs. */
ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
if (eaSize < 0) {
if (errno != ENOTSUP && errno != ENODATA)
throw SysError("querying extended attributes of '%s'", path);
} else if (eaSize > 0) {
std::vector<char> eaBuf(eaSize);
if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0)
throw SysError("querying extended attributes of '%s'", path);
for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
if (settings.ignoredAcls.get().count(eaName)) continue;
if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
}
}
#endif
/* Fail if the file is not owned by the build user. This prevents
us from messing up the ownership/permissions of files
hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
However, ignore files that we chown'ed ourselves previously to
ensure that we don't fail on hard links within the same build
(i.e. "touch $out/foo; ln $out/foo $out/bar"). */
if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) {
if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
throw BuildError("invalid ownership on file '%1%'", path);
mode_t mode = st.st_mode & ~S_IFMT;
assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
return;
}
inodesSeen.insert(Inode(st.st_dev, st.st_ino));
canonicaliseTimestampAndPermissions(path, st);
/* Change ownership to the current uid. If it's a symlink, use
lchown if available, otherwise don't bother. Wrong ownership
of a symlink doesn't matter, since the owning user can't change
the symlink and can't delete it because the directory is not
writable. The only exception is top-level paths in the Nix
store (since that directory is group-writable for the Nix build
users group); we check for this case below. */
if (st.st_uid != geteuid()) {
#if HAVE_LCHOWN
if (lchown(path.c_str(), geteuid(), getegid()) == -1)
#else
if (!S_ISLNK(st.st_mode) &&
chown(path.c_str(), geteuid(), getegid()) == -1)
#endif
throw SysError("changing owner of '%1%' to %2%",
path, geteuid());
}
if (S_ISDIR(st.st_mode)) {
DirEntries entries = readDirectory(path);
for (auto & i : entries)
canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen);
}
}
void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,
InodesSeen & inodesSeen)
{
canonicalisePathMetaData_(path, uidRange, inodesSeen);
/* On platforms that don't have lchown(), the top-level path can't
be a symlink, since we can't change its ownership. */
auto st = lstat(path);
if (st.st_uid != geteuid()) {
assert(S_ISLNK(st.st_mode));
throw Error("wrong ownership of top-level store path '%1%'", path);
}
}
void canonicalisePathMetaData(const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange)
{
InodesSeen inodesSeen;
canonicalisePathMetaData(path, uidRange, inodesSeen);
}
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
{
experimentalFeatureSettings.require(Xp::CaDerivations);

34
src/libstore/local-store.hh
View file

@ -371,38 +371,4 @@ private:
friend struct DerivationGoal;
};
typedef std::pair<dev_t, ino_t> Inode;
typedef std::set<Inode> InodesSeen;
/**
* "Fix", or canonicalise, the meta-data of the files in a store path
* after it has been built. In particular:
*
* - the last modification date on each file is set to 1 (i.e.,
* 00:00:01 1/1/1970 UTC)
*
* - the permissions are set of 444 or 555 (i.e., read-only with or
* without execute permission; setuid bits etc. are cleared)
*
* - the owner and group are set to the Nix user and group, if we're
* running as root.
*
* If uidRange is not empty, this function will throw an error if it
* encounters files owned by a user outside of the closed interval
* [uidRange->first, uidRange->second].
*/
void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,
InodesSeen & inodesSeen);
void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange);
void canonicaliseTimestampAndPermissions(const Path & path);
MakeError(PathInUse, Error);
}

18
src/libstore/mounted-ssh-store.md Normal file
View file

@ -0,0 +1,18 @@
R"(
**Store URL format**: `mounted-ssh-ng://[username@]hostname`
Experimental store type that allows full access to a Nix store on a remote machine,
and additionally requires that store be mounted in the local file system.
The mounting of that store is not managed by Nix, and must by managed manually.
It could be accomplished with SSHFS or NFS, for example.
The local file system is used to optimize certain operations.
For example, rather than serializing Nix archives and sending over the Nix channel,
we can directly access the file system data via the mount-point.
The local file system is also used to make certain operations possible that wouldn't otherwise be.
For example, persistent GC roots can be created if they reside on the same file system as the remote store:
the remote side will create the symlinks necessary to avoid race conditions.
)"

1
src/libstore/optimise-store.cc
View file

@ -1,6 +1,7 @@
#include "local-store.hh"
#include "globals.hh"
#include "signals.hh"
#include "posix-fs-canonicalise.hh"
#include <cstdlib>
#include <cstring>

169
src/libstore/posix-fs-canonicalise.cc Normal file
View file

@ -0,0 +1,169 @@
#include <sys/xattr.h>
#include "posix-fs-canonicalise.hh"
#include "file-system.hh"
#include "signals.hh"
#include "util.hh"
#include "globals.hh"
#include "store-api.hh"
namespace nix {
const time_t mtimeStore = 1; /* 1 second into the epoch */
static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
{
if (!S_ISLNK(st.st_mode)) {
/* Mask out all type related bits. */
mode_t mode = st.st_mode & ~S_IFMT;
if (mode != 0444 && mode != 0555) {
mode = (st.st_mode & S_IFMT)
| 0444
| (st.st_mode & S_IXUSR ? 0111 : 0);
if (chmod(path.c_str(), mode) == -1)
throw SysError("changing mode of '%1%' to %2$o", path, mode);
}
}
if (st.st_mtime != mtimeStore) {
struct timeval times[2];
times[0].tv_sec = st.st_atime;
times[0].tv_usec = 0;
times[1].tv_sec = mtimeStore;
times[1].tv_usec = 0;
#if HAVE_LUTIMES
if (lutimes(path.c_str(), times) == -1)
if (errno != ENOSYS ||
(!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
#else
if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
#endif
throw SysError("changing modification time of '%1%'", path);
}
}
void canonicaliseTimestampAndPermissions(const Path & path)
{
canonicaliseTimestampAndPermissions(path, lstat(path));
}
static void canonicalisePathMetaData_(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,
InodesSeen & inodesSeen)
{
checkInterrupt();
#if __APPLE__
/* Remove flags, in particular UF_IMMUTABLE which would prevent
the file from being garbage-collected. FIXME: Use
setattrlist() to remove other attributes as well. */
if (lchflags(path.c_str(), 0)) {
if (errno != ENOTSUP)
throw SysError("clearing flags of path '%1%'", path);
}
#endif
auto st = lstat(path);
/* Really make sure that the path is of a supported type. */
if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
throw Error("file '%1%' has an unsupported type", path);
#if __linux__
/* Remove extended attributes / ACLs. */
ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
if (eaSize < 0) {
if (errno != ENOTSUP && errno != ENODATA)
throw SysError("querying extended attributes of '%s'", path);
} else if (eaSize > 0) {
std::vector<char> eaBuf(eaSize);
if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0)
throw SysError("querying extended attributes of '%s'", path);
for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
if (settings.ignoredAcls.get().count(eaName)) continue;
if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
}
}
#endif
/* Fail if the file is not owned by the build user. This prevents
us from messing up the ownership/permissions of files
hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
However, ignore files that we chown'ed ourselves previously to
ensure that we don't fail on hard links within the same build
(i.e. "touch $out/foo; ln $out/foo $out/bar"). */
if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) {
if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
throw BuildError("invalid ownership on file '%1%'", path);
mode_t mode = st.st_mode & ~S_IFMT;
assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
return;
}
inodesSeen.insert(Inode(st.st_dev, st.st_ino));
canonicaliseTimestampAndPermissions(path, st);
/* Change ownership to the current uid. If it's a symlink, use
lchown if available, otherwise don't bother. Wrong ownership
of a symlink doesn't matter, since the owning user can't change
the symlink and can't delete it because the directory is not
writable. The only exception is top-level paths in the Nix
store (since that directory is group-writable for the Nix build
users group); we check for this case below. */
if (st.st_uid != geteuid()) {
#if HAVE_LCHOWN
if (lchown(path.c_str(), geteuid(), getegid()) == -1)
#else
if (!S_ISLNK(st.st_mode) &&
chown(path.c_str(), geteuid(), getegid()) == -1)
#endif
throw SysError("changing owner of '%1%' to %2%",
path, geteuid());
}
if (S_ISDIR(st.st_mode)) {
DirEntries entries = readDirectory(path);
for (auto & i : entries)
canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen);
}
}
void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,
InodesSeen & inodesSeen)
{
canonicalisePathMetaData_(path, uidRange, inodesSeen);
/* On platforms that don't have lchown(), the top-level path can't
be a symlink, since we can't change its ownership. */
auto st = lstat(path);
if (st.st_uid != geteuid()) {
assert(S_ISLNK(st.st_mode));
throw Error("wrong ownership of top-level store path '%1%'", path);
}
}
void canonicalisePathMetaData(const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange)
{
InodesSeen inodesSeen;
canonicalisePathMetaData(path, uidRange, inodesSeen);
}
}

45
src/libstore/posix-fs-canonicalise.hh Normal file
View file

@ -0,0 +1,45 @@
#pragma once
///@file
#include <sys/stat.h>
#include <sys/time.h>
#include "types.hh"
#include "error.hh"
namespace nix {
typedef std::pair<dev_t, ino_t> Inode;
typedef std::set<Inode> InodesSeen;
/**
* "Fix", or canonicalise, the meta-data of the files in a store path
* after it has been built. In particular:
*
* - the last modification date on each file is set to 1 (i.e.,
* 00:00:01 1/1/1970 UTC)
*
* - the permissions are set of 444 or 555 (i.e., read-only with or
* without execute permission; setuid bits etc. are cleared)
*
* - the owner and group are set to the Nix user and group, if we're
* running as root.
*
* If uidRange is not empty, this function will throw an error if it
* encounters files owned by a user outside of the closed interval
* [uidRange->first, uidRange->second].
*/
void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange,
InodesSeen & inodesSeen);
void canonicalisePathMetaData(
const Path & path,
std::optional<std::pair<uid_t, uid_t>> uidRange);
void canonicaliseTimestampAndPermissions(const Path & path);
MakeError(PathInUse, Error);
}

123
src/libstore/ssh-store.cc
View file

@ -3,9 +3,10 @@
#include "local-fs-store.hh"
#include "remote-store.hh"
#include "remote-store-connection.hh"
#include "remote-fs-accessor.hh"
#include "source-accessor.hh"
#include "archive.hh"
#include "worker-protocol.hh"
#include "worker-protocol-impl.hh"
#include "pool.hh"
#include "ssh.hh"
@ -78,6 +79,8 @@ protected:
std::string host;
std::vector<std::string> extraRemoteProgramArgs;
SSHMaster master;
void setOptions(RemoteStore::Connection & conn) override
@ -91,6 +94,121 @@ protected:
};
};
struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig
{
using SSHStoreConfig::SSHStoreConfig;
using LocalFSStoreConfig::LocalFSStoreConfig;
MountedSSHStoreConfig(StringMap params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
{
}
const std::string name() override { return "Experimental SSH Store with filesytem mounted"; }
std::string doc() override
{
return
#include "mounted-ssh-store.md"
;
}
std::optional<ExperimentalFeature> experimentalFeature() const override
{
return ExperimentalFeature::MountedSSHStore;
}
};
/**
* The mounted ssh store assumes that filesystems on the remote host are
* shared with the local host. This means that the remote nix store is
* available locally and is therefore treated as a local filesystem
* store.
*
* MountedSSHStore is very similar to UDSRemoteStore --- ignoring the
* superficial differnce of SSH vs Unix domain sockets, they both are
* accessing remote stores, and they both assume the store will be
* mounted in the local filesystem.
*
* The difference lies in how they manage GC roots. See addPermRoot
* below for details.
*/
class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore
{
public:
MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
, MountedSSHStoreConfig(params)
, Store(params)
, RemoteStore(params)
, SSHStore(scheme, host, params)
, LocalFSStore(params)
{
extraRemoteProgramArgs = {
"--process-ops",
};
}
static std::set<std::string> uriSchemes()
{
return {"mounted-ssh-ng"};
}
std::string getUri() override
{
return *uriSchemes().begin() + "://" + host;
}
void narFromPath(const StorePath & path, Sink & sink) override
{
return LocalFSStore::narFromPath(path, sink);
}
ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
{
return LocalFSStore::getFSAccessor(requireValidPath);
}
std::optional<std::string> getBuildLogExact(const StorePath & path) override
{
return LocalFSStore::getBuildLogExact(path);
}
/**
* This is the key difference from UDSRemoteStore: UDSRemote store
* has the client create the direct root, and the remote side create
* the indirect root.
*
* We could also do that, but the race conditions (will the remote
* side see the direct root the client made?) seems bigger.
*
* In addition, the remote-side will have a process associated with
* the authenticating user handling the connection (even if there
* is a system-wide daemon or similar). This process can safely make
* the direct and indirect roots without there being such a risk of
* privilege escalation / symlinks in directories owned by the
* originating requester that they cannot delete.
*/
Path addPermRoot(const StorePath & path, const Path & gcRoot) override
{
auto conn(getConnection());
conn->to << WorkerProto::Op::AddPermRoot;
WorkerProto::write(*this, *conn, path);
WorkerProto::write(*this, *conn, gcRoot);
conn.processStderr();
return readString(conn->from);
}
};
ref<RemoteStore::Connection> SSHStore::openConnection()
{
auto conn = make_ref<Connection>();
@ -98,6 +216,8 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
std::string command = remoteProgram + " --stdio";
if (remoteStore.get() != "")
command += " --store " + shellEscape(remoteStore.get());
for (auto & arg : extraRemoteProgramArgs)
command += " " + shellEscape(arg);
conn->sshConn = master.startCommand(command);
conn->to = FdSink(conn->sshConn->in.get());
@ -106,5 +226,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
}
static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore;
static RegisterStoreImplementation<MountedSSHStore, MountedSSHStoreConfig> regMountedSSHStore;
}

3
src/libstore/worker-protocol.hh
View file

@ -9,7 +9,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION (1 << 8 | 35)
#define PROTOCOL_VERSION (1 << 8 | 36)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -161,6 +161,7 @@ enum struct WorkerProto::Op : uint64_t
AddMultipleToStore = 44,
AddBuildLog = 45,
BuildPathsWithResults = 46,
AddPermRoot = 47,
};
/**

7
src/libutil/experimental-features.cc
View file

@ -262,6 +262,13 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting.
)",
},
{
.tag = Xp::MountedSSHStore,
.name = "mounted-ssh-store",
.description = R"(
Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
)",
},
{
.tag = Xp::VerifiedFetches,
.name = "verified-fetches",

1
src/libutil/experimental-features.hh
View file

@ -34,6 +34,7 @@ enum struct ExperimentalFeature
ParseTomlTimestamps,
ReadOnlyLocalStore,
ConfigurableImpureEnv,
MountedSSHStore,
VerifiedFetches,
};

2
src/libutil/posix-source-accessor.hh
View file

@ -7,7 +7,7 @@ namespace nix {
/**
* A source accessor that uses the Unix filesystem.
*/
struct PosixSourceAccessor : SourceAccessor
struct PosixSourceAccessor : virtual SourceAccessor
{
/**
* The most recent mtime seen by lstat(). This is a hack to

9
src/libutil/source-accessor.cc
View file

@ -7,6 +7,7 @@ static std::atomic<size_t> nextNumber{0};
SourceAccessor::SourceAccessor()
: number(++nextNumber)
, displayPrefix{"«unknown»"}
{
}
@ -55,9 +56,15 @@ SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path)
throw Error("path '%s' does not exist", showPath(path));
}
void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix)
{
this->displayPrefix = std::move(displayPrefix);
this->displaySuffix = std::move(displaySuffix);
}
std::string SourceAccessor::showPath(const CanonPath & path)
{
return path.abs();
return displayPrefix + path.abs() + displaySuffix;
}
}

4
src/libutil/source-accessor.hh
View file

@ -17,6 +17,8 @@ struct SourceAccessor
{
const size_t number;
std::string displayPrefix, displaySuffix;
SourceAccessor();
virtual ~SourceAccessor()
@ -117,6 +119,8 @@ struct SourceAccessor
return number < x.number;
}
void setPathDisplay(std::string displayPrefix, std::string displaySuffix = "");
virtual std::string showPath(const CanonPath & path);
};

9
src/nix-env/nix-env.cc
View file

@ -922,7 +922,7 @@ static VersionDiff compareVersionAgainstSet(
}
static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printMeta)
static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printDrvPath, bool printMeta)
{
using nlohmann::json;
json topObj = json::object();
@ -953,6 +953,11 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
}
}
if (printDrvPath) {
auto drvPath = i.queryDrvPath();
if (drvPath) pkgObj["drvPath"] = globals.state->store->printStorePath(*drvPath);
}
if (printMeta) {
json &metaObj = pkgObj["meta"];
metaObj = json::object();
@ -1079,7 +1084,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
/* Print the desired columns, or XML output. */
if (jsonOutput) {
queryJSON(globals, elems, printOutPath, printMeta);
queryJSON(globals, elems, printOutPath, printDrvPath, printMeta);
cout << '\n';
return;
}

1
src/nix-store/nix-store.cc
View file

@ -14,6 +14,7 @@
#include "graphml.hh"
#include "legacy.hh"
#include "path-with-outputs.hh"
#include "posix-fs-canonicalise.hh"
#include <iostream>
#include <algorithm>

33
src/nix/daemon.cc
View file

@ -443,16 +443,23 @@ static void processStdioConnection(ref<Store> store, TrustedFlag trustClient)
*
* @param forceTrustClientOpt See `daemonLoop()` and the parameter with
* the same name over there for details.
*
* @param procesOps Whether to force processing ops even if the next
* store also is a remote store and could process it directly.
*/
static void runDaemon(bool stdio, std::optional<TrustedFlag> forceTrustClientOpt)
static void runDaemon(bool stdio, std::optional<TrustedFlag> forceTrustClientOpt, bool processOps)
{
if (stdio) {
auto store = openUncachedStore();
std::shared_ptr<RemoteStore> remoteStore;
// If --force-untrusted is passed, we cannot forward the connection and
// must process it ourselves (before delegating to the next store) to
// force untrusting the client.
if (auto remoteStore = store.dynamic_pointer_cast<RemoteStore>(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted))
processOps |= !forceTrustClientOpt || *forceTrustClientOpt != NotTrusted;
if (!processOps && (remoteStore = store.dynamic_pointer_cast<RemoteStore>()))
forwardStdioConnection(*remoteStore);
else
// `Trusted` is passed in the auto (no override case) because we
@ -468,6 +475,7 @@ static int main_nix_daemon(int argc, char * * argv)
{
auto stdio = false;
std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
auto processOps = false;
parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
if (*arg == "--daemon")
@ -487,11 +495,14 @@ static int main_nix_daemon(int argc, char * * argv)
} else if (*arg == "--default-trust") {
experimentalFeatureSettings.require(Xp::DaemonTrustOverride);
isTrustedOpt = std::nullopt;
} else if (*arg == "--process-ops") {
experimentalFeatureSettings.require(Xp::MountedSSHStore);
processOps = true;
} else return false;
return true;
});
runDaemon(stdio, isTrustedOpt);
runDaemon(stdio, isTrustedOpt, processOps);
return 0;
}
@ -503,6 +514,7 @@ struct CmdDaemon : StoreCommand
{
bool stdio = false;
std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
bool processOps = false;
CmdDaemon()
{
@ -538,6 +550,19 @@ struct CmdDaemon : StoreCommand
}},
.experimentalFeature = Xp::DaemonTrustOverride,
});
addFlag({
.longName = "process-ops",
.description = R"(
Forces the daemon to process received commands itself rather than forwarding the commands straight to the remote store.
This is useful for the `mounted-ssh://` store where some actions need to be performed on the remote end but as connected user, and not as the user of the underlying daemon on the remote end.
)",
.handler = {[&]() {
processOps = true;
}},
.experimentalFeature = Xp::MountedSSHStore,
});
}
std::string description() override
@ -556,7 +581,7 @@ struct CmdDaemon : StoreCommand
void run(ref<Store> store) override
{
runDaemon(stdio, isTrustedOpt);
runDaemon(stdio, isTrustedOpt, processOps);
}
};

2
src/nix/flake-init.md
View file

@ -30,7 +30,7 @@ using `-t`.
# Template definitions
A flake can declare templates through its `templates` output
attribute. A template has two attributes:
attribute. A template has the following attributes:
* `description`: A one-line description of the template, in CommonMark
syntax.

2
src/nix/path-info.cc
View file

@ -54,7 +54,7 @@ static json pathInfoToJSON(
jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure);
if (auto * narInfo = dynamic_cast<const NarInfo *>(&*info)) {
if (dynamic_cast<const NarInfo *>(&*info)) {
uint64_t totalDownloadSize = 0;
for (auto & p : closure) {
auto depInfo = store.queryPathInfo(p);

22
tests/functional/build-remote-with-mounted-ssh-ng.sh Normal file
View file

@ -0,0 +1,22 @@
source common.sh
requireSandboxSupport
[[ $busybox =~ busybox ]] || skipTest "no busybox"
enableFeatures mounted-ssh-store
nix build -Lvf simple.nix \
--arg busybox $busybox \
--out-link $TEST_ROOT/result-from-remote \
--store mounted-ssh-ng://localhost
nix build -Lvf simple.nix \
--arg busybox $busybox \
--out-link $TEST_ROOT/result-from-remote-new-cli \
--store 'mounted-ssh-ng://localhost?remote-program=nix daemon'
# This verifies that the out link was actually created and valid. The ability
# to create out links (permanent gc roots) is the distinguishing feature of
# the mounted-ssh-ng store.
cat $TEST_ROOT/result-from-remote/hello | grepQuiet 'Hello World!'
cat $TEST_ROOT/result-from-remote-new-cli/hello | grepQuiet 'Hello World!'

4
tests/functional/fetchGit.sh
View file

@ -51,9 +51,7 @@ git -C $repo add differentbranch
git -C $repo commit -m 'Test2'
git -C $repo checkout master
devrev=$(git -C $repo rev-parse devtest)
out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$?
[[ $status == 1 ]]
[[ $out =~ 'Cannot find Git revision' ]]
nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }"
[[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]]

1
tests/functional/local.mk
View file

@ -69,6 +69,7 @@ nix_tests = \
build-remote-trustless-should-pass-2.sh \
build-remote-trustless-should-pass-3.sh \
build-remote-trustless-should-fail-0.sh \
build-remote-with-mounted-ssh-ng.sh \
nar-access.sh \
pure-eval.sh \
eval.sh \

1
tests/functional/user-envs.sh
View file

@ -26,6 +26,7 @@ nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name ==
.outputName == "out",
(.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1"))
] | all'
nix-env -f ./user-envs.nix -qa --json --drv-path | jq -e '.[] | select(.name == "bar-0.1") | (.drvPath | test("'$NIX_STORE_DIR'.*-0\\.1\\.drv"))'
# Query descriptions.
nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly