From 06b8902562089811e5724aa0b8d719f891ab73f2 Mon Sep 17 00:00:00 2001 From: Matej Urbas Date: Sat, 15 Apr 2023 11:02:41 +0100 Subject: [PATCH] MountedSSHStore: stores on shared filesystems --- src/libstore/indirect-root-store.hh | 24 ++++++ src/libstore/mounted-ssh-store.md | 18 ++++ src/libstore/ssh-store.cc | 123 +++++++++++++++++++++++++++- 3 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 src/libstore/mounted-ssh-store.md diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh index 59e45af45..c11679fe8 100644 --- a/src/libstore/indirect-root-store.hh +++ b/src/libstore/indirect-root-store.hh @@ -11,6 +11,30 @@ namespace nix { * reference. * * See methods for details on the operations it represents. + * + * @note + * To understand the purpose of this class, it might help to do some + * "closed-world" rather than "open-world" reasoning, and consider the + * problem it solved for us. This class was factored out from + * `LocalFSStore` in order to support the following table, which + * contains 4 concrete store types (non-abstract classes, exposed to the + * user), and how they implemented the two GC root methods: + * + * @note + * | | `addPermRoot()` | `addIndirectRoot()` | + * |-------------------|-----------------|---------------------| + * | `LocalStore` | local | local | + * | `UDSRemoteStore` | local | remote | + * | `SSHStore` | doesn't have | doesn't have | + * | `MountedSSHStore` | remote | doesn't have | + * + * @note + * Note how only the local implementations of `addPermRoot()` need + * `addIndirectRoot()`; that is what this class enforces. Without it, + * and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we + * would accidentally be allowing for a combinatorial explosion of + * possible implementations many of which make no sense. Having this and + * that invariant enforced cuts down that space. */ struct IndirectRootStore : public virtual LocalFSStore { diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md new file mode 100644 index 000000000..1ebfe3081 --- /dev/null +++ b/src/libstore/mounted-ssh-store.md @@ -0,0 +1,18 @@ +R"( + +**Store URL format**: `mounted-ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a remote machine, +and additionally requires that store be mounted in the local file system. + +The mounting of that store is not managed by Nix, and must by managed manually. +It could be accomplished with SSHFS or NFS, for example. + +The local file system is used to optimize certain operations. +For example, rather than serializing Nix archives and sending over the Nix channel, +we can directly access the file system data via the mount-point. + +The local file system is also used to make certain operations possible that wouldn't otherwise be. +For example, persistent GC roots can be created if they reside on the same file system as the remote store: +the remote side will create the symlinks necessary to avoid race conditions. +)" diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 4a6aad449..d4c8ab5b2 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -3,9 +3,10 @@ #include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "remote-fs-accessor.hh" +#include "source-accessor.hh" #include "archive.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "pool.hh" #include "ssh.hh" @@ -78,6 +79,8 @@ protected: std::string host; + std::vector extraRemoteProgramArgs; + SSHMaster master; void setOptions(RemoteStore::Connection & conn) override @@ -91,6 +94,121 @@ protected: }; }; +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using SSHStoreConfig::SSHStoreConfig; + using LocalFSStoreConfig::LocalFSStoreConfig; + + MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + { + } + + const std::string name() override { return "Experimental SSH Store with filesytem mounted"; } + + std::string doc() override + { + return + #include "mounted-ssh-store.md" + ; + } + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + +/** + * The mounted ssh store assumes that filesystems on the remote host are + * shared with the local host. This means that the remote nix store is + * available locally and is therefore treated as a local filesystem + * store. + * + * MountedSSHStore is very similar to UDSRemoteStore --- ignoring the + * superficial differnce of SSH vs Unix domain sockets, they both are + * accessing remote stores, and they both assume the store will be + * mounted in the local filesystem. + * + * The difference lies in how they manage GC roots. See addPermRoot + * below for details. + */ +class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +{ +public: + + MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + , MountedSSHStoreConfig(params) + , Store(params) + , RemoteStore(params) + , SSHStore(scheme, host, params) + , LocalFSStore(params) + { + extraRemoteProgramArgs = { + "--process-ops", + }; + } + + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + + std::string getUri() override + { + return *uriSchemes().begin() + "://" + host; + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + return LocalFSStore::narFromPath(path, sink); + } + + ref getFSAccessor(bool requireValidPath) override + { + return LocalFSStore::getFSAccessor(requireValidPath); + } + + std::optional getBuildLogExact(const StorePath & path) override + { + return LocalFSStore::getBuildLogExact(path); + } + + /** + * This is the key difference from UDSRemoteStore: UDSRemote store + * has the client create the direct root, and the remote side create + * the indirect root. + * + * We could also do that, but the race conditions (will the remote + * side see the direct root the client made?) seems bigger. + * + * In addition, the remote-side will have a process associated with + * the authenticating user handling the connection (even if there + * is a system-wide daemon or similar). This process can safely make + * the direct and indirect roots without there being such a risk of + * privilege escalation / symlinks in directories owned by the + * originating requester that they cannot delete. + */ + Path addPermRoot(const StorePath & path, const Path & gcRoot) override + { + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddPermRoot; + WorkerProto::write(*this, *conn, path); + WorkerProto::write(*this, *conn, gcRoot); + conn.processStderr(); + return readString(conn->from); + } +}; + ref SSHStore::openConnection() { auto conn = make_ref(); @@ -98,6 +216,8 @@ ref SSHStore::openConnection() std::string command = remoteProgram + " --stdio"; if (remoteStore.get() != "") command += " --store " + shellEscape(remoteStore.get()); + for (auto & arg : extraRemoteProgramArgs) + command += " " + shellEscape(arg); conn->sshConn = master.startCommand(command); conn->to = FdSink(conn->sshConn->in.get()); @@ -106,5 +226,6 @@ ref SSHStore::openConnection() } static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; }