mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-25 23:36:16 +02:00
Create header for LegacySSHStore
In https://github.com/NixOS/nix/pull/6134#issuecomment-1079199888, @thuffschmitt proposed exposing `LegacySSHStore` in Nix for deduplication with Hydra, at least temporarily. I think that is a good idea. Note that the diff will look bad unless one ignores whitespace! Also try this locally: ```shell-session git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.cc git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.hh ```
This commit is contained in:
John Ericson
parent
b7e016ab24
commit
deadb3bfe9
2 changed files with 466 additions and 392 deletions
|
|
@ -1,3 +1,4 @@
|
||||||
|
#include "legacy-ssh-store.hh"
|
||||||
#include "ssh-store-config.hh"
|
#include "ssh-store-config.hh"
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
#include "pool.hh"
|
#include "pool.hh"
|
||||||
|
|
@ -13,414 +14,355 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
|
std::string LegacySSHStoreConfig::doc()
|
||||||
{
|
{
|
||||||
using CommonSSHStoreConfig::CommonSSHStoreConfig;
|
return
|
||||||
|
#include "legacy-ssh-store.md"
|
||||||
|
;
|
||||||
|
}
|
||||||
|
|
||||||
const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
|
|
||||||
"Path to the `nix-store` executable on the remote machine."};
|
|
||||||
|
|
||||||
const Setting<int> maxConnections{this, 1, "max-connections",
|
struct LegacySSHStore::Connection
|
||||||
"Maximum number of concurrent SSH connections."};
|
|
||||||
|
|
||||||
const std::string name() override { return "SSH Store"; }
|
|
||||||
|
|
||||||
std::string doc() override
|
|
||||||
{
|
|
||||||
return
|
|
||||||
#include "legacy-ssh-store.md"
|
|
||||||
;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
|
|
||||||
{
|
{
|
||||||
// Hack for getting remote build log output.
|
std::unique_ptr<SSHMaster::Connection> sshConn;
|
||||||
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
|
FdSink to;
|
||||||
// the documentation
|
FdSource from;
|
||||||
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
ServeProto::Version remoteVersion;
|
||||||
|
bool good = true;
|
||||||
struct Connection
|
|
||||||
{
|
|
||||||
std::unique_ptr<SSHMaster::Connection> sshConn;
|
|
||||||
FdSink to;
|
|
||||||
FdSource from;
|
|
||||||
ServeProto::Version remoteVersion;
|
|
||||||
bool good = true;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Coercion to `ServeProto::ReadConn`. This makes it easy to use the
|
|
||||||
* factored out serve protocol searlizers with a
|
|
||||||
* `LegacySSHStore::Connection`.
|
|
||||||
*
|
|
||||||
* The serve protocol connection types are unidirectional, unlike
|
|
||||||
* this type.
|
|
||||||
*/
|
|
||||||
operator ServeProto::ReadConn ()
|
|
||||||
{
|
|
||||||
return ServeProto::ReadConn {
|
|
||||||
.from = from,
|
|
||||||
.version = remoteVersion,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Coercion to `ServeProto::WriteConn`. This makes it easy to use the
|
|
||||||
* factored out serve protocol searlizers with a
|
|
||||||
* `LegacySSHStore::Connection`.
|
|
||||||
*
|
|
||||||
* The serve protocol connection types are unidirectional, unlike
|
|
||||||
* this type.
|
|
||||||
*/
|
|
||||||
operator ServeProto::WriteConn ()
|
|
||||||
{
|
|
||||||
return ServeProto::WriteConn {
|
|
||||||
.to = to,
|
|
||||||
.version = remoteVersion,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
std::string host;
|
|
||||||
|
|
||||||
ref<Pool<Connection>> connections;
|
|
||||||
|
|
||||||
SSHMaster master;
|
|
||||||
|
|
||||||
static std::set<std::string> uriSchemes() { return {"ssh"}; }
|
|
||||||
|
|
||||||
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
|
|
||||||
: StoreConfig(params)
|
|
||||||
, CommonSSHStoreConfig(params)
|
|
||||||
, LegacySSHStoreConfig(params)
|
|
||||||
, Store(params)
|
|
||||||
, host(host)
|
|
||||||
, connections(make_ref<Pool<Connection>>(
|
|
||||||
std::max(1, (int) maxConnections),
|
|
||||||
[this]() { return openConnection(); },
|
|
||||||
[](const ref<Connection> & r) { return r->good; }
|
|
||||||
))
|
|
||||||
, master(
|
|
||||||
host,
|
|
||||||
sshKey,
|
|
||||||
sshPublicHostKey,
|
|
||||||
// Use SSH master only if using more than 1 connection.
|
|
||||||
connections->capacity() > 1,
|
|
||||||
compress,
|
|
||||||
logFD)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
ref<Connection> openConnection()
|
|
||||||
{
|
|
||||||
auto conn = make_ref<Connection>();
|
|
||||||
conn->sshConn = master.startCommand(
|
|
||||||
fmt("%s --serve --write", remoteProgram)
|
|
||||||
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
|
|
||||||
conn->to = FdSink(conn->sshConn->in.get());
|
|
||||||
conn->from = FdSource(conn->sshConn->out.get());
|
|
||||||
|
|
||||||
try {
|
|
||||||
conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
|
|
||||||
conn->to.flush();
|
|
||||||
|
|
||||||
StringSink saved;
|
|
||||||
try {
|
|
||||||
TeeSource tee(conn->from, saved);
|
|
||||||
unsigned int magic = readInt(tee);
|
|
||||||
if (magic != SERVE_MAGIC_2)
|
|
||||||
throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
|
|
||||||
} catch (SerialisationError & e) {
|
|
||||||
/* In case the other side is waiting for our input,
|
|
||||||
close it. */
|
|
||||||
conn->sshConn->in.close();
|
|
||||||
auto msg = conn->from.drain();
|
|
||||||
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
|
|
||||||
host, chomp(saved.s + msg));
|
|
||||||
}
|
|
||||||
conn->remoteVersion = readInt(conn->from);
|
|
||||||
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
|
|
||||||
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
|
|
||||||
|
|
||||||
} catch (EndOfFile & e) {
|
|
||||||
throw Error("cannot connect to '%1%'", host);
|
|
||||||
}
|
|
||||||
|
|
||||||
return conn;
|
|
||||||
};
|
|
||||||
|
|
||||||
std::string getUri() override
|
|
||||||
{
|
|
||||||
return *uriSchemes().begin() + "://" + host;
|
|
||||||
}
|
|
||||||
|
|
||||||
void queryPathInfoUncached(const StorePath & path,
|
|
||||||
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
auto conn(connections->get());
|
|
||||||
|
|
||||||
/* No longer support missing NAR hash */
|
|
||||||
assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
|
|
||||||
|
|
||||||
debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
|
|
||||||
|
|
||||||
conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)};
|
|
||||||
conn->to.flush();
|
|
||||||
|
|
||||||
auto p = readString(conn->from);
|
|
||||||
if (p.empty()) return callback(nullptr);
|
|
||||||
auto path2 = parseStorePath(p);
|
|
||||||
assert(path == path2);
|
|
||||||
auto info = std::make_shared<ValidPathInfo>(
|
|
||||||
path,
|
|
||||||
ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
|
|
||||||
|
|
||||||
if (info->narHash == Hash::dummy)
|
|
||||||
throw Error("NAR hash is now mandatory");
|
|
||||||
|
|
||||||
auto s = readString(conn->from);
|
|
||||||
assert(s == "");
|
|
||||||
|
|
||||||
callback(std::move(info));
|
|
||||||
} catch (...) { callback.rethrow(); }
|
|
||||||
}
|
|
||||||
|
|
||||||
void addToStore(const ValidPathInfo & info, Source & source,
|
|
||||||
RepairFlag repair, CheckSigsFlag checkSigs) override
|
|
||||||
{
|
|
||||||
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
|
|
||||||
|
|
||||||
auto conn(connections->get());
|
|
||||||
|
|
||||||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
|
|
||||||
|
|
||||||
conn->to
|
|
||||||
<< ServeProto::Command::AddToStoreNar
|
|
||||||
<< printStorePath(info.path)
|
|
||||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
|
||||||
<< info.narHash.to_string(HashFormat::Base16, false);
|
|
||||||
ServeProto::write(*this, *conn, info.references);
|
|
||||||
conn->to
|
|
||||||
<< info.registrationTime
|
|
||||||
<< info.narSize
|
|
||||||
<< info.ultimate
|
|
||||||
<< info.sigs
|
|
||||||
<< renderContentAddress(info.ca);
|
|
||||||
try {
|
|
||||||
copyNAR(source, conn->to);
|
|
||||||
} catch (...) {
|
|
||||||
conn->good = false;
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
conn->to.flush();
|
|
||||||
|
|
||||||
} else {
|
|
||||||
|
|
||||||
conn->to
|
|
||||||
<< ServeProto::Command::ImportPaths
|
|
||||||
<< 1;
|
|
||||||
try {
|
|
||||||
copyNAR(source, conn->to);
|
|
||||||
} catch (...) {
|
|
||||||
conn->good = false;
|
|
||||||
throw;
|
|
||||||
}
|
|
||||||
conn->to
|
|
||||||
<< exportMagic
|
|
||||||
<< printStorePath(info.path);
|
|
||||||
ServeProto::write(*this, *conn, info.references);
|
|
||||||
conn->to
|
|
||||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
|
||||||
<< 0
|
|
||||||
<< 0;
|
|
||||||
conn->to.flush();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if (readInt(conn->from) != 1)
|
|
||||||
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
|
|
||||||
}
|
|
||||||
|
|
||||||
void narFromPath(const StorePath & path, Sink & sink) override
|
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
|
||||||
|
|
||||||
conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
|
|
||||||
conn->to.flush();
|
|
||||||
copyNAR(conn->from, sink);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
|
|
||||||
{ unsupported("queryPathFromHashPart"); }
|
|
||||||
|
|
||||||
StorePath addToStore(
|
|
||||||
std::string_view name,
|
|
||||||
const Path & srcPath,
|
|
||||||
FileIngestionMethod method,
|
|
||||||
HashAlgorithm hashAlgo,
|
|
||||||
PathFilter & filter,
|
|
||||||
RepairFlag repair,
|
|
||||||
const StorePathSet & references) override
|
|
||||||
{ unsupported("addToStore"); }
|
|
||||||
|
|
||||||
StorePath addTextToStore(
|
|
||||||
std::string_view name,
|
|
||||||
std::string_view s,
|
|
||||||
const StorePathSet & references,
|
|
||||||
RepairFlag repair) override
|
|
||||||
{ unsupported("addTextToStore"); }
|
|
||||||
|
|
||||||
private:
|
|
||||||
|
|
||||||
void putBuildSettings(Connection & conn)
|
|
||||||
{
|
|
||||||
ServeProto::write(*this, conn, ServeProto::BuildOptions {
|
|
||||||
.maxSilentTime = settings.maxSilentTime,
|
|
||||||
.buildTimeout = settings.buildTimeout,
|
|
||||||
.maxLogSize = settings.maxLogSize,
|
|
||||||
.nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
|
|
||||||
.enforceDeterminism = 0,
|
|
||||||
.keepFailed = settings.keepFailed,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
|
||||||
BuildMode buildMode) override
|
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
|
||||||
|
|
||||||
conn->to
|
|
||||||
<< ServeProto::Command::BuildDerivation
|
|
||||||
<< printStorePath(drvPath);
|
|
||||||
writeDerivation(conn->to, *this, drv);
|
|
||||||
|
|
||||||
putBuildSettings(*conn);
|
|
||||||
|
|
||||||
conn->to.flush();
|
|
||||||
|
|
||||||
return ServeProto::Serialise<BuildResult>::read(*this, *conn);
|
|
||||||
}
|
|
||||||
|
|
||||||
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
|
|
||||||
{
|
|
||||||
if (evalStore && evalStore.get() != this)
|
|
||||||
throw Error("building on an SSH store is incompatible with '--eval-store'");
|
|
||||||
|
|
||||||
auto conn(connections->get());
|
|
||||||
|
|
||||||
conn->to << ServeProto::Command::BuildPaths;
|
|
||||||
Strings ss;
|
|
||||||
for (auto & p : drvPaths) {
|
|
||||||
auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
|
|
||||||
std::visit(overloaded {
|
|
||||||
[&](const StorePathWithOutputs & s) {
|
|
||||||
ss.push_back(s.to_string(*this));
|
|
||||||
},
|
|
||||||
[&](const StorePath & drvPath) {
|
|
||||||
throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
|
|
||||||
},
|
|
||||||
[&](std::monostate) {
|
|
||||||
throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://");
|
|
||||||
},
|
|
||||||
}, sOrDrvPath);
|
|
||||||
}
|
|
||||||
conn->to << ss;
|
|
||||||
|
|
||||||
putBuildSettings(*conn);
|
|
||||||
|
|
||||||
conn->to.flush();
|
|
||||||
|
|
||||||
BuildResult result;
|
|
||||||
result.status = (BuildResult::Status) readInt(conn->from);
|
|
||||||
|
|
||||||
if (!result.success()) {
|
|
||||||
conn->from >> result.errorMsg;
|
|
||||||
throw Error(result.status, result.errorMsg);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void ensurePath(const StorePath & path) override
|
|
||||||
{ unsupported("ensurePath"); }
|
|
||||||
|
|
||||||
virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
|
|
||||||
{ unsupported("getFSAccessor"); }
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The default instance would schedule the work on the client side, but
|
* Coercion to `ServeProto::ReadConn`. This makes it easy to use the
|
||||||
* for consistency with `buildPaths` and `buildDerivation` it should happen
|
* factored out serve protocol searlizers with a
|
||||||
* on the remote side.
|
* `LegacySSHStore::Connection`.
|
||||||
*
|
*
|
||||||
* We make this fail for now so we can add implement this properly later
|
* The serve protocol connection types are unidirectional, unlike
|
||||||
* without it being a breaking change.
|
* this type.
|
||||||
*/
|
*/
|
||||||
void repairPath(const StorePath & path) override
|
operator ServeProto::ReadConn ()
|
||||||
{ unsupported("repairPath"); }
|
|
||||||
|
|
||||||
void computeFSClosure(const StorePathSet & paths,
|
|
||||||
StorePathSet & out, bool flipDirection = false,
|
|
||||||
bool includeOutputs = false, bool includeDerivers = false) override
|
|
||||||
{
|
{
|
||||||
if (flipDirection || includeDerivers) {
|
return ServeProto::ReadConn {
|
||||||
Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
|
.from = from,
|
||||||
return;
|
.version = remoteVersion,
|
||||||
}
|
};
|
||||||
|
|
||||||
auto conn(connections->get());
|
|
||||||
|
|
||||||
conn->to
|
|
||||||
<< ServeProto::Command::QueryClosure
|
|
||||||
<< includeOutputs;
|
|
||||||
ServeProto::write(*this, *conn, paths);
|
|
||||||
conn->to.flush();
|
|
||||||
|
|
||||||
for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn))
|
|
||||||
out.insert(i);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
StorePathSet queryValidPaths(const StorePathSet & paths,
|
/*
|
||||||
SubstituteFlag maybeSubstitute = NoSubstitute) override
|
* Coercion to `ServeProto::WriteConn`. This makes it easy to use the
|
||||||
{
|
* factored out serve protocol searlizers with a
|
||||||
auto conn(connections->get());
|
* `LegacySSHStore::Connection`.
|
||||||
|
*
|
||||||
conn->to
|
* The serve protocol connection types are unidirectional, unlike
|
||||||
<< ServeProto::Command::QueryValidPaths
|
* this type.
|
||||||
<< false // lock
|
|
||||||
<< maybeSubstitute;
|
|
||||||
ServeProto::write(*this, *conn, paths);
|
|
||||||
conn->to.flush();
|
|
||||||
|
|
||||||
return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
|
|
||||||
}
|
|
||||||
|
|
||||||
void connect() override
|
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
|
||||||
}
|
|
||||||
|
|
||||||
unsigned int getProtocol() override
|
|
||||||
{
|
|
||||||
auto conn(connections->get());
|
|
||||||
return conn->remoteVersion;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The legacy ssh protocol doesn't support checking for trusted-user.
|
|
||||||
* Try using ssh-ng:// instead if you want to know.
|
|
||||||
*/
|
*/
|
||||||
std::optional<TrustedFlag> isTrustedClient() override
|
operator ServeProto::WriteConn ()
|
||||||
{
|
{
|
||||||
return std::nullopt;
|
return ServeProto::WriteConn {
|
||||||
|
.to = to,
|
||||||
|
.version = remoteVersion,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
void queryRealisationUncached(const DrvOutput &,
|
|
||||||
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
|
|
||||||
// TODO: Implement
|
|
||||||
{ unsupported("queryRealisation"); }
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
|
||||||
|
: StoreConfig(params)
|
||||||
|
, CommonSSHStoreConfig(params)
|
||||||
|
, LegacySSHStoreConfig(params)
|
||||||
|
, Store(params)
|
||||||
|
, host(host)
|
||||||
|
, connections(make_ref<Pool<Connection>>(
|
||||||
|
std::max(1, (int) maxConnections),
|
||||||
|
[this]() { return openConnection(); },
|
||||||
|
[](const ref<Connection> & r) { return r->good; }
|
||||||
|
))
|
||||||
|
, master(
|
||||||
|
host,
|
||||||
|
sshKey,
|
||||||
|
sshPublicHostKey,
|
||||||
|
// Use SSH master only if using more than 1 connection.
|
||||||
|
connections->capacity() > 1,
|
||||||
|
compress,
|
||||||
|
logFD)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
|
||||||
|
{
|
||||||
|
auto conn = make_ref<Connection>();
|
||||||
|
conn->sshConn = master.startCommand(
|
||||||
|
fmt("%s --serve --write", remoteProgram)
|
||||||
|
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
|
||||||
|
conn->to = FdSink(conn->sshConn->in.get());
|
||||||
|
conn->from = FdSource(conn->sshConn->out.get());
|
||||||
|
|
||||||
|
try {
|
||||||
|
conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
|
||||||
|
conn->to.flush();
|
||||||
|
|
||||||
|
StringSink saved;
|
||||||
|
try {
|
||||||
|
TeeSource tee(conn->from, saved);
|
||||||
|
unsigned int magic = readInt(tee);
|
||||||
|
if (magic != SERVE_MAGIC_2)
|
||||||
|
throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
|
||||||
|
} catch (SerialisationError & e) {
|
||||||
|
/* In case the other side is waiting for our input,
|
||||||
|
close it. */
|
||||||
|
conn->sshConn->in.close();
|
||||||
|
auto msg = conn->from.drain();
|
||||||
|
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
|
||||||
|
host, chomp(saved.s + msg));
|
||||||
|
}
|
||||||
|
conn->remoteVersion = readInt(conn->from);
|
||||||
|
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
|
||||||
|
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
|
||||||
|
|
||||||
|
} catch (EndOfFile & e) {
|
||||||
|
throw Error("cannot connect to '%1%'", host);
|
||||||
|
}
|
||||||
|
|
||||||
|
return conn;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
std::string LegacySSHStore::getUri()
|
||||||
|
{
|
||||||
|
return *uriSchemes().begin() + "://" + host;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
|
||||||
|
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
auto conn(connections->get());
|
||||||
|
|
||||||
|
/* No longer support missing NAR hash */
|
||||||
|
assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
|
||||||
|
|
||||||
|
debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
|
||||||
|
|
||||||
|
conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)};
|
||||||
|
conn->to.flush();
|
||||||
|
|
||||||
|
auto p = readString(conn->from);
|
||||||
|
if (p.empty()) return callback(nullptr);
|
||||||
|
auto path2 = parseStorePath(p);
|
||||||
|
assert(path == path2);
|
||||||
|
auto info = std::make_shared<ValidPathInfo>(
|
||||||
|
path,
|
||||||
|
ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
|
||||||
|
|
||||||
|
if (info->narHash == Hash::dummy)
|
||||||
|
throw Error("NAR hash is now mandatory");
|
||||||
|
|
||||||
|
auto s = readString(conn->from);
|
||||||
|
assert(s == "");
|
||||||
|
|
||||||
|
callback(std::move(info));
|
||||||
|
} catch (...) { callback.rethrow(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||||
|
RepairFlag repair, CheckSigsFlag checkSigs)
|
||||||
|
{
|
||||||
|
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
|
||||||
|
|
||||||
|
auto conn(connections->get());
|
||||||
|
|
||||||
|
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
|
||||||
|
|
||||||
|
conn->to
|
||||||
|
<< ServeProto::Command::AddToStoreNar
|
||||||
|
<< printStorePath(info.path)
|
||||||
|
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||||
|
<< info.narHash.to_string(HashFormat::Base16, false);
|
||||||
|
ServeProto::write(*this, *conn, info.references);
|
||||||
|
conn->to
|
||||||
|
<< info.registrationTime
|
||||||
|
<< info.narSize
|
||||||
|
<< info.ultimate
|
||||||
|
<< info.sigs
|
||||||
|
<< renderContentAddress(info.ca);
|
||||||
|
try {
|
||||||
|
copyNAR(source, conn->to);
|
||||||
|
} catch (...) {
|
||||||
|
conn->good = false;
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
conn->to.flush();
|
||||||
|
|
||||||
|
} else {
|
||||||
|
|
||||||
|
conn->to
|
||||||
|
<< ServeProto::Command::ImportPaths
|
||||||
|
<< 1;
|
||||||
|
try {
|
||||||
|
copyNAR(source, conn->to);
|
||||||
|
} catch (...) {
|
||||||
|
conn->good = false;
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
conn->to
|
||||||
|
<< exportMagic
|
||||||
|
<< printStorePath(info.path);
|
||||||
|
ServeProto::write(*this, *conn, info.references);
|
||||||
|
conn->to
|
||||||
|
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||||
|
<< 0
|
||||||
|
<< 0;
|
||||||
|
conn->to.flush();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if (readInt(conn->from) != 1)
|
||||||
|
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
|
||||||
|
{
|
||||||
|
auto conn(connections->get());
|
||||||
|
|
||||||
|
conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
|
||||||
|
conn->to.flush();
|
||||||
|
copyNAR(conn->from, sink);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void LegacySSHStore::putBuildSettings(Connection & conn)
|
||||||
|
{
|
||||||
|
ServeProto::write(*this, conn, ServeProto::BuildOptions {
|
||||||
|
.maxSilentTime = settings.maxSilentTime,
|
||||||
|
.buildTimeout = settings.buildTimeout,
|
||||||
|
.maxLogSize = settings.maxLogSize,
|
||||||
|
.nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
|
||||||
|
.enforceDeterminism = 0,
|
||||||
|
.keepFailed = settings.keepFailed,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||||
|
BuildMode buildMode)
|
||||||
|
{
|
||||||
|
auto conn(connections->get());
|
||||||
|
|
||||||
|
conn->to
|
||||||
|
<< ServeProto::Command::BuildDerivation
|
||||||
|
<< printStorePath(drvPath);
|
||||||
|
writeDerivation(conn->to, *this, drv);
|
||||||
|
|
||||||
|
putBuildSettings(*conn);
|
||||||
|
|
||||||
|
conn->to.flush();
|
||||||
|
|
||||||
|
return ServeProto::Serialise<BuildResult>::read(*this, *conn);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void LegacySSHStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
|
||||||
|
{
|
||||||
|
if (evalStore && evalStore.get() != this)
|
||||||
|
throw Error("building on an SSH store is incompatible with '--eval-store'");
|
||||||
|
|
||||||
|
auto conn(connections->get());
|
||||||
|
|
||||||
|
conn->to << ServeProto::Command::BuildPaths;
|
||||||
|
Strings ss;
|
||||||
|
for (auto & p : drvPaths) {
|
||||||
|
auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
|
||||||
|
std::visit(overloaded {
|
||||||
|
[&](const StorePathWithOutputs & s) {
|
||||||
|
ss.push_back(s.to_string(*this));
|
||||||
|
},
|
||||||
|
[&](const StorePath & drvPath) {
|
||||||
|
throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
|
||||||
|
},
|
||||||
|
[&](std::monostate) {
|
||||||
|
throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://");
|
||||||
|
},
|
||||||
|
}, sOrDrvPath);
|
||||||
|
}
|
||||||
|
conn->to << ss;
|
||||||
|
|
||||||
|
putBuildSettings(*conn);
|
||||||
|
|
||||||
|
conn->to.flush();
|
||||||
|
|
||||||
|
BuildResult result;
|
||||||
|
result.status = (BuildResult::Status) readInt(conn->from);
|
||||||
|
|
||||||
|
if (!result.success()) {
|
||||||
|
conn->from >> result.errorMsg;
|
||||||
|
throw Error(result.status, result.errorMsg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void LegacySSHStore::computeFSClosure(const StorePathSet & paths,
|
||||||
|
StorePathSet & out, bool flipDirection,
|
||||||
|
bool includeOutputs, bool includeDerivers)
|
||||||
|
{
|
||||||
|
if (flipDirection || includeDerivers) {
|
||||||
|
Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto conn(connections->get());
|
||||||
|
|
||||||
|
conn->to
|
||||||
|
<< ServeProto::Command::QueryClosure
|
||||||
|
<< includeOutputs;
|
||||||
|
ServeProto::write(*this, *conn, paths);
|
||||||
|
conn->to.flush();
|
||||||
|
|
||||||
|
for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn))
|
||||||
|
out.insert(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
|
||||||
|
SubstituteFlag maybeSubstitute)
|
||||||
|
{
|
||||||
|
auto conn(connections->get());
|
||||||
|
|
||||||
|
conn->to
|
||||||
|
<< ServeProto::Command::QueryValidPaths
|
||||||
|
<< false // lock
|
||||||
|
<< maybeSubstitute;
|
||||||
|
ServeProto::write(*this, *conn, paths);
|
||||||
|
conn->to.flush();
|
||||||
|
|
||||||
|
return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void LegacySSHStore::connect()
|
||||||
|
{
|
||||||
|
auto conn(connections->get());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
unsigned int LegacySSHStore::getProtocol()
|
||||||
|
{
|
||||||
|
auto conn(connections->get());
|
||||||
|
return conn->remoteVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The legacy ssh protocol doesn't support checking for trusted-user.
|
||||||
|
* Try using ssh-ng:// instead if you want to know.
|
||||||
|
*/
|
||||||
|
std::optional<TrustedFlag> isTrustedClient()
|
||||||
|
{
|
||||||
|
return std::nullopt;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
|
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
132
src/libstore/legacy-ssh-store.hh
Normal file
132
src/libstore/legacy-ssh-store.hh
Normal file
|
|
@ -0,0 +1,132 @@
|
||||||
|
#pragma once
|
||||||
|
///@file
|
||||||
|
|
||||||
|
#include "ssh-store-config.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
#include "ssh.hh"
|
||||||
|
#include "callback.hh"
|
||||||
|
#include "pool.hh"
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
|
||||||
|
{
|
||||||
|
using CommonSSHStoreConfig::CommonSSHStoreConfig;
|
||||||
|
|
||||||
|
const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
|
||||||
|
"Path to the `nix-store` executable on the remote machine."};
|
||||||
|
|
||||||
|
const Setting<int> maxConnections{this, 1, "max-connections",
|
||||||
|
"Maximum number of concurrent SSH connections."};
|
||||||
|
|
||||||
|
const std::string name() override { return "SSH Store"; }
|
||||||
|
|
||||||
|
std::string doc() override;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
|
||||||
|
{
|
||||||
|
// Hack for getting remote build log output.
|
||||||
|
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
|
||||||
|
// the documentation
|
||||||
|
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
||||||
|
|
||||||
|
struct Connection;
|
||||||
|
|
||||||
|
std::string host;
|
||||||
|
|
||||||
|
ref<Pool<Connection>> connections;
|
||||||
|
|
||||||
|
SSHMaster master;
|
||||||
|
|
||||||
|
static std::set<std::string> uriSchemes() { return {"ssh"}; }
|
||||||
|
|
||||||
|
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params);
|
||||||
|
|
||||||
|
ref<Connection> openConnection();
|
||||||
|
|
||||||
|
std::string getUri() override;
|
||||||
|
|
||||||
|
void queryPathInfoUncached(const StorePath & path,
|
||||||
|
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
|
||||||
|
|
||||||
|
void addToStore(const ValidPathInfo & info, Source & source,
|
||||||
|
RepairFlag repair, CheckSigsFlag checkSigs) override;
|
||||||
|
|
||||||
|
void narFromPath(const StorePath & path, Sink & sink) override;
|
||||||
|
|
||||||
|
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
|
||||||
|
{ unsupported("queryPathFromHashPart"); }
|
||||||
|
|
||||||
|
StorePath addToStore(
|
||||||
|
std::string_view name,
|
||||||
|
const Path & srcPath,
|
||||||
|
FileIngestionMethod method,
|
||||||
|
HashAlgorithm hashAlgo,
|
||||||
|
PathFilter & filter,
|
||||||
|
RepairFlag repair,
|
||||||
|
const StorePathSet & references) override
|
||||||
|
{ unsupported("addToStore"); }
|
||||||
|
|
||||||
|
StorePath addTextToStore(
|
||||||
|
std::string_view name,
|
||||||
|
std::string_view s,
|
||||||
|
const StorePathSet & references,
|
||||||
|
RepairFlag repair) override
|
||||||
|
{ unsupported("addTextToStore"); }
|
||||||
|
|
||||||
|
private:
|
||||||
|
|
||||||
|
void putBuildSettings(Connection & conn);
|
||||||
|
|
||||||
|
public:
|
||||||
|
|
||||||
|
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||||
|
BuildMode buildMode) override;
|
||||||
|
|
||||||
|
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
|
||||||
|
|
||||||
|
void ensurePath(const StorePath & path) override
|
||||||
|
{ unsupported("ensurePath"); }
|
||||||
|
|
||||||
|
virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
|
||||||
|
{ unsupported("getFSAccessor"); }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The default instance would schedule the work on the client side, but
|
||||||
|
* for consistency with `buildPaths` and `buildDerivation` it should happen
|
||||||
|
* on the remote side.
|
||||||
|
*
|
||||||
|
* We make this fail for now so we can add implement this properly later
|
||||||
|
* without it being a breaking change.
|
||||||
|
*/
|
||||||
|
void repairPath(const StorePath & path) override
|
||||||
|
{ unsupported("repairPath"); }
|
||||||
|
|
||||||
|
void computeFSClosure(const StorePathSet & paths,
|
||||||
|
StorePathSet & out, bool flipDirection = false,
|
||||||
|
bool includeOutputs = false, bool includeDerivers = false) override;
|
||||||
|
|
||||||
|
StorePathSet queryValidPaths(const StorePathSet & paths,
|
||||||
|
SubstituteFlag maybeSubstitute = NoSubstitute) override;
|
||||||
|
|
||||||
|
void connect() override;
|
||||||
|
|
||||||
|
unsigned int getProtocol() override;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The legacy ssh protocol doesn't support checking for trusted-user.
|
||||||
|
* Try using ssh-ng:// instead if you want to know.
|
||||||
|
*/
|
||||||
|
std::optional<TrustedFlag> isTrustedClient() override
|
||||||
|
{
|
||||||
|
return std::nullopt;
|
||||||
|
}
|
||||||
|
|
||||||
|
void queryRealisationUncached(const DrvOutput &,
|
||||||
|
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
|
||||||
|
// TODO: Implement
|
||||||
|
{ unsupported("queryRealisation"); }
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue