max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2024-11-22 05:56:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #9839 from obsidiansystems/more-machine-cleanup

Browse source 
Create `StoreReference` and use it in `Machine`
This commit is contained in:
John Ericson 2024-05-22 17:01:57 -04:00 committed by GitHub
commit 4a19f4a866
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 589 additions and 197 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.clang-format
View file

@ -15,7 +15,7 @@ SpaceAfterCStyleCast: true
SpaceAfterTemplateKeyword: false
AccessModifierOffset: -4
AlignAfterOpenBracket: AlwaysBreak
AlignEscapedNewlines: DontAlign
AlignEscapedNewlines: Left
ColumnLimit: 120
BreakStringLiterals: false
BitFieldColonSpacing: None
@ -30,3 +30,5 @@ BreakBeforeBinaryOperators: NonAssignment
AlwaysBreakBeforeMultilineStrings: true
IndentPPDirectives: AfterHash
PPIndentWidth: 2
BinPackArguments: false
BinPackParameters: false

66
maintainers/flake-module.nix
View file

@ -17,7 +17,8 @@
excludes = [
# We don't want to format test data
# ''tests/(?!nixos/).*\.nix''
''^tests/.*''
''^tests/functional/.*$''
''^tests/unit/[^/]*/data/.*$''
# Don't format vendored code
''^src/toml11/.*''
@ -426,6 +427,69 @@
''^src/nix/upgrade-nix\.cc$''
''^src/nix/verify\.cc$''
''^src/nix/why-depends\.cc$''
''^tests/nixos/ca-fd-leak/sender\.c''
''^tests/nixos/ca-fd-leak/smuggler\.c''
''^tests/unit/libexpr-support/tests/libexpr\.hh''
''^tests/unit/libexpr-support/tests/value/context\.cc''
''^tests/unit/libexpr-support/tests/value/context\.hh''
''^tests/unit/libexpr/derived-path\.cc''
''^tests/unit/libexpr/error_traces\.cc''
''^tests/unit/libexpr/eval\.cc''
''^tests/unit/libexpr/flake/flakeref\.cc''
''^tests/unit/libexpr/flake/url-name\.cc''
''^tests/unit/libexpr/json\.cc''
''^tests/unit/libexpr/main\.cc''
''^tests/unit/libexpr/primops\.cc''
''^tests/unit/libexpr/search-path\.cc''
''^tests/unit/libexpr/trivial\.cc''
''^tests/unit/libexpr/value/context\.cc''
''^tests/unit/libexpr/value/print\.cc''
''^tests/unit/libfetchers/public-key\.cc''
''^tests/unit/libstore-support/tests/derived-path\.cc''
''^tests/unit/libstore-support/tests/derived-path\.hh''
''^tests/unit/libstore-support/tests/libstore\.hh''
''^tests/unit/libstore-support/tests/nix_api_store\.hh''
''^tests/unit/libstore-support/tests/outputs-spec\.cc''
''^tests/unit/libstore-support/tests/outputs-spec\.hh''
''^tests/unit/libstore-support/tests/path\.cc''
''^tests/unit/libstore-support/tests/path\.hh''
''^tests/unit/libstore-support/tests/protocol\.hh''
''^tests/unit/libstore/common-protocol\.cc''
''^tests/unit/libstore/content-address\.cc''
''^tests/unit/libstore/derivation\.cc''
''^tests/unit/libstore/derived-path\.cc''
''^tests/unit/libstore/downstream-placeholder\.cc''
''^tests/unit/libstore/machines\.cc''
''^tests/unit/libstore/nar-info-disk-cache\.cc''
''^tests/unit/libstore/nar-info\.cc''
''^tests/unit/libstore/outputs-spec\.cc''
''^tests/unit/libstore/path-info\.cc''
''^tests/unit/libstore/path\.cc''
''^tests/unit/libstore/serve-protocol\.cc''
''^tests/unit/libstore/worker-protocol\.cc''
''^tests/unit/libutil-support/tests/characterization\.hh''
''^tests/unit/libutil-support/tests/hash\.cc''
''^tests/unit/libutil-support/tests/hash\.hh''
''^tests/unit/libutil/args\.cc''
''^tests/unit/libutil/canon-path\.cc''
''^tests/unit/libutil/chunked-vector\.cc''
''^tests/unit/libutil/closure\.cc''
''^tests/unit/libutil/compression\.cc''
''^tests/unit/libutil/config\.cc''
''^tests/unit/libutil/file-content-address\.cc''
''^tests/unit/libutil/git\.cc''
''^tests/unit/libutil/hash\.cc''
''^tests/unit/libutil/hilite\.cc''
''^tests/unit/libutil/json-utils\.cc''
''^tests/unit/libutil/logging\.cc''
''^tests/unit/libutil/lru-cache\.cc''
''^tests/unit/libutil/pool\.cc''
''^tests/unit/libutil/references\.cc''
''^tests/unit/libutil/suggestions\.cc''
''^tests/unit/libutil/tests\.cc''
''^tests/unit/libutil/url\.cc''
''^tests/unit/libutil/xml-writer\.cc''
];
};

22
src/build-remote/build-remote.cc
View file

@ -37,7 +37,7 @@ static std::string currentLoad;
static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
{
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true);
}
static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
@ -99,7 +99,7 @@ static int main_build_remote(int argc, char * * argv)
}
std::optional<StorePath> drvPath;
std::string storeUri;
StoreReference storeUri;
while (true) {
@ -135,7 +135,7 @@ static int main_build_remote(int argc, char * * argv)
Machine * bestMachine = nullptr;
uint64_t bestLoad = 0;
for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
debug("considering building on remote machine '%s'", m.storeUri.render());
if (m.enabled &&
m.systemSupported(neededSystem) &&
@ -233,7 +233,7 @@ static int main_build_remote(int argc, char * * argv)
try {
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render()));
sshStore = bestMachine->openStore();
sshStore->connect();
@ -242,7 +242,7 @@ static int main_build_remote(int argc, char * * argv)
} catch (std::exception & e) {
auto msg = chomp(drainFD(5, false));
printError("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
bestMachine->storeUri.render(), e.what(),
msg.empty() ? "" : ": " + msg);
bestMachine->enabled = false;
continue;
@ -257,15 +257,15 @@ connected:
assert(sshStore);
std::cerr << "# accept\n" << storeUri << "\n";
std::cerr << "# accept\n" << storeUri.render() << "\n";
auto inputs = readStrings<PathSet>(source);
auto wantedOutputs = readStrings<StringSet>(source);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true);
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render()));
auto old = signal(SIGALRM, handleAlarm);
alarm(15 * 60);
@ -278,7 +278,7 @@ connected:
auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render()));
copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
}
@ -316,7 +316,7 @@ connected:
optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
auto & result = *optResult;
if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg);
} else {
copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
auto res = sshStore->buildPathsWithResults({
@ -359,7 +359,7 @@ connected:
}
if (!missingPaths.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render()));
if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
for (auto & path : missingPaths)
localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */

5
src/libcmd/network-proxy.cc
View file

@ -25,7 +25,10 @@ static StringSet getExcludingNoProxyVariables()
static const StringSet excludeVariables{"no_proxy", "NO_PROXY"};
StringSet variables;
std::set_difference(
networkProxyVariables.begin(), networkProxyVariables.end(), excludeVariables.begin(), excludeVariables.end(),
networkProxyVariables.begin(),
networkProxyVariables.end(),
excludeVariables.begin(),
excludeVariables.end(),
std::inserter(variables, variables.begin()));
return variables;
}

35
src/libstore/machines.cc
View file

@ -6,7 +6,8 @@
namespace nix {
Machine::Machine(decltype(storeUri) storeUri,
Machine::Machine(
const std::string & storeUri,
decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs,
@ -14,7 +15,7 @@ Machine::Machine(decltype(storeUri) storeUri,
decltype(supportedFeatures) supportedFeatures,
decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey) :
storeUri(
storeUri(StoreReference::parse(
// Backwards compatibility: if the URI is schemeless, is not a path,
// and is not one of the special store connection words, prepend
// ssh://.
@ -28,7 +29,7 @@ Machine::Machine(decltype(storeUri) storeUri,
|| hasPrefix(storeUri, "local?")
|| hasPrefix(storeUri, "?")
? storeUri
: "ssh://" + storeUri),
: "ssh://" + storeUri)),
systemTypes(systemTypes),
sshKey(sshKey),
maxJobs(maxJobs),
@ -63,23 +64,26 @@ bool Machine::mandatoryMet(const std::set<std::string> & features) const
});
}
ref<Store> Machine::openStore() const
StoreReference Machine::completeStoreReference() const
{
Store::Params storeParams;
if (hasPrefix(storeUri, "ssh://")) {
storeParams["max-connections"] = "1";
storeParams["log-fd"] = "4";
auto storeUri = this->storeUri;
auto * generic = std::get_if<StoreReference::Specified>(&storeUri.variant);
if (generic && generic->scheme == "ssh") {
storeUri.params["max-connections"] = "1";
storeUri.params["log-fd"] = "4";
}
if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) {
if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) {
if (sshKey != "")
storeParams["ssh-key"] = sshKey;
storeUri.params["ssh-key"] = sshKey;
if (sshPublicHostKey != "")
storeParams["base64-ssh-public-host-key"] = sshPublicHostKey;
storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey;
}
{
auto & fs = storeParams["system-features"];
auto & fs = storeUri.params["system-features"];
auto append = [&](auto feats) {
for (auto & f : feats) {
if (fs.size() > 0) fs += ' ';
@ -90,7 +94,12 @@ ref<Store> Machine::openStore() const
append(mandatoryFeatures);
}
return nix::openStore(storeUri, storeParams);
return storeUri;
}
ref<Store> Machine::openStore() const
{
return nix::openStore(completeStoreReference());
}
static std::vector<std::string> expandBuilderLines(const std::string & builders)

21
src/libstore/machines.hh
View file

@ -2,6 +2,7 @@
///@file
#include "types.hh"
#include "store-reference.hh"
namespace nix {
@ -9,7 +10,7 @@ class Store;
struct Machine {
const std::string storeUri;
const StoreReference storeUri;
const std::set<std::string> systemTypes;
const std::string sshKey;
const unsigned int maxJobs;
@ -36,7 +37,8 @@ struct Machine {
*/
bool mandatoryMet(const std::set<std::string> & features) const;
Machine(decltype(storeUri) storeUri,
Machine(
const std::string & storeUri,
decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs,
@ -45,6 +47,21 @@ struct Machine {
decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey);
/**
* Elaborate `storeUri` into a complete store reference,
* incorporating information from the other fields of the `Machine`
* as applicable.
*/
StoreReference completeStoreReference() const;
/**
* Open a `Store` for this machine.
*
* Just a simple function composition:
* ```c++
* nix::openStore(completeStoreReference())
* ```
*/
ref<Store> openStore() const;
};

147
src/libstore/store-api.cc
View file

@ -8,7 +8,6 @@
#include "util.hh"
#include "nar-info-disk-cache.hh"
#include "thread-pool.hh"
#include "url.hh"
#include "references.hh"
#include "archive.hh"
#include "callback.hh"
@ -1267,109 +1266,63 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath)
namespace nix {
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
{
auto uri(uri_);
Store::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q);
}
return {uri, params};
}
static bool isNonUriPath(const std::string & spec)
{
return
// is not a URL
spec.find("://") == std::string::npos
// Has at least one path separator, and so isn't a single word that
// might be special like "auto"
&& spec.find("/") != std::string::npos;
}
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
{
// TODO reenable on Windows once we have `LocalStore` and
// `UDSRemoteStore`.
if (uri == "" || uri == "auto") {
auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params);
else if (pathExists(settings.nixDaemonSocketFile))
return std::make_shared<UDSRemoteStore>(params);
#if __linux__
else if (!pathExists(stateDir)
&& params.empty()
&& !isRootUser()
&& !getEnv("NIX_STORE_DIR").has_value()
&& !getEnv("NIX_STATE_DIR").has_value())
{
/* If /nix doesn't exist, there is no daemon socket, and
we're not root, then automatically set up a chroot
store in ~/.local/share/nix/root. */
auto chrootStore = getDataDir() + "/nix/root";
if (!pathExists(chrootStore)) {
try {
createDirs(chrootStore);
} catch (Error & e) {
return std::make_shared<LocalStore>(params);
}
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
} else
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
return std::make_shared<LocalStore>("local", chrootStore, params);
}
#endif
else
return std::make_shared<LocalStore>(params);
} else if (uri == "daemon") {
return std::make_shared<UDSRemoteStore>(params);
} else if (uri == "local") {
return std::make_shared<LocalStore>(params);
} else if (isNonUriPath(uri)) {
return std::make_shared<LocalStore>("local", absPath(uri), params);
} else {
return nullptr;
}
}
ref<Store> openStore(const std::string & uri_,
ref<Store> openStore(const std::string & uri,
const Store::Params & extraParams)
{
auto params = extraParams;
try {
auto parsedUri = parseURL(uri_);
params.insert(parsedUri.query.begin(), parsedUri.query.end());
return openStore(StoreReference::parse(uri, extraParams));
}
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
ref<Store> openStore(StoreReference && storeURI)
{
auto & params = storeURI.params;
for (auto implem : *Implementations::registered) {
if (implem.uriSchemes.count(parsedUri.scheme)) {
auto store = implem.create(parsedUri.scheme, baseURI, params);
if (store) {
experimentalFeatureSettings.require(store->experimentalFeature());
store->init();
store->warnUnknownSettings();
return ref<Store>(store);
}
auto store = std::visit(overloaded {
[&](const StoreReference::Auto &) -> std::shared_ptr<Store> {
auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params);
else if (pathExists(settings.nixDaemonSocketFile))
return std::make_shared<UDSRemoteStore>(params);
#if __linux__
else if (!pathExists(stateDir)
&& params.empty()
&& !isRootUser()
&& !getEnv("NIX_STORE_DIR").has_value()
&& !getEnv("NIX_STATE_DIR").has_value())
{
/* If /nix doesn't exist, there is no daemon socket, and
we're not root, then automatically set up a chroot
store in ~/.local/share/nix/root. */
auto chrootStore = getDataDir() + "/nix/root";
if (!pathExists(chrootStore)) {
try {
createDirs(chrootStore);
} catch (Error & e) {
return std::make_shared<LocalStore>(params);
}
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
} else
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
return std::make_shared<LocalStore>("local", chrootStore, params);
}
}
}
catch (BadURL &) {
auto [uri, uriParams] = splitUriAndParams(uri_);
params.insert(uriParams.begin(), uriParams.end());
#endif
else
return std::make_shared<LocalStore>(params);
},
[&](const StoreReference::Specified & g) {
for (auto implem : *Implementations::registered)
if (implem.uriSchemes.count(g.scheme))
return implem.create(g.scheme, g.authority, params);
if (auto store = openFromNonUri(uri, params)) {
experimentalFeatureSettings.require(store->experimentalFeature());
store->warnUnknownSettings();
return ref<Store>(store);
}
}
throw Error("don't know how to open Nix store with scheme '%s'", g.scheme);
},
}, storeURI.variant);
throw Error("don't know how to open Nix store '%s'", uri_);
experimentalFeatureSettings.require(store->experimentalFeature());
store->warnUnknownSettings();
store->init();
return ref<Store> { store };
}
std::list<ref<Store>> getDefaultSubstituters()

45
src/libstore/store-api.hh
View file

@ -13,6 +13,7 @@
#include "path-info.hh"
#include "repair-flag.hh"
#include "store-dir-config.hh"
#include "store-reference.hh"
#include "source-path.hh"
#include <nlohmann/json_fwd.hpp>
@ -65,7 +66,7 @@ MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
MakeError(InvalidStoreURI, Error);
MakeError(InvalidStoreReference, Error);
struct Realisation;
struct RealisedPath;
@ -102,7 +103,7 @@ typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
struct StoreConfig : public StoreDirConfig
{
typedef std::map<std::string, std::string> Params;
using Params = StoreReference::Params;
using StoreDirConfig::StoreDirConfig;
@ -859,34 +860,13 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
/**
* @return a Store object to access the Nix store denoted by
* ‘uri’ (slight misnomer...).
*
* @param uri Supported values are:
*
* - ‘local’: The Nix store in /nix/store and database in
* /nix/var/nix/db, accessed directly.
*
* - ‘daemon’: The Nix store accessed via a Unix domain socket
* connection to nix-daemon.
*
* - ‘unix://<path>’: The Nix store accessed via a Unix domain socket
* connection to nix-daemon, with the socket located at <path>.
*
* - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
* whether the user has write access to the local Nix
* store/database.
*
* - ‘file://<path>’: A binary cache stored in <path>.
*
* - ‘https://<path>’: A binary cache accessed via HTTP.
*
* - ‘s3://<path>’: A writable binary cache stored on Amazon's Simple
* Storage Service.
*
* - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
* ‘nix-store --serve’ via SSH.
*
* You can pass parameters to the store type by appending
* ‘?key=value&key=value&...’ to the URI.
*/
ref<Store> openStore(StoreReference && storeURI);
/**
* Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse`
*/
ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
const Store::Params & extraParams = Store::Params());
@ -957,11 +937,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
std::istream & str,
std::optional<HashResult> hashGiven = std::nullopt);
/**
* Split URI into protocol+hierarchy part and its parameter set.
*/
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
std::map<DrvOutput, StorePath> drvOutputReferences(

116
src/libstore/store-reference.cc Normal file
View file

@ -0,0 +1,116 @@
#include <regex>
#include "error.hh"
#include "url.hh"
#include "store-reference.hh"
#include "file-system.hh"
#include "util.hh"
namespace nix {
static bool isNonUriPath(const std::string & spec)
{
return
// is not a URL
spec.find("://") == std::string::npos
// Has at least one path separator, and so isn't a single word that
// might be special like "auto"
&& spec.find("/") != std::string::npos;
}
std::string StoreReference::render() const
{
std::string res;
std::visit(
overloaded{
[&](const StoreReference::Auto &) { res = "auto"; },
[&](const StoreReference::Specified & g) {
res = g.scheme;
res += "://";
res += g.authority;
},
},
variant);
if (!params.empty()) {
res += "?";
res += encodeQuery(params);
}
return res;
}
StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams)
{
auto params = extraParams;
try {
auto parsedUri = parseURL(uri);
params.insert(parsedUri.query.begin(), parsedUri.query.end());
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
return {
.variant =
Specified{
.scheme = std::move(parsedUri.scheme),
.authority = std::move(baseURI),
},
.params = std::move(params),
};
} catch (BadURL &) {
auto [baseURI, uriParams] = splitUriAndParams(uri);
params.insert(uriParams.begin(), uriParams.end());
if (baseURI == "" || baseURI == "auto") {
return {
.variant = Auto{},
.params = std::move(params),
};
} else if (baseURI == "daemon") {
return {
.variant =
Specified{
.scheme = "unix",
.authority = "",
},
.params = std::move(params),
};
} else if (baseURI == "local") {
return {
.variant =
Specified{
.scheme = "local",
.authority = "",
},
.params = std::move(params),
};
} else if (isNonUriPath(baseURI)) {
return {
.variant =
Specified{
.scheme = "local",
.authority = absPath(baseURI),
},
.params = std::move(params),
};
}
}
throw UsageError("Cannot parse Nix store '%s'", uri);
}
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri_)
{
auto uri(uri_);
StoreReference::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q);
}
return {uri, params};
}
}

92
src/libstore/store-reference.hh Normal file
View file

@ -0,0 +1,92 @@
#pragma once
///@file
#include <variant>
#include "types.hh"
namespace nix {
/**
* A parsed Store URI (URI is a slight misnomer...), parsed but not yet
* resolved to a specific instance and query parms validated.
*
* Supported values are:
*
* - ‘local’: The Nix store in /nix/store and database in
* /nix/var/nix/db, accessed directly.
*
* - ‘daemon’: The Nix store accessed via a Unix domain socket
* connection to nix-daemon.
*
* - ‘unix://<path>’: The Nix store accessed via a Unix domain socket
* connection to nix-daemon, with the socket located at <path>.
*
* - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
* whether the user has write access to the local Nix
* store/database.
*
* - ‘file://<path>’: A binary cache stored in <path>.
*
* - ‘https://<path>’: A binary cache accessed via HTTP.
*
* - ‘s3://<path>’: A writable binary cache stored on Amazon's Simple
* Storage Service.
*
* - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
* ‘nix-store --serve’ via SSH.
*
* You can pass parameters to the store type by appending
* ‘?key=value&key=value&...’ to the URI.
*/
struct StoreReference
{
using Params = std::map<std::string, std::string>;
/**
* Special store reference `""` or `"auto"`
*/
struct Auto
{
inline bool operator==(const Auto & rhs) const = default;
inline auto operator<=>(const Auto & rhs) const = default;
};
/**
* General case, a regular `scheme://authority` URL.
*/
struct Specified
{
std::string scheme;
std::string authority = "";
bool operator==(const Specified & rhs) const = default;
auto operator<=>(const Specified & rhs) const = default;
};
typedef std::variant<Auto, Specified> Variant;
Variant variant;
Params params;
bool operator==(const StoreReference & rhs) const = default;
auto operator<=>(const StoreReference & rhs) const = default;
/**
* Render the whole store reference as a URI, including parameters.
*/
std::string render() const;
/**
* Parse a URI into a store reference.
*/
static StoreReference parse(const std::string & uri, const Params & extraParams = Params{});
};
/**
* Split URI into protocol+hierarchy part and its parameter set.
*/
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri);
}

9
src/libstore/windows/pathlocks.cc
View file

@ -35,8 +35,13 @@ void PathLocks::unlock()
AutoCloseFD openLockFile(const Path & path, bool create)
{
AutoCloseFD desc = CreateFileA(
path.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
create ? OPEN_ALWAYS : OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, NULL);
path.c_str(),
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL,
create ? OPEN_ALWAYS : OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS,
NULL);
if (desc.get() == INVALID_HANDLE_VALUE)
warn("%s: %s", path, std::to_string(GetLastError()));

13
src/libutil/compression.cc
View file

@ -263,8 +263,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink
checkInterrupt();
if (!BrotliEncoderCompressStream(
state, data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, &avail_in, &next_in,
&avail_out, &next_out, nullptr))
state,
data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
&avail_in,
&next_in,
&avail_out,
&next_out,
nullptr))
throw CompressionError("error while compressing brotli compression");
if (avail_out < sizeof(outbuf) || avail_in == 0) {
@ -280,8 +285,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
{
std::vector<std::string> la_supports = {"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4",
"lzip", "lzma", "lzop", "xz", "zstd"};
std::vector<std::string> la_supports = {
"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"};
if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
}

2
src/libutil/url.hh
View file

@ -33,6 +33,8 @@ std::string percentEncode(std::string_view s, std::string_view keep="");
std::map<std::string, std::string> decodeQuery(const std::string & query);
std::string encodeQuery(const std::map<std::string, std::string> & query);
ParsedURL parseURL(const std::string & url);
/**

9
src/libutil/windows/file-system.cc
View file

@ -5,8 +5,13 @@ namespace nix {
Descriptor openDirectory(const std::filesystem::path & path)
{
return CreateFileW(
path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, NULL);
path.c_str(),
GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
NULL,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS,
NULL);
}
}

21
tests/unit/libexpr/nix_api_value.cc
View file

@ -256,10 +256,13 @@ TEST_F(nix_api_expr_test, nix_value_init)
Value * f = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(
ctx, state, R"(
ctx,
state,
R"(
a: a * a
)",
"<test>", f);
"<test>",
f);
// Test
@ -325,20 +328,26 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg)
Value * f = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(
ctx, state, R"(
ctx,
state,
R"(
a: { foo = a; }
)",
"<test>", f);
"<test>",
f);
assert_ctx_ok();
Value * e = nix_alloc_value(ctx, state);
{
Value * g = nix_alloc_value(ctx, state);
nix_expr_eval_from_string(
ctx, state, R"(
ctx,
state,
R"(
_ignore: throw "error message for test case nix_value_init_apply_lazy_arg"
)",
"<test>", g);
"<test>",
g);
assert_ctx_ok();
nix_init_apply(ctx, e, g, g);

1
tests/unit/libstore/data/store-reference/auto.txt Normal file
View file

@ -0,0 +1 @@
auto

1
tests/unit/libstore/data/store-reference/auto_param.txt Normal file
View file

@ -0,0 +1 @@
auto?root=/foo/bar/baz

1
tests/unit/libstore/data/store-reference/local_1.txt Normal file
View file

@ -0,0 +1 @@
local://?root=/foo/bar/baz

1
tests/unit/libstore/data/store-reference/local_2.txt Normal file
View file

@ -0,0 +1 @@
local:///foo/bar/baz?trusted=true

1
tests/unit/libstore/data/store-reference/local_shorthand_1.txt Normal file
View file

@ -0,0 +1 @@
local?root=/foo/bar/baz

1
tests/unit/libstore/data/store-reference/local_shorthand_2.txt Normal file
View file

@ -0,0 +1 @@
/foo/bar/baz?trusted=true

1
tests/unit/libstore/data/store-reference/ssh.txt Normal file
View file

@ -0,0 +1 @@
ssh://localhost

1
tests/unit/libstore/data/store-reference/unix.txt Normal file
View file

@ -0,0 +1 @@
unix://?max-connections=7&trusted=true

1
tests/unit/libstore/data/store-reference/unix_shorthand.txt Normal file
View file

@ -0,0 +1 @@
daemon?max-connections=7&trusted=true

47
tests/unit/libstore/machines.cc
View file

@ -3,24 +3,16 @@
#include "file-system.hh"
#include "util.hh"
#include <gtest/gtest.h>
#include <gmock/gmock-matchers.h>
using testing::Contains;
using testing::ElementsAre;
using testing::EndsWith;
using testing::Eq;
using testing::Field;
using testing::SizeIs;
using nix::absPath;
using nix::FormatError;
using nix::UsageError;
using nix::getMachines;
using nix::Machine;
using nix::Machines;
using nix::pathExists;
using nix::Settings;
using nix::settings;
using namespace nix;
class Environment : public ::testing::Environment {
public:
@ -40,7 +32,7 @@ TEST(machines, getMachinesUriOnly) {
settings.builders = "nix@scratchy.labs.cs.uu.nl";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1)));
@ -54,7 +46,7 @@ TEST(machines, getMachinesDefaults) {
settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1)));
@ -64,20 +56,31 @@ TEST(machines, getMachinesDefaults) {
EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0)));
}
MATCHER_P(AuthorityMatches, authority, "") {
*result_listener
<< "where the authority of "
<< arg.render()
<< " is "
<< authority;
auto * generic = std::get_if<StoreReference::Specified>(&arg.variant);
if (!generic) return false;
return generic->authority == authority;
}
TEST(machines, getMachinesWithNewLineSeparator) {
settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(2));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
}
TEST(machines, getMachinesWithSemicolonSeparator) {
settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl";
Machines actual = getMachines();
EXPECT_THAT(actual, SizeIs(2));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
}
TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) {
@ -86,7 +89,7 @@ TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) {
"benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED==";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto")));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8)));
@ -104,7 +107,7 @@ TEST(machines,
"KEY+BASE64+ENCODED==";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto")));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8)));
@ -120,7 +123,7 @@ TEST(machines, getMachinesWithMultiOptions) {
"MandatoryFeature1,MandatoryFeature2";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2")));
EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2")));
EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2")));
@ -146,9 +149,9 @@ TEST(machines, getMachinesWithCorrectFileReference) {
settings.builders = std::string("@") + path;
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(3));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@poochie.labs.cs.uu.nl"))));
}
TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) {

123
tests/unit/libstore/store-reference.cc Normal file
View file

@ -0,0 +1,123 @@
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
#include "file-system.hh"
#include "store-reference.hh"
#include "tests/characterization.hh"
#include "tests/libstore.hh"
namespace nix {
using nlohmann::json;
class StoreReferenceTest : public CharacterizationTest, public LibStoreTest
{
Path unitTestData = getUnitTestData() + "/store-reference";
Path goldenMaster(PathView testStem) const override
{
return unitTestData + "/" + testStem + ".txt";
}
};
#define URI_TEST_READ(STEM, OBJ) \
TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \
{ \
readTest(#STEM, ([&](const auto & encoded) { \
StoreReference expected = OBJ; \
auto got = StoreReference::parse(encoded); \
ASSERT_EQ(got, expected); \
})); \
}
#define URI_TEST_WRITE(STEM, OBJ) \
TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \
{ \
writeTest( \
#STEM, \
[&]() -> StoreReference { return OBJ; }, \
[](const auto & file) { return StoreReference::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \
}
#define URI_TEST(STEM, OBJ) \
URI_TEST_READ(STEM, OBJ) \
URI_TEST_WRITE(STEM, OBJ)
URI_TEST(
auto,
(StoreReference{
.variant = StoreReference::Auto{},
.params = {},
}))
URI_TEST(
auto_param,
(StoreReference{
.variant = StoreReference::Auto{},
.params =
{
{"root", "/foo/bar/baz"},
},
}))
static StoreReference localExample_1{
.variant =
StoreReference::Specified{
.scheme = "local",
},
.params =
{
{"root", "/foo/bar/baz"},
},
};
static StoreReference localExample_2{
.variant =
StoreReference::Specified{
.scheme = "local",
.authority = "/foo/bar/baz",
},
.params =
{
{"trusted", "true"},
},
};
URI_TEST(local_1, localExample_1)
URI_TEST(local_2, localExample_2)
URI_TEST_READ(local_shorthand_1, localExample_1)
URI_TEST_READ(local_shorthand_2, localExample_2)
static StoreReference unixExample{
.variant =
StoreReference::Specified{
.scheme = "unix",
},
.params =
{
{"max-connections", "7"},
{"trusted", "true"},
},
};
URI_TEST(unix, unixExample)
URI_TEST_READ(unix_shorthand, unixExample)
URI_TEST(
ssh,
(StoreReference{
.variant =
StoreReference::Specified{
.scheme = "ssh",
.authority = "localhost",
},
.params = {},
}))
}