mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-10 08:16:15 +02:00
Merge remote-tracking branch 'origin/master' into github-api-token
This commit is contained in:
Gregory Hale
commit
faa5607f54
82 changed files with 979 additions and 753 deletions
9
.gitignore
vendored
9
.gitignore
vendored
|
|
@ -12,15 +12,6 @@ perl/Makefile.config
|
|||
/svn-revision
|
||||
/libtool
|
||||
|
||||
/corepkgs/config.nix
|
||||
|
||||
# /corepkgs/channels/
|
||||
/corepkgs/channels/unpack.sh
|
||||
|
||||
# /corepkgs/nar/
|
||||
/corepkgs/nar/nar.sh
|
||||
/corepkgs/nar/unnar.sh
|
||||
|
||||
# /doc/manual/
|
||||
/doc/manual/*.1
|
||||
/doc/manual/*.5
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -26,7 +26,7 @@ OPTIMIZE = 1
|
|||
ifeq ($(OPTIMIZE), 1)
|
||||
GLOBAL_CXXFLAGS += -O3
|
||||
else
|
||||
GLOBAL_CXXFLAGS += -O0
|
||||
GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE
|
||||
endif
|
||||
|
||||
include mk/lib.mk
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
# FIXME: remove this file?
|
||||
let
|
||||
fromEnv = var: def:
|
||||
let val = builtins.getEnv var; in
|
||||
if val != "" then val else def;
|
||||
in rec {
|
||||
nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
|
||||
nixPrefix = "@prefix@";
|
||||
nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@";
|
||||
nixLocalstateDir = "@localstatedir@";
|
||||
nixSysconfDir = "@sysconfdir@";
|
||||
nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@";
|
||||
}
|
||||
|
|
@ -1,8 +1,4 @@
|
|||
corepkgs_FILES = \
|
||||
unpack-channel.nix \
|
||||
derivation.nix \
|
||||
fetchurl.nix
|
||||
|
||||
$(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
|
||||
|
||||
template-files += $(d)/config.nix
|
||||
$(foreach file,$(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ Nix store is also printed.
|
|||
result to the Nix store. The resulting hash can be used with
|
||||
functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`.
|
||||
|
||||
- `--executable`
|
||||
Set the executable bit on the downloaded file.
|
||||
|
||||
- `--name` *name*
|
||||
Override the name of the file in the Nix store. By default, this is
|
||||
`hash-basename`, where *basename* is the last component of *url*.
|
||||
|
|
|
|||
11
flake.lock
11
flake.lock
|
|
@ -3,16 +3,15 @@
|
|||
"lowdown-src": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1598296217,
|
||||
"narHash": "sha256-ha7lyNY1d8m+osmDpPc9f/bfZ3ZC1IVIXwfyklSWg8I=",
|
||||
"owner": "edolstra",
|
||||
"lastModified": 1598695561,
|
||||
"narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=",
|
||||
"owner": "kristapsdz",
|
||||
"repo": "lowdown",
|
||||
"rev": "c7a4e715af1e233080842db82d15b261cb74cb28",
|
||||
"rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "edolstra",
|
||||
"ref": "no-structs-in-anonymous-unions",
|
||||
"owner": "kristapsdz",
|
||||
"repo": "lowdown",
|
||||
"type": "github"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
description = "The purely functional package manager";
|
||||
|
||||
inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
|
||||
inputs.lowdown-src = { url = "github:edolstra/lowdown/no-structs-in-anonymous-unions"; flake = false; };
|
||||
inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
|
||||
|
||||
outputs = { self, nixpkgs, lowdown-src }:
|
||||
|
||||
|
|
@ -58,6 +58,7 @@
|
|||
configureFlags =
|
||||
lib.optionals stdenv.isLinux [
|
||||
"--with-sandbox-shell=${sh}/bin/busybox"
|
||||
"LDFLAGS=-fuse-ld=gold"
|
||||
];
|
||||
|
||||
buildDeps =
|
||||
|
|
@ -136,7 +137,7 @@
|
|||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
makeFlags = "profiledir=$(out)/etc/profile.d";
|
||||
makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
|
||||
|
||||
doCheck = true;
|
||||
|
||||
|
|
@ -334,9 +335,6 @@
|
|||
# syntax-check generated dot files, it still requires some
|
||||
# fonts. So provide those.
|
||||
FONTCONFIG_FILE = texFunctions.fontsConf;
|
||||
|
||||
# To test building without precompiled headers.
|
||||
makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
|
||||
};
|
||||
|
||||
# System tests.
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
<dict>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>NIX_SSL_CERT_FILE</key>
|
||||
<string>/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt</string>
|
||||
<key>OBJC_DISABLE_INITIALIZE_FORK_SAFETY</key>
|
||||
<string>YES</string>
|
||||
</dict>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
PRECOMPILE_HEADERS ?= 1
|
||||
PRECOMPILE_HEADERS ?= 0
|
||||
|
||||
print-var-help += \
|
||||
echo " PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build";
|
||||
|
|
|
|||
|
|
@ -70,19 +70,6 @@ PKG_CHECK_MODULES([NIX], [nix-store])
|
|||
|
||||
NEED_PROG([NIX], [nix])
|
||||
|
||||
# Get nix configure values
|
||||
export NIX_REMOTE=daemon
|
||||
nixbindir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixBinDir)
|
||||
nixlibexecdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLibexecDir)
|
||||
nixlocalstatedir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLocalstateDir)
|
||||
nixsysconfdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixSysconfDir)
|
||||
nixstoredir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixStoreDir)
|
||||
AC_SUBST(nixbindir)
|
||||
AC_SUBST(nixlibexecdir)
|
||||
AC_SUBST(nixlocalstatedir)
|
||||
AC_SUBST(nixsysconfdir)
|
||||
AC_SUBST(nixstoredir)
|
||||
|
||||
# Expand all variables in config.status.
|
||||
test "$prefix" = NONE && prefix=$ac_default_prefix
|
||||
test "$exec_prefix" = NONE && exec_prefix='${prefix}'
|
||||
|
|
|
|||
|
|
@ -4,14 +4,8 @@ use MIME::Base64;
|
|||
|
||||
$version = "@PACKAGE_VERSION@";
|
||||
|
||||
$binDir = $ENV{"NIX_BIN_DIR"} || "@nixbindir@";
|
||||
$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@nixlibexecdir@";
|
||||
$stateDir = $ENV{"NIX_STATE_DIR"} || "@nixlocalstatedir@/nix";
|
||||
$logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix";
|
||||
$confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix";
|
||||
$storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@";
|
||||
|
||||
$useBindings = 1;
|
||||
$binDir = Nix::Store::getBinDir;
|
||||
$storeDir = Nix::Store::getStoreDir;
|
||||
|
||||
%config = ();
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package Nix::Store;
|
|||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Nix::Config;
|
||||
|
||||
require Exporter;
|
||||
|
||||
|
|
@ -22,6 +21,7 @@ our @EXPORT = qw(
|
|||
addToStore makeFixedOutputPath
|
||||
derivationFromPath
|
||||
addTempRoot
|
||||
getBinDir getStoreDir
|
||||
);
|
||||
|
||||
our $VERSION = '0.15';
|
||||
|
|
@ -34,62 +34,8 @@ sub backtick {
|
|||
return $res;
|
||||
}
|
||||
|
||||
if ($Nix::Config::useBindings) {
|
||||
require XSLoader;
|
||||
XSLoader::load('Nix::Store', $VERSION);
|
||||
} else {
|
||||
|
||||
# Provide slow fallbacks of some functions on platforms that don't
|
||||
# support the Perl bindings.
|
||||
|
||||
use File::Temp;
|
||||
use Fcntl qw/F_SETFD/;
|
||||
|
||||
*hashFile = sub {
|
||||
my ($algo, $base32, $path) = @_;
|
||||
my $res = backtick("$Nix::Config::binDir/nix-hash", "--flat", $path, "--type", $algo, $base32 ? "--base32" : ());
|
||||
chomp $res;
|
||||
return $res;
|
||||
};
|
||||
|
||||
*hashPath = sub {
|
||||
my ($algo, $base32, $path) = @_;
|
||||
my $res = backtick("$Nix::Config::binDir/nix-hash", $path, "--type", $algo, $base32 ? "--base32" : ());
|
||||
chomp $res;
|
||||
return $res;
|
||||
};
|
||||
|
||||
*hashString = sub {
|
||||
my ($algo, $base32, $s) = @_;
|
||||
my $fh = File::Temp->new();
|
||||
print $fh $s;
|
||||
my $res = backtick("$Nix::Config::binDir/nix-hash", $fh->filename, "--type", $algo, $base32 ? "--base32" : ());
|
||||
chomp $res;
|
||||
return $res;
|
||||
};
|
||||
|
||||
*addToStore = sub {
|
||||
my ($srcPath, $recursive, $algo) = @_;
|
||||
die "not implemented" if $recursive || $algo ne "sha256";
|
||||
my $res = backtick("$Nix::Config::binDir/nix-store", "--add", $srcPath);
|
||||
chomp $res;
|
||||
return $res;
|
||||
};
|
||||
|
||||
*isValidPath = sub {
|
||||
my ($path) = @_;
|
||||
my $res = backtick("$Nix::Config::binDir/nix-store", "--check-validity", "--print-invalid", $path);
|
||||
chomp $res;
|
||||
return $res ne $path;
|
||||
};
|
||||
|
||||
*queryPathHash = sub {
|
||||
my ($path) = @_;
|
||||
my $res = backtick("$Nix::Config::binDir/nix-store", "--query", "--hash", $path);
|
||||
chomp $res;
|
||||
return $res;
|
||||
};
|
||||
}
|
||||
require XSLoader;
|
||||
XSLoader::load('Nix::Store', $VERSION);
|
||||
|
||||
1;
|
||||
__END__
|
||||
|
|
|
|||
|
|
@ -351,3 +351,13 @@ void addTempRoot(char * storePath)
|
|||
} catch (Error & e) {
|
||||
croak("%s", e.what());
|
||||
}
|
||||
|
||||
|
||||
SV * getBinDir()
|
||||
PPCODE:
|
||||
XPUSHs(sv_2mortal(newSVpv(settings.nixBinDir.c_str(), 0)));
|
||||
|
||||
|
||||
SV * getStoreDir()
|
||||
PPCODE:
|
||||
XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0)));
|
||||
|
|
|
|||
|
|
@ -356,6 +356,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
|
|||
, sEpsilon(symbols.create(""))
|
||||
, repair(NoRepair)
|
||||
, store(store)
|
||||
, regexCache(makeRegexCache())
|
||||
, baseEnv(allocEnv(128))
|
||||
, staticBaseEnv(false, 0)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
#include "symbol-table.hh"
|
||||
#include "config.hh"
|
||||
|
||||
#include <regex>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
|
|
@ -65,6 +64,11 @@ typedef std::list<SearchPathElem> SearchPath;
|
|||
void initGC();
|
||||
|
||||
|
||||
struct RegexCache;
|
||||
|
||||
std::shared_ptr<RegexCache> makeRegexCache();
|
||||
|
||||
|
||||
class EvalState
|
||||
{
|
||||
public:
|
||||
|
|
@ -120,7 +124,7 @@ private:
|
|||
std::unordered_map<Path, Path> resolvedPaths;
|
||||
|
||||
/* Cache used by prim_match(). */
|
||||
std::unordered_map<std::string, std::regex> regexCache;
|
||||
std::shared_ptr<RegexCache> regexCache;
|
||||
|
||||
public:
|
||||
|
||||
|
|
|
|||
|
|
@ -48,17 +48,17 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
|
|||
resolvedRef = originalRef.resolve(state.store);
|
||||
auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
|
||||
if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
|
||||
flakeCache.push_back({resolvedRef, fetchedResolved.value()});
|
||||
fetched.emplace(fetchedResolved.value());
|
||||
flakeCache.push_back({resolvedRef, *fetchedResolved});
|
||||
fetched.emplace(*fetchedResolved);
|
||||
}
|
||||
else {
|
||||
throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
|
||||
}
|
||||
}
|
||||
flakeCache.push_back({originalRef, fetched.value()});
|
||||
flakeCache.push_back({originalRef, *fetched});
|
||||
}
|
||||
|
||||
auto [tree, lockedRef] = fetched.value();
|
||||
auto [tree, lockedRef] = *fetched;
|
||||
|
||||
debug("got tree '%s' from '%s'",
|
||||
state.store->printStorePath(tree.storePath), lockedRef);
|
||||
|
|
@ -215,10 +215,9 @@ static Flake getFlake(
|
|||
|
||||
if (auto outputs = vInfo.attrs->get(sOutputs)) {
|
||||
expectType(state, tLambda, *outputs->value, *outputs->pos);
|
||||
flake.vOutputs = allocRootValue(outputs->value);
|
||||
|
||||
if ((*flake.vOutputs)->lambda.fun->matchAttrs) {
|
||||
for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) {
|
||||
if (outputs->value->lambda.fun->matchAttrs) {
|
||||
for (auto & formal : outputs->value->lambda.fun->formals->formals) {
|
||||
if (formal.name != state.sSelf)
|
||||
flake.inputs.emplace(formal.name, FlakeInput {
|
||||
.ref = parseFlakeRef(formal.name)
|
||||
|
|
@ -367,7 +366,7 @@ LockedFlake lockFlake(
|
|||
|
||||
/* If we have an --update-input flag for an input
|
||||
of this input, then we must fetch the flake to
|
||||
to update it. */
|
||||
update it. */
|
||||
auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
|
||||
|
||||
auto hasChildUpdate =
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ struct Flake
|
|||
std::optional<std::string> description;
|
||||
std::shared_ptr<const fetchers::Tree> sourceInfo;
|
||||
FlakeInputs inputs;
|
||||
RootValue vOutputs;
|
||||
~Flake();
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#include "flakeref.hh"
|
||||
#include "store-api.hh"
|
||||
#include "url.hh"
|
||||
#include "url-parts.hh"
|
||||
#include "fetchers.hh"
|
||||
#include "registry.hh"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#include "lockfile.hh"
|
||||
#include "store-api.hh"
|
||||
#include "url-parts.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
|
|
|
|||
|
|
@ -115,6 +115,14 @@ public:
|
|||
{
|
||||
return handle_value<void(Value&, const char*)>(mkString, val.c_str());
|
||||
}
|
||||
#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
|
||||
bool binary(binary_t&)
|
||||
{
|
||||
// This function ought to be unreachable
|
||||
assert(false);
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool start_object(std::size_t len)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -42,6 +42,6 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
|
|||
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
|
||||
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
|
||||
|
||||
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
|
||||
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh
|
||||
|
||||
$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
|
||||
|
|
|
|||
|
|
@ -614,8 +614,7 @@ Path resolveExprPath(Path path)
|
|||
// Basic cycle/depth limit to avoid infinite loops.
|
||||
if (++followCount >= maxFollow)
|
||||
throw Error("too many symbolic links encountered while traversing the path '%s'", path);
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting status of '%s'", path);
|
||||
st = lstat(path);
|
||||
if (!S_ISLNK(st.st_mode)) break;
|
||||
path = absPath(readLink(path), dirOf(path));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2236,6 +2236,10 @@ static RegisterPrimOp primop_catAttrs({
|
|||
static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
{
|
||||
state.forceValue(*args[0], pos);
|
||||
if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) {
|
||||
state.mkAttrs(v, 0);
|
||||
return;
|
||||
}
|
||||
if (args[0]->type != tLambda)
|
||||
throw TypeError({
|
||||
.hint = hintfmt("'functionArgs' requires a function"),
|
||||
|
|
@ -3085,17 +3089,25 @@ static RegisterPrimOp primop_hashString({
|
|||
.fun = prim_hashString,
|
||||
});
|
||||
|
||||
/* Match a regular expression against a string and return either
|
||||
‘null’ or a list containing substring matches. */
|
||||
struct RegexCache
|
||||
{
|
||||
std::unordered_map<std::string, std::regex> cache;
|
||||
};
|
||||
|
||||
std::shared_ptr<RegexCache> makeRegexCache()
|
||||
{
|
||||
return std::make_shared<RegexCache>();
|
||||
}
|
||||
|
||||
void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
{
|
||||
auto re = state.forceStringNoCtx(*args[0], pos);
|
||||
|
||||
try {
|
||||
|
||||
auto regex = state.regexCache.find(re);
|
||||
if (regex == state.regexCache.end())
|
||||
regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first;
|
||||
auto regex = state.regexCache->cache.find(re);
|
||||
if (regex == state.regexCache->cache.end())
|
||||
regex = state.regexCache->cache.emplace(re, std::regex(re, std::regex::extended)).first;
|
||||
|
||||
PathSet context;
|
||||
const std::string str = state.forceString(*args[1], context, pos);
|
||||
|
|
@ -3565,13 +3577,11 @@ void EvalState::createBaseEnv()
|
|||
|
||||
/* Add a wrapper around the derivation primop that computes the
|
||||
`drvPath' and `outPath' attributes lazily. */
|
||||
try {
|
||||
string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true);
|
||||
sDerivationNix = symbols.create(path);
|
||||
evalFile(path, v);
|
||||
addConstant("derivation", v);
|
||||
} catch (SysError &) {
|
||||
}
|
||||
sDerivationNix = symbols.create("//builtin/derivation.nix");
|
||||
eval(parse(
|
||||
#include "primops/derivation.nix.gen.hh"
|
||||
, foFile, sDerivationNix, "/", staticBaseEnv), v);
|
||||
addConstant("derivation", v);
|
||||
|
||||
/* Now that we've added all primops, sort the `builtins' set,
|
||||
because attribute lookups expect it to be sorted. */
|
||||
|
|
|
|||
|
|
@ -3,8 +3,7 @@
|
|||
#include "store-api.hh"
|
||||
#include "fetchers.hh"
|
||||
#include "url.hh"
|
||||
|
||||
#include <regex>
|
||||
#include "url-parts.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
#include "globals.hh"
|
||||
#include "tarfile.hh"
|
||||
#include "store-api.hh"
|
||||
#include "url-parts.hh"
|
||||
|
||||
#include <sys/time.h>
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include "globals.hh"
|
||||
#include "store-api.hh"
|
||||
#include "types.hh"
|
||||
#include "url-parts.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
|
|
@ -21,9 +22,9 @@ struct DownloadUrl
|
|||
: url(url), access_token_header(access_token_header) { }
|
||||
};
|
||||
|
||||
// A github or gitlab url
|
||||
const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
|
||||
std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
|
||||
// A github or gitlab host
|
||||
const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
|
||||
std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
|
||||
|
||||
struct GitArchiveInputScheme : InputScheme
|
||||
{
|
||||
|
|
@ -65,9 +66,9 @@ struct GitArchiveInputScheme : InputScheme
|
|||
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
|
||||
ref = value;
|
||||
}
|
||||
else if (name == "url") {
|
||||
if (!std::regex_match(value, urlRegex))
|
||||
throw BadURL("URL '%s' contains an invalid instance url", url.url);
|
||||
else if (name == "host") {
|
||||
if (!std::regex_match(value, hostRegex))
|
||||
throw BadURL("URL '%s' contains an invalid instance host", url.url);
|
||||
host_url = value;
|
||||
}
|
||||
// FIXME: barf on unsupported attributes
|
||||
|
|
@ -82,7 +83,7 @@ struct GitArchiveInputScheme : InputScheme
|
|||
input.attrs.insert_or_assign("repo", path[1]);
|
||||
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) input.attrs.insert_or_assign("ref", *ref);
|
||||
if (host_url) input.attrs.insert_or_assign("url", *host_url);
|
||||
if (host_url) input.attrs.insert_or_assign("host", *host_url);
|
||||
|
||||
return input;
|
||||
}
|
||||
|
|
@ -92,7 +93,7 @@ struct GitArchiveInputScheme : InputScheme
|
|||
if (maybeGetStrAttr(attrs, "type") != type()) return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
|
||||
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
|
||||
throw Error("unsupported input attribute '%s'", name);
|
||||
|
||||
getStrAttr(attrs, "owner");
|
||||
|
|
@ -230,7 +231,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
|
|||
{
|
||||
// FIXME: use regular /archive URLs instead? api.github.com
|
||||
// might have stricter rate limits.
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
|
||||
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
|
||||
input.getRev()->to_string(Base16, false));
|
||||
|
|
@ -264,7 +265,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
|
|||
|
||||
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
|
||||
{
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
|
||||
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#include "fetchers.hh"
|
||||
#include "url-parts.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
#include "globals.hh"
|
||||
#include "tarfile.hh"
|
||||
#include "store-api.hh"
|
||||
#include "url-parts.hh"
|
||||
|
||||
#include <sys/time.h>
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
#include "nar-accessor.hh"
|
||||
#include "json.hh"
|
||||
#include "thread-pool.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#include <chrono>
|
||||
#include <future>
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#include "daemon.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "topo-sort.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
|
|
@ -295,9 +296,21 @@ public:
|
|||
~Worker();
|
||||
|
||||
/* Make a goal (with caching). */
|
||||
GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath,
|
||||
const BasicDerivation & drv, BuildMode buildMode = bmNormal);
|
||||
|
||||
/* derivation goal */
|
||||
private:
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
|
||||
const StorePath & drvPath, const StringSet & wantedOutputs,
|
||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
|
||||
public:
|
||||
std::shared_ptr<DerivationGoal> makeDerivationGoal(
|
||||
const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
|
||||
const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
|
||||
|
||||
/* substitution goal */
|
||||
GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
|
||||
|
||||
/* Remove a dead goal. */
|
||||
|
|
@ -948,10 +961,12 @@ private:
|
|||
friend struct RestrictedStore;
|
||||
|
||||
public:
|
||||
DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs,
|
||||
Worker & worker, BuildMode buildMode = bmNormal);
|
||||
DerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
Worker & worker, BuildMode buildMode = bmNormal);
|
||||
const StringSet & wantedOutputs, Worker & worker,
|
||||
BuildMode buildMode = bmNormal);
|
||||
~DerivationGoal();
|
||||
|
||||
/* Whether we need to perform hash rewriting if there are valid output paths. */
|
||||
|
|
@ -1084,8 +1099,8 @@ private:
|
|||
const Path DerivationGoal::homeDir = "/homeless-shelter";
|
||||
|
||||
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs,
|
||||
Worker & worker, BuildMode buildMode)
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker)
|
||||
, useDerivation(true)
|
||||
, drvPath(drvPath)
|
||||
|
|
@ -1093,7 +1108,9 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want
|
|||
, buildMode(buildMode)
|
||||
{
|
||||
state = &DerivationGoal::getDerivation;
|
||||
name = fmt("building of '%s'", worker.store.printStorePath(this->drvPath));
|
||||
name = fmt(
|
||||
"building of '%s' from .drv file",
|
||||
StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
|
||||
trace("created");
|
||||
|
||||
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
|
||||
|
|
@ -1102,15 +1119,18 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & want
|
|||
|
||||
|
||||
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
Worker & worker, BuildMode buildMode)
|
||||
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
|
||||
: Goal(worker)
|
||||
, useDerivation(false)
|
||||
, drvPath(drvPath)
|
||||
, wantedOutputs(wantedOutputs)
|
||||
, buildMode(buildMode)
|
||||
{
|
||||
this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
|
||||
state = &DerivationGoal::haveDerivation;
|
||||
name = fmt("building of %s", StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
|
||||
name = fmt(
|
||||
"building of '%s' from in-memory derivation",
|
||||
StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
|
||||
trace("created");
|
||||
|
||||
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
|
||||
|
|
@ -1674,7 +1694,34 @@ void DerivationGoal::tryLocalBuild() {
|
|||
}
|
||||
|
||||
|
||||
void replaceValidPath(const Path & storePath, const Path tmpPath)
|
||||
static void chmod_(const Path & path, mode_t mode)
|
||||
{
|
||||
if (chmod(path.c_str(), mode) == -1)
|
||||
throw SysError("setting permissions on '%s'", path);
|
||||
}
|
||||
|
||||
|
||||
/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
|
||||
it's a directory and we're not root (to be able to update the
|
||||
directory's parent link ".."). */
|
||||
static void movePath(const Path & src, const Path & dst)
|
||||
{
|
||||
auto st = lstat(src);
|
||||
|
||||
bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
|
||||
|
||||
if (changePerm)
|
||||
chmod_(src, st.st_mode | S_IWUSR);
|
||||
|
||||
if (rename(src.c_str(), dst.c_str()))
|
||||
throw SysError("renaming '%1%' to '%2%'", src, dst);
|
||||
|
||||
if (changePerm)
|
||||
chmod_(dst, st.st_mode);
|
||||
}
|
||||
|
||||
|
||||
void replaceValidPath(const Path & storePath, const Path & tmpPath)
|
||||
{
|
||||
/* We can't atomically replace storePath (the original) with
|
||||
tmpPath (the replacement), so we have to move it out of the
|
||||
|
|
@ -1682,11 +1729,20 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
|
|||
we're repairing (say) Glibc, we end up with a broken system. */
|
||||
Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str();
|
||||
if (pathExists(storePath))
|
||||
rename(storePath.c_str(), oldPath.c_str());
|
||||
if (rename(tmpPath.c_str(), storePath.c_str()) == -1) {
|
||||
rename(oldPath.c_str(), storePath.c_str()); // attempt to recover
|
||||
throw SysError("moving '%s' to '%s'", tmpPath, storePath);
|
||||
movePath(storePath, oldPath);
|
||||
|
||||
try {
|
||||
movePath(tmpPath, storePath);
|
||||
} catch (...) {
|
||||
try {
|
||||
// attempt to recover
|
||||
movePath(oldPath, storePath);
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
throw;
|
||||
}
|
||||
|
||||
deletePath(oldPath);
|
||||
}
|
||||
|
||||
|
|
@ -2004,13 +2060,6 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
}
|
||||
|
||||
|
||||
static void chmod_(const Path & path, mode_t mode)
|
||||
{
|
||||
if (chmod(path.c_str(), mode) == -1)
|
||||
throw SysError("setting permissions on '%s'", path);
|
||||
}
|
||||
|
||||
|
||||
int childEntry(void * arg)
|
||||
{
|
||||
((DerivationGoal *) arg)->runChild();
|
||||
|
|
@ -2366,10 +2415,7 @@ void DerivationGoal::startBuilder()
|
|||
for (auto & i : inputPaths) {
|
||||
auto p = worker.store.printStorePath(i);
|
||||
Path r = worker.store.toRealPath(p);
|
||||
struct stat st;
|
||||
if (lstat(r.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%s'", p);
|
||||
if (S_ISDIR(st.st_mode))
|
||||
if (S_ISDIR(lstat(r).st_mode))
|
||||
dirsInChroot.insert_or_assign(p, r);
|
||||
else
|
||||
linkOrCopy(r, chrootRootDir + p);
|
||||
|
|
@ -2949,14 +2995,6 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf
|
|||
goal.addDependency(info.path);
|
||||
}
|
||||
|
||||
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
|
||||
{
|
||||
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
|
||||
goal.addDependency(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
StorePath addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair = NoRepair) override
|
||||
{
|
||||
|
|
@ -3151,9 +3189,7 @@ void DerivationGoal::addDependency(const StorePath & path)
|
|||
if (pathExists(target))
|
||||
throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
|
||||
|
||||
struct stat st;
|
||||
if (lstat(source.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%s'", source);
|
||||
auto st = lstat(source);
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
|
||||
|
|
@ -3742,29 +3778,6 @@ void DerivationGoal::runChild()
|
|||
}
|
||||
|
||||
|
||||
static void moveCheckToStore(const Path & src, const Path & dst)
|
||||
{
|
||||
/* For the rename of directory to succeed, we must be running as root or
|
||||
the directory must be made temporarily writable (to update the
|
||||
directory's parent link ".."). */
|
||||
struct stat st;
|
||||
if (lstat(src.c_str(), &st) == -1) {
|
||||
throw SysError("getting attributes of path '%1%'", src);
|
||||
}
|
||||
|
||||
bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
|
||||
|
||||
if (changePerm)
|
||||
chmod_(src, st.st_mode | S_IWUSR);
|
||||
|
||||
if (rename(src.c_str(), dst.c_str()))
|
||||
throw SysError("renaming '%1%' to '%2%'", src, dst);
|
||||
|
||||
if (changePerm)
|
||||
chmod_(dst, st.st_mode);
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::registerOutputs()
|
||||
{
|
||||
/* When using a build hook, the build hook can register the output
|
||||
|
|
@ -3865,7 +3878,7 @@ void DerivationGoal::registerOutputs()
|
|||
something like that. */
|
||||
canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
|
||||
|
||||
debug("scanning for references for output %1 in temp location '%1%'", outputName, actualPath);
|
||||
debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
|
||||
|
||||
/* Pass blank Sink as we are not ready to hash data at this stage. */
|
||||
NullSink blank;
|
||||
|
|
@ -3920,7 +3933,6 @@ void DerivationGoal::registerOutputs()
|
|||
outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
|
||||
};
|
||||
|
||||
bool rewritten = false;
|
||||
std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
|
||||
[&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
|
||||
finish(skippedFinalPath.path);
|
||||
|
|
@ -3951,7 +3963,9 @@ void DerivationGoal::registerOutputs()
|
|||
StringSource source(*sink.s);
|
||||
restorePath(actualPath, source);
|
||||
|
||||
rewritten = true;
|
||||
/* FIXME: set proper permissions in restorePath() so
|
||||
we don't have to do another traversal. */
|
||||
canonicalisePathMetaData(actualPath, -1, inodesSeen);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -4034,7 +4048,7 @@ void DerivationGoal::registerOutputs()
|
|||
[&](DerivationOutputInputAddressed output) {
|
||||
/* input-addressed case */
|
||||
auto requiredFinalPath = output.path;
|
||||
/* Preemtively add rewrite rule for final hash, as that is
|
||||
/* Preemptively add rewrite rule for final hash, as that is
|
||||
what the NAR hash will use rather than normalized-self references */
|
||||
if (scratchPath != requiredFinalPath)
|
||||
outputRewrites.insert_or_assign(
|
||||
|
|
@ -4108,44 +4122,21 @@ void DerivationGoal::registerOutputs()
|
|||
else. No moving needed. */
|
||||
assert(newInfo.ca);
|
||||
} else {
|
||||
/* Temporarily add write perm so we can move, will be fixed
|
||||
later. */
|
||||
{
|
||||
struct stat st;
|
||||
auto & mode = st.st_mode;
|
||||
if (lstat(actualPath.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", actualPath);
|
||||
mode |= 0200;
|
||||
/* Try to change the perms, but only if the file isn't a
|
||||
symlink as symlinks permissions are mostly ignored and
|
||||
calling `chmod` on it will just forward the call to the
|
||||
target of the link. */
|
||||
if (!S_ISLNK(st.st_mode))
|
||||
if (chmod(actualPath.c_str(), mode) == -1)
|
||||
throw SysError("changing mode of '%1%' to %2$o", actualPath, mode);
|
||||
}
|
||||
if (rename(
|
||||
actualPath.c_str(),
|
||||
worker.store.toRealPath(finalDestPath).c_str()) == -1)
|
||||
throw SysError("moving build output '%1%' from it's temporary location to the Nix store", finalDestPath);
|
||||
actualPath = worker.store.toRealPath(finalDestPath);
|
||||
auto destPath = worker.store.toRealPath(finalDestPath);
|
||||
movePath(actualPath, destPath);
|
||||
actualPath = destPath;
|
||||
}
|
||||
}
|
||||
|
||||
/* Get rid of all weird permissions. This also checks that
|
||||
all files are owned by the build user, if applicable. */
|
||||
canonicalisePathMetaData(actualPath,
|
||||
buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen);
|
||||
|
||||
if (buildMode == bmCheck) {
|
||||
if (!worker.store.isValidPath(newInfo.path)) continue;
|
||||
ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
|
||||
if (newInfo.narHash != oldInfo.narHash) {
|
||||
worker.checkMismatch = true;
|
||||
if (settings.runDiffHook || settings.keepFailed) {
|
||||
Path dst = worker.store.toRealPath(finalDestPath + checkSuffix);
|
||||
auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
|
||||
deletePath(dst);
|
||||
moveCheckToStore(actualPath, dst);
|
||||
movePath(actualPath, dst);
|
||||
|
||||
handleDiffHook(
|
||||
buildUser ? buildUser->getUID() : getuid(),
|
||||
|
|
@ -5067,35 +5058,52 @@ Worker::~Worker()
|
|||
}
|
||||
|
||||
|
||||
GoalPtr Worker::makeDerivationGoal(const StorePath & path,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode)
|
||||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
|
||||
const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs,
|
||||
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
|
||||
{
|
||||
GoalPtr goal = derivationGoals[path].lock(); // FIXME
|
||||
if (!goal) {
|
||||
goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode);
|
||||
derivationGoals.insert_or_assign(path, goal);
|
||||
WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
|
||||
GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
|
||||
std::shared_ptr<DerivationGoal> goal;
|
||||
if (!abstract_goal) {
|
||||
goal = mkDrvGoal();
|
||||
abstract_goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
} else
|
||||
(dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs);
|
||||
} else {
|
||||
goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
|
||||
assert(goal);
|
||||
goal->addWantedOutputs(wantedOutputs);
|
||||
}
|
||||
return goal;
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
|
||||
const BasicDerivation & drv, BuildMode buildMode)
|
||||
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
|
||||
const StringSet & wantedOutputs, BuildMode buildMode)
|
||||
{
|
||||
auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode);
|
||||
wakeUp(goal);
|
||||
return goal;
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
|
||||
return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
|
||||
const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
|
||||
{
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
|
||||
return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
|
||||
{
|
||||
GoalPtr goal = substitutionGoals[path].lock(); // FIXME
|
||||
WeakGoalPtr & goal_weak = substitutionGoals[path];
|
||||
GoalPtr goal = goal_weak.lock(); // FIXME
|
||||
if (!goal) {
|
||||
goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
|
||||
substitutionGoals.insert_or_assign(path, goal);
|
||||
goal_weak = goal;
|
||||
wakeUp(goal);
|
||||
}
|
||||
return goal;
|
||||
|
|
@ -5526,7 +5534,7 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe
|
|||
BuildMode buildMode)
|
||||
{
|
||||
Worker worker(*this);
|
||||
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode);
|
||||
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
|
||||
|
||||
BuildResult result;
|
||||
|
||||
|
|
|
|||
|
|
@ -4,11 +4,13 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
std::string FixedOutputHash::printMethodAlgo() const {
|
||||
std::string FixedOutputHash::printMethodAlgo() const
|
||||
{
|
||||
return makeFileIngestionPrefix(method) + printHashType(hash.type);
|
||||
}
|
||||
|
||||
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
|
||||
std::string makeFileIngestionPrefix(const FileIngestionMethod m)
|
||||
{
|
||||
switch (m) {
|
||||
case FileIngestionMethod::Flat:
|
||||
return "";
|
||||
|
|
@ -26,7 +28,8 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
|
|||
+ hash.to_string(Base32, true);
|
||||
}
|
||||
|
||||
std::string renderContentAddress(ContentAddress ca) {
|
||||
std::string renderContentAddress(ContentAddress ca)
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](TextHash th) {
|
||||
return "text:" + th.hash.to_string(Base32, true);
|
||||
|
|
@ -37,54 +40,97 @@ std::string renderContentAddress(ContentAddress ca) {
|
|||
}, ca);
|
||||
}
|
||||
|
||||
ContentAddress parseContentAddress(std::string_view rawCa) {
|
||||
auto rest = rawCa;
|
||||
std::string renderContentAddressMethod(ContentAddressMethod cam)
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](TextHashMethod &th) {
|
||||
return std::string{"text:"} + printHashType(htSHA256);
|
||||
},
|
||||
[](FixedOutputHashMethod &fshm) {
|
||||
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
|
||||
}
|
||||
}, cam);
|
||||
}
|
||||
|
||||
/*
|
||||
Parses content address strings up to the hash.
|
||||
*/
|
||||
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
|
||||
{
|
||||
std::string_view wholeInput { rest };
|
||||
|
||||
std::string_view prefix;
|
||||
{
|
||||
auto optPrefix = splitPrefixTo(rest, ':');
|
||||
if (!optPrefix)
|
||||
throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", rawCa);
|
||||
throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", wholeInput);
|
||||
prefix = *optPrefix;
|
||||
}
|
||||
|
||||
auto parseHashType_ = [&](){
|
||||
auto hashTypeRaw = splitPrefixTo(rest, ':');
|
||||
if (!hashTypeRaw)
|
||||
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", rawCa);
|
||||
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
|
||||
HashType hashType = parseHashType(*hashTypeRaw);
|
||||
return std::move(hashType);
|
||||
};
|
||||
|
||||
// Switch on prefix
|
||||
if (prefix == "text") {
|
||||
// No parsing of the method, "text" only support flat.
|
||||
// No parsing of the ingestion method, "text" only support flat.
|
||||
HashType hashType = parseHashType_();
|
||||
if (hashType != htSHA256)
|
||||
throw Error("text content address hash should use %s, but instead uses %s",
|
||||
printHashType(htSHA256), printHashType(hashType));
|
||||
return TextHash {
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
|
||||
};
|
||||
return TextHashMethod {};
|
||||
} else if (prefix == "fixed") {
|
||||
// Parse method
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
if (splitPrefix(rest, "r:"))
|
||||
method = FileIngestionMethod::Recursive;
|
||||
HashType hashType = parseHashType_();
|
||||
return FixedOutputHash {
|
||||
.method = method,
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
|
||||
return FixedOutputHashMethod {
|
||||
.fileIngestionMethod = method,
|
||||
.hashType = std::move(hashType),
|
||||
};
|
||||
} else
|
||||
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
|
||||
}
|
||||
|
||||
ContentAddress parseContentAddress(std::string_view rawCa) {
|
||||
auto rest = rawCa;
|
||||
|
||||
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
|
||||
|
||||
return std::visit(
|
||||
overloaded {
|
||||
[&](TextHashMethod thm) {
|
||||
return ContentAddress(TextHash {
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
|
||||
});
|
||||
},
|
||||
[&](FixedOutputHashMethod fohMethod) {
|
||||
return ContentAddress(FixedOutputHash {
|
||||
.method = fohMethod.fileIngestionMethod,
|
||||
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
|
||||
});
|
||||
},
|
||||
}, caMethod);
|
||||
}
|
||||
|
||||
ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
|
||||
{
|
||||
std::string_view asPrefix {std::string{caMethod} + ":"};
|
||||
return parseContentAddressMethodPrefix(asPrefix);
|
||||
}
|
||||
|
||||
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt)
|
||||
{
|
||||
return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt);
|
||||
};
|
||||
|
||||
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
|
||||
return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
|
||||
};
|
||||
|
||||
std::string renderContentAddress(std::optional<ContentAddress> ca) {
|
||||
std::string renderContentAddress(std::optional<ContentAddress> ca)
|
||||
{
|
||||
return ca ? renderContentAddress(*ca) : "";
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -55,4 +55,23 @@ std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
|
|||
|
||||
Hash getContentAddressHash(const ContentAddress & ca);
|
||||
|
||||
/*
|
||||
We only have one way to hash text with references, so this is single-value
|
||||
type is only useful in std::variant.
|
||||
*/
|
||||
struct TextHashMethod { };
|
||||
struct FixedOutputHashMethod {
|
||||
FileIngestionMethod fileIngestionMethod;
|
||||
HashType hashType;
|
||||
};
|
||||
|
||||
typedef std::variant<
|
||||
TextHashMethod,
|
||||
FixedOutputHashMethod
|
||||
> ContentAddressMethod;
|
||||
|
||||
ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod);
|
||||
|
||||
std::string renderContentAddressMethod(ContentAddressMethod caMethod);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
#include "monitor-fd.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "store-api.hh"
|
||||
#include "local-store.hh"
|
||||
#include "finally.hh"
|
||||
#include "affinity.hh"
|
||||
#include "archive.hh"
|
||||
|
|
@ -240,6 +239,23 @@ struct ClientSettings
|
|||
}
|
||||
};
|
||||
|
||||
static void writeValidPathInfo(
|
||||
ref<Store> store,
|
||||
unsigned int clientVersion,
|
||||
Sink & to,
|
||||
std::shared_ptr<const ValidPathInfo> info)
|
||||
{
|
||||
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
|
||||
<< info->narHash.to_string(Base16, false);
|
||||
writeStorePaths(*store, to, info->references);
|
||||
to << info->registrationTime << info->narSize;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
|
||||
to << info->ultimate
|
||||
<< info->sigs
|
||||
<< renderContentAddress(info->ca);
|
||||
}
|
||||
}
|
||||
|
||||
static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
|
||||
Source & from, BufferedSink & to, unsigned int op)
|
||||
|
|
@ -350,47 +366,83 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
}
|
||||
|
||||
case wopAddToStore: {
|
||||
HashType hashAlgo;
|
||||
std::string baseName;
|
||||
FileIngestionMethod method;
|
||||
{
|
||||
bool fixed;
|
||||
uint8_t recursive;
|
||||
std::string hashAlgoRaw;
|
||||
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
|
||||
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
|
||||
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
|
||||
method = FileIngestionMethod { recursive };
|
||||
/* Compatibility hack. */
|
||||
if (!fixed) {
|
||||
hashAlgoRaw = "sha256";
|
||||
method = FileIngestionMethod::Recursive;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
|
||||
auto name = readString(from);
|
||||
auto camStr = readString(from);
|
||||
auto refs = readStorePaths<StorePathSet>(*store, from);
|
||||
bool repairBool;
|
||||
from >> repairBool;
|
||||
auto repair = RepairFlag{repairBool};
|
||||
|
||||
logger->startWork();
|
||||
auto pathInfo = [&]() {
|
||||
// NB: FramedSource must be out of scope before logger->stopWork();
|
||||
ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr);
|
||||
FramedSource source(from);
|
||||
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
|
||||
return std::visit(overloaded {
|
||||
[&](TextHashMethod &_) {
|
||||
// We could stream this by changing Store
|
||||
std::string contents = source.drain();
|
||||
auto path = store->addTextToStore(name, contents, refs, repair);
|
||||
return store->queryPathInfo(path);
|
||||
},
|
||||
[&](FixedOutputHashMethod &fohm) {
|
||||
if (!refs.empty())
|
||||
throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
|
||||
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
|
||||
return store->queryPathInfo(path);
|
||||
},
|
||||
}, contentAddressMethod);
|
||||
}();
|
||||
logger->stopWork();
|
||||
|
||||
to << store->printStorePath(pathInfo->path);
|
||||
writeValidPathInfo(store, clientVersion, to, pathInfo);
|
||||
|
||||
} else {
|
||||
HashType hashAlgo;
|
||||
std::string baseName;
|
||||
FileIngestionMethod method;
|
||||
{
|
||||
bool fixed;
|
||||
uint8_t recursive;
|
||||
std::string hashAlgoRaw;
|
||||
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
|
||||
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
|
||||
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
|
||||
method = FileIngestionMethod { recursive };
|
||||
/* Compatibility hack. */
|
||||
if (!fixed) {
|
||||
hashAlgoRaw = "sha256";
|
||||
method = FileIngestionMethod::Recursive;
|
||||
}
|
||||
hashAlgo = parseHashType(hashAlgoRaw);
|
||||
}
|
||||
hashAlgo = parseHashType(hashAlgoRaw);
|
||||
|
||||
StringSink saved;
|
||||
TeeSource savedNARSource(from, saved);
|
||||
RetrieveRegularNARSink savedRegular { saved };
|
||||
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
/* Get the entire NAR dump from the client and save it to
|
||||
a string so that we can pass it to
|
||||
addToStoreFromDump(). */
|
||||
ParseSink sink; /* null sink; just parse the NAR */
|
||||
parseDump(sink, savedNARSource);
|
||||
} else
|
||||
parseDump(savedRegular, from);
|
||||
|
||||
logger->startWork();
|
||||
if (!savedRegular.regular) throw Error("regular file expected");
|
||||
|
||||
// FIXME: try to stream directly from `from`.
|
||||
StringSource dumpSource { *saved.s };
|
||||
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
|
||||
logger->stopWork();
|
||||
|
||||
to << store->printStorePath(path);
|
||||
}
|
||||
|
||||
StringSink saved;
|
||||
TeeSource savedNARSource(from, saved);
|
||||
RetrieveRegularNARSink savedRegular { saved };
|
||||
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
/* Get the entire NAR dump from the client and save it to
|
||||
a string so that we can pass it to
|
||||
addToStoreFromDump(). */
|
||||
ParseSink sink; /* null sink; just parse the NAR */
|
||||
parseDump(sink, savedNARSource);
|
||||
} else
|
||||
parseDump(savedRegular, from);
|
||||
|
||||
logger->startWork();
|
||||
if (!savedRegular.regular) throw Error("regular file expected");
|
||||
|
||||
// FIXME: try to stream directly from `from`.
|
||||
StringSource dumpSource { *saved.s };
|
||||
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
|
||||
logger->stopWork();
|
||||
|
||||
to << store->printStorePath(path);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -494,6 +546,20 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
are in fact content-addressed if we don't trust them. */
|
||||
assert(derivationIsCA(drv.type()) || trusted);
|
||||
|
||||
/* Recompute the derivation path when we cannot trust the original. */
|
||||
if (!trusted) {
|
||||
/* Recomputing the derivation path for input-address derivations
|
||||
makes it harder to audit them after the fact, since we need the
|
||||
original not-necessarily-resolved derivation to verify the drv
|
||||
derivation as adequate claim to the input-addressed output
|
||||
paths. */
|
||||
assert(derivationIsCA(drv.type()));
|
||||
|
||||
Derivation drv2;
|
||||
static_cast<BasicDerivation &>(drv2) = drv;
|
||||
drvPath = writeDerivation(*store, Derivation { drv2 });
|
||||
}
|
||||
|
||||
auto res = store->buildDerivation(drvPath, drv, buildMode);
|
||||
logger->stopWork();
|
||||
to << res.status << res.errorMsg;
|
||||
|
|
@ -675,15 +741,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
if (info) {
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
|
||||
to << 1;
|
||||
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
|
||||
<< info->narHash.to_string(Base16, false);
|
||||
writeStorePaths(*store, to, info->references);
|
||||
to << info->registrationTime << info->narSize;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
|
||||
to << info->ultimate
|
||||
<< info->sigs
|
||||
<< renderContentAddress(info->ca);
|
||||
}
|
||||
writeValidPathInfo(store, clientVersion, to, info);
|
||||
} else {
|
||||
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
|
||||
to << 0;
|
||||
|
|
@ -749,59 +807,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
info.ultimate = false;
|
||||
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
|
||||
|
||||
struct FramedSource : Source
|
||||
{
|
||||
Source & from;
|
||||
bool eof = false;
|
||||
std::vector<unsigned char> pending;
|
||||
size_t pos = 0;
|
||||
|
||||
FramedSource(Source & from) : from(from)
|
||||
{ }
|
||||
|
||||
~FramedSource()
|
||||
{
|
||||
if (!eof) {
|
||||
while (true) {
|
||||
auto n = readInt(from);
|
||||
if (!n) break;
|
||||
std::vector<unsigned char> data(n);
|
||||
from(data.data(), n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t read(unsigned char * data, size_t len) override
|
||||
{
|
||||
if (eof) throw EndOfFile("reached end of FramedSource");
|
||||
|
||||
if (pos >= pending.size()) {
|
||||
size_t len = readInt(from);
|
||||
if (!len) {
|
||||
eof = true;
|
||||
return 0;
|
||||
}
|
||||
pending = std::vector<unsigned char>(len);
|
||||
pos = 0;
|
||||
from(pending.data(), len);
|
||||
}
|
||||
|
||||
auto n = std::min(len, pending.size() - pos);
|
||||
memcpy(data, pending.data() + pos, n);
|
||||
pos += n;
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
logger->startWork();
|
||||
|
||||
{
|
||||
FramedSource source(from);
|
||||
store->addToStore(info, source, (RepairFlag) repair,
|
||||
dontCheckSigs ? NoCheckSigs : CheckSigs);
|
||||
}
|
||||
|
||||
logger->stopWork();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#include "store-api.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
#include "s3.hh"
|
||||
#include "compression.hh"
|
||||
#include "finally.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#ifdef ENABLE_S3
|
||||
#include <aws/core/client/ClientConfiguration.h>
|
||||
|
|
|
|||
|
|
@ -574,9 +574,12 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
|
|||
/* If keep-derivations is set and this is a derivation, then
|
||||
don't delete the derivation if any of the outputs are alive. */
|
||||
if (state.gcKeepDerivations && path.isDerivation()) {
|
||||
for (auto & i : queryDerivationOutputs(path))
|
||||
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
|
||||
incoming.insert(i);
|
||||
for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
|
||||
if (maybeOutPath &&
|
||||
isValidPath(*maybeOutPath) &&
|
||||
queryPathInfo(*maybeOutPath)->deriver == path
|
||||
)
|
||||
incoming.insert(*maybeOutPath);
|
||||
}
|
||||
|
||||
/* If keep-outputs is set, then don't delete this path if there
|
||||
|
|
@ -660,9 +663,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
|
|||
if (name == "." || name == "..") continue;
|
||||
Path path = linksDir + "/" + name;
|
||||
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st) == -1)
|
||||
throw SysError("statting '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
|
||||
if (st.st_nlink != 1) {
|
||||
actualSize += st.st_size;
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include "util.hh"
|
||||
#include "archive.hh"
|
||||
#include "args.hh"
|
||||
#include "abstract-setting-to-json.hh"
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
|
|
@ -41,6 +42,7 @@ Settings::Settings()
|
|||
{
|
||||
buildUsersGroup = getuid() == 0 ? "nixbld" : "";
|
||||
lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
|
||||
allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
|
||||
|
||||
caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
|
||||
if (caFile == "") {
|
||||
|
|
@ -146,6 +148,12 @@ bool Settings::isWSL1()
|
|||
|
||||
const string nixVersion = PACKAGE_VERSION;
|
||||
|
||||
NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
|
||||
{SandboxMode::smEnabled, true},
|
||||
{SandboxMode::smRelaxed, "relaxed"},
|
||||
{SandboxMode::smDisabled, false},
|
||||
});
|
||||
|
||||
template<> void BaseSetting<SandboxMode>::set(const std::string & str)
|
||||
{
|
||||
if (str == "true") value = smEnabled;
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
#include "types.hh"
|
||||
#include "config.hh"
|
||||
#include "abstractsettingtojson.hh"
|
||||
#include "util.hh"
|
||||
|
||||
#include <map>
|
||||
|
|
@ -884,6 +883,19 @@ public:
|
|||
|
||||
Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
|
||||
"Path or URI of the global flake registry."};
|
||||
|
||||
Setting<bool> allowSymlinkedStore{
|
||||
this, false, "allow-symlinked-store",
|
||||
R"(
|
||||
If set to `true`, Nix will stop complaining if the store directory
|
||||
(typically /nix/store) contains symlink components.
|
||||
|
||||
This risks making some builds "impure" because builders sometimes
|
||||
"canonicalise" paths by resolving all symlink components. Problems
|
||||
occur if those builds are then deployed to machines where /nix/store
|
||||
resolves to a different location from that of the build machine. You
|
||||
can enable this setting if you are sure you're not going to do that.
|
||||
)"};
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include "filetransfer.hh"
|
||||
#include "globals.hh"
|
||||
#include "nar-info-disk-cache.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include "worker-protocol.hh"
|
||||
#include "ssh.hh"
|
||||
#include "derivations.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#include "derivations.hh"
|
||||
#include "nar-info.hh"
|
||||
#include "references.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#include <iostream>
|
||||
#include <algorithm>
|
||||
|
|
@ -109,12 +110,11 @@ LocalStore::LocalStore(const Params & params)
|
|||
}
|
||||
|
||||
/* Ensure that the store and its parents are not symlinks. */
|
||||
if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") {
|
||||
if (!settings.allowSymlinkedStore) {
|
||||
Path path = realStoreDir;
|
||||
struct stat st;
|
||||
while (path != "/") {
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting status of '%1%'", path);
|
||||
st = lstat(path);
|
||||
if (S_ISLNK(st.st_mode))
|
||||
throw Error(
|
||||
"the path '%1%' is a symlink; "
|
||||
|
|
@ -418,10 +418,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
|
|||
|
||||
void canonicaliseTimestampAndPermissions(const Path & path)
|
||||
{
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
canonicaliseTimestampAndPermissions(path, st);
|
||||
canonicaliseTimestampAndPermissions(path, lstat(path));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -439,9 +436,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
|
|||
}
|
||||
#endif
|
||||
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
|
||||
/* Really make sure that the path is of a supported type. */
|
||||
if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
|
||||
|
|
@ -477,8 +472,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
|
|||
ensure that we don't fail on hard links within the same build
|
||||
(i.e. "touch $out/foo; ln $out/foo $out/bar"). */
|
||||
if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
|
||||
assert(!S_ISDIR(st.st_mode));
|
||||
if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
|
||||
if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
|
||||
throw BuildError("invalid ownership on file '%1%'", path);
|
||||
mode_t mode = st.st_mode & ~S_IFMT;
|
||||
assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
|
||||
|
|
@ -521,9 +515,7 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino
|
|||
|
||||
/* On platforms that don't have lchown(), the top-level path can't
|
||||
be a symlink, since we can't change its ownership. */
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
|
||||
if (st.st_uid != geteuid()) {
|
||||
assert(S_ISLNK(st.st_mode));
|
||||
|
|
@ -1454,7 +1446,7 @@ static void makeMutable(const Path & path)
|
|||
{
|
||||
checkInterrupt();
|
||||
|
||||
struct stat st = lstat(path);
|
||||
auto st = lstat(path);
|
||||
|
||||
if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return;
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
#include "store-api.hh"
|
||||
#include "thread-pool.hh"
|
||||
#include "topo-sort.hh"
|
||||
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,18 @@
|
|||
#include "names.hh"
|
||||
#include "util.hh"
|
||||
|
||||
#include <regex>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
struct Regex
|
||||
{
|
||||
std::regex regex;
|
||||
};
|
||||
|
||||
|
||||
DrvName::DrvName()
|
||||
{
|
||||
name = "";
|
||||
|
|
@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0)
|
|||
}
|
||||
|
||||
|
||||
DrvName::~DrvName()
|
||||
{ }
|
||||
|
||||
|
||||
bool DrvName::matches(DrvName & n)
|
||||
{
|
||||
if (name != "*") {
|
||||
if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended));
|
||||
if (!std::regex_match(n.name, *regex)) return false;
|
||||
if (!regex) {
|
||||
regex = std::make_unique<Regex>();
|
||||
regex->regex = std::regex(name, std::regex::extended);
|
||||
}
|
||||
if (!std::regex_match(n.name, regex->regex)) return false;
|
||||
}
|
||||
if (version != "" && version != n.version) return false;
|
||||
return true;
|
||||
|
|
@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs)
|
|||
{
|
||||
DrvNames result;
|
||||
for (auto & i : opArgs)
|
||||
result.push_back(DrvName(i));
|
||||
result.emplace_back(i);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,10 +3,11 @@
|
|||
#include <memory>
|
||||
|
||||
#include "types.hh"
|
||||
#include <regex>
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct Regex;
|
||||
|
||||
struct DrvName
|
||||
{
|
||||
string fullName;
|
||||
|
|
@ -16,10 +17,12 @@ struct DrvName
|
|||
|
||||
DrvName();
|
||||
DrvName(std::string_view s);
|
||||
~DrvName();
|
||||
|
||||
bool matches(DrvName & n);
|
||||
|
||||
private:
|
||||
std::unique_ptr<std::regex> regex;
|
||||
std::unique_ptr<Regex> regex;
|
||||
};
|
||||
|
||||
typedef list<DrvName> DrvNames;
|
||||
|
|
|
|||
|
|
@ -17,9 +17,7 @@ namespace nix {
|
|||
|
||||
static void makeWritable(const Path & path)
|
||||
{
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
|
||||
throw SysError("changing writability of '%1%'", path);
|
||||
}
|
||||
|
|
@ -94,9 +92,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
|||
{
|
||||
checkInterrupt();
|
||||
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
|
||||
#if __APPLE__
|
||||
/* HFS/macOS has some undocumented security feature disabling hardlinking for
|
||||
|
|
@ -187,9 +183,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
|||
|
||||
/* Yes! We've seen a file with the same contents. Replace the
|
||||
current file with a hard link to that file. */
|
||||
struct stat stLink;
|
||||
if (lstat(linkPath.c_str(), &stLink))
|
||||
throw SysError("getting attributes of path '%1%'", linkPath);
|
||||
auto stLink = lstat(linkPath);
|
||||
|
||||
if (st.st_ino == stLink.st_ino) {
|
||||
debug(format("'%1%' is already linked to '%2%'") % path % linkPath);
|
||||
|
|
|
|||
|
|
@ -39,13 +39,10 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
|
|||
for (auto & i : readDirectory(profileDir)) {
|
||||
if (auto n = parseName(profileName, i.name)) {
|
||||
auto path = profileDir + "/" + i.name;
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st) != 0)
|
||||
throw SysError("statting '%1%'", path);
|
||||
gens.push_back({
|
||||
.number = *n,
|
||||
.path = path,
|
||||
.creationTime = st.st_mtime
|
||||
.creationTime = lstat(path).st_mtime
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include "pool.hh"
|
||||
#include "finally.hh"
|
||||
#include "logging.hh"
|
||||
#include "callback.hh"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
|
|
@ -97,7 +98,16 @@ RemoteStore::RemoteStore(const Params & params)
|
|||
, RemoteStoreConfig(params)
|
||||
, connections(make_ref<Pool<Connection>>(
|
||||
std::max(1, (int) maxConnections),
|
||||
[this]() { return openConnectionWrapper(); },
|
||||
[this]() {
|
||||
auto conn = openConnectionWrapper();
|
||||
try {
|
||||
initConnection(*conn);
|
||||
} catch (...) {
|
||||
failed = true;
|
||||
throw;
|
||||
}
|
||||
return conn;
|
||||
},
|
||||
[this](const ref<Connection> & r) {
|
||||
return
|
||||
r->to.good()
|
||||
|
|
@ -182,8 +192,6 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
|
|||
|
||||
conn->startTime = std::chrono::steady_clock::now();
|
||||
|
||||
initConnection(*conn);
|
||||
|
||||
return conn;
|
||||
}
|
||||
|
||||
|
|
@ -299,6 +307,8 @@ struct ConnectionHandle
|
|||
std::rethrow_exception(ex);
|
||||
}
|
||||
}
|
||||
|
||||
void withFramedSink(std::function<void(Sink & sink)> fun);
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -412,11 +422,28 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
|
|||
}
|
||||
|
||||
|
||||
ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path)
|
||||
{
|
||||
auto deriver = readString(conn->from);
|
||||
auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
|
||||
auto info = make_ref<ValidPathInfo>(path, narHash);
|
||||
if (deriver != "") info->deriver = parseStorePath(deriver);
|
||||
info->references = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
conn->from >> info->registrationTime >> info->narSize;
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
|
||||
conn->from >> info->ultimate;
|
||||
info->sigs = readStrings<StringSet>(conn->from);
|
||||
info->ca = parseContentAddressOpt(readString(conn->from));
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
|
||||
void RemoteStore::queryPathInfoUncached(const StorePath & path,
|
||||
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
|
||||
{
|
||||
try {
|
||||
std::shared_ptr<ValidPathInfo> info;
|
||||
std::shared_ptr<const ValidPathInfo> info;
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << wopQueryPathInfo << printStorePath(path);
|
||||
|
|
@ -432,17 +459,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
|
|||
bool valid; conn->from >> valid;
|
||||
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
|
||||
}
|
||||
auto deriver = readString(conn->from);
|
||||
auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
|
||||
info = std::make_shared<ValidPathInfo>(path, narHash);
|
||||
if (deriver != "") info->deriver = parseStorePath(deriver);
|
||||
info->references = readStorePaths<StorePathSet>(*this, conn->from);
|
||||
conn->from >> info->registrationTime >> info->narSize;
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
|
||||
conn->from >> info->ultimate;
|
||||
info->sigs = readStrings<StringSet>(conn->from);
|
||||
info->ca = parseContentAddressOpt(readString(conn->from));
|
||||
}
|
||||
info = readValidPathInfo(conn, path);
|
||||
}
|
||||
callback(std::move(info));
|
||||
} catch (...) { callback.rethrow(); }
|
||||
|
|
@ -517,6 +534,93 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
|
|||
}
|
||||
|
||||
|
||||
ref<const ValidPathInfo> RemoteStore::addCAToStore(
|
||||
Source & dump,
|
||||
const string & name,
|
||||
ContentAddressMethod caMethod,
|
||||
const StorePathSet & references,
|
||||
RepairFlag repair)
|
||||
{
|
||||
std::optional<ConnectionHandle> conn_(getConnection());
|
||||
auto & conn = *conn_;
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) {
|
||||
|
||||
conn->to
|
||||
<< wopAddToStore
|
||||
<< name
|
||||
<< renderContentAddressMethod(caMethod);
|
||||
writeStorePaths(*this, conn->to, references);
|
||||
conn->to << repair;
|
||||
|
||||
conn.withFramedSink([&](Sink & sink) {
|
||||
dump.drainInto(sink);
|
||||
});
|
||||
|
||||
auto path = parseStorePath(readString(conn->from));
|
||||
return readValidPathInfo(conn, path);
|
||||
}
|
||||
else {
|
||||
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
|
||||
|
||||
std::visit(overloaded {
|
||||
[&](TextHashMethod thm) -> void {
|
||||
std::string s = dump.drain();
|
||||
conn->to << wopAddTextToStore << name << s;
|
||||
writeStorePaths(*this, conn->to, references);
|
||||
conn.processStderr();
|
||||
},
|
||||
[&](FixedOutputHashMethod fohm) -> void {
|
||||
conn->to
|
||||
<< wopAddToStore
|
||||
<< name
|
||||
<< ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
|
||||
<< (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0)
|
||||
<< printHashType(fohm.hashType);
|
||||
|
||||
try {
|
||||
conn->to.written = 0;
|
||||
conn->to.warn = true;
|
||||
connections->incCapacity();
|
||||
{
|
||||
Finally cleanup([&]() { connections->decCapacity(); });
|
||||
if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) {
|
||||
dump.drainInto(conn->to);
|
||||
} else {
|
||||
std::string contents = dump.drain();
|
||||
dumpString(contents, conn->to);
|
||||
}
|
||||
}
|
||||
conn->to.warn = false;
|
||||
conn.processStderr();
|
||||
} catch (SysError & e) {
|
||||
/* Daemon closed while we were sending the path. Probably OOM
|
||||
or I/O error. */
|
||||
if (e.errNo == EPIPE)
|
||||
try {
|
||||
conn.processStderr();
|
||||
} catch (EndOfFile & e) { }
|
||||
throw;
|
||||
}
|
||||
|
||||
}
|
||||
}, caMethod);
|
||||
auto path = parseStorePath(readString(conn->from));
|
||||
// Release our connection to prevent a deadlock in queryPathInfo().
|
||||
conn_.reset();
|
||||
return queryPathInfo(path);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method, HashType hashType, RepairFlag repair)
|
||||
{
|
||||
StorePathSet references;
|
||||
return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
|
||||
}
|
||||
|
||||
|
||||
void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs)
|
||||
{
|
||||
|
|
@ -557,78 +661,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
<< repair << !checkSigs;
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
|
||||
|
||||
conn->to.flush();
|
||||
|
||||
std::exception_ptr ex;
|
||||
|
||||
struct FramedSink : BufferedSink
|
||||
{
|
||||
ConnectionHandle & conn;
|
||||
std::exception_ptr & ex;
|
||||
|
||||
FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex)
|
||||
{ }
|
||||
|
||||
~FramedSink()
|
||||
{
|
||||
try {
|
||||
conn->to << 0;
|
||||
conn->to.flush();
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
|
||||
void write(const unsigned char * data, size_t len) override
|
||||
{
|
||||
/* Don't send more data if the remote has
|
||||
encountered an error. */
|
||||
if (ex) {
|
||||
auto ex2 = ex;
|
||||
ex = nullptr;
|
||||
std::rethrow_exception(ex2);
|
||||
}
|
||||
conn->to << len;
|
||||
conn->to(data, len);
|
||||
};
|
||||
};
|
||||
|
||||
/* Handle log messages / exceptions from the remote on a
|
||||
separate thread. */
|
||||
std::thread stderrThread([&]()
|
||||
{
|
||||
try {
|
||||
conn.processStderr(nullptr, nullptr, false);
|
||||
} catch (...) {
|
||||
ex = std::current_exception();
|
||||
}
|
||||
});
|
||||
|
||||
Finally joinStderrThread([&]()
|
||||
{
|
||||
if (stderrThread.joinable()) {
|
||||
stderrThread.join();
|
||||
if (ex) {
|
||||
try {
|
||||
std::rethrow_exception(ex);
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
{
|
||||
FramedSink sink(conn, ex);
|
||||
conn.withFramedSink([&](Sink & sink) {
|
||||
copyNAR(source, sink);
|
||||
sink.flush();
|
||||
}
|
||||
|
||||
stderrThread.join();
|
||||
if (ex)
|
||||
std::rethrow_exception(ex);
|
||||
|
||||
});
|
||||
} else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
|
||||
conn.processStderr(0, &source);
|
||||
} else {
|
||||
|
|
@ -639,57 +674,11 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
}
|
||||
|
||||
|
||||
StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
|
||||
|
||||
auto conn(getConnection());
|
||||
|
||||
Path srcPath(absPath(_srcPath));
|
||||
|
||||
conn->to
|
||||
<< wopAddToStore
|
||||
<< name
|
||||
<< ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
|
||||
<< (method == FileIngestionMethod::Recursive ? 1 : 0)
|
||||
<< printHashType(hashAlgo);
|
||||
|
||||
try {
|
||||
conn->to.written = 0;
|
||||
conn->to.warn = true;
|
||||
connections->incCapacity();
|
||||
{
|
||||
Finally cleanup([&]() { connections->decCapacity(); });
|
||||
dumpPath(srcPath, conn->to, filter);
|
||||
}
|
||||
conn->to.warn = false;
|
||||
conn.processStderr();
|
||||
} catch (SysError & e) {
|
||||
/* Daemon closed while we were sending the path. Probably OOM
|
||||
or I/O error. */
|
||||
if (e.errNo == EPIPE)
|
||||
try {
|
||||
conn.processStderr();
|
||||
} catch (EndOfFile & e) { }
|
||||
throw;
|
||||
}
|
||||
|
||||
return parseStorePath(readString(conn->from));
|
||||
}
|
||||
|
||||
|
||||
StorePath RemoteStore::addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair)
|
||||
{
|
||||
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
|
||||
|
||||
auto conn(getConnection());
|
||||
conn->to << wopAddTextToStore << name << s;
|
||||
writeStorePaths(*this, conn->to, references);
|
||||
|
||||
conn.processStderr();
|
||||
return parseStorePath(readString(conn->from));
|
||||
StringSource source(s);
|
||||
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -985,6 +974,49 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
|
||||
{
|
||||
(*this)->to.flush();
|
||||
|
||||
std::exception_ptr ex;
|
||||
|
||||
/* Handle log messages / exceptions from the remote on a
|
||||
separate thread. */
|
||||
std::thread stderrThread([&]()
|
||||
{
|
||||
try {
|
||||
processStderr(nullptr, nullptr, false);
|
||||
} catch (...) {
|
||||
ex = std::current_exception();
|
||||
}
|
||||
});
|
||||
|
||||
Finally joinStderrThread([&]()
|
||||
{
|
||||
if (stderrThread.joinable()) {
|
||||
stderrThread.join();
|
||||
if (ex) {
|
||||
try {
|
||||
std::rethrow_exception(ex);
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
{
|
||||
FramedSink sink((*this)->to, ex);
|
||||
fun(sink);
|
||||
sink.flush();
|
||||
}
|
||||
|
||||
stderrThread.join();
|
||||
if (ex)
|
||||
std::rethrow_exception(ex);
|
||||
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regStore;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -63,13 +63,21 @@ public:
|
|||
void querySubstitutablePathInfos(const StorePathCAMap & paths,
|
||||
SubstitutablePathInfos & infos) override;
|
||||
|
||||
/* Add a content-addressable store path. `dump` will be drained. */
|
||||
ref<const ValidPathInfo> addCAToStore(
|
||||
Source & dump,
|
||||
const string & name,
|
||||
ContentAddressMethod caMethod,
|
||||
const StorePathSet & references,
|
||||
RepairFlag repair);
|
||||
|
||||
/* Add a content-addressable store path. Does not support references. `dump` will be drained. */
|
||||
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
|
||||
|
||||
void addToStore(const ValidPathInfo & info, Source & nar,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs) override;
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
|
||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
|
||||
|
||||
StorePath addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair) override;
|
||||
|
||||
|
|
@ -106,8 +114,6 @@ public:
|
|||
|
||||
void flushBadConnections();
|
||||
|
||||
protected:
|
||||
|
||||
struct Connection
|
||||
{
|
||||
AutoCloseFD fd;
|
||||
|
|
@ -123,6 +129,8 @@ protected:
|
|||
|
||||
ref<Connection> openConnectionWrapper();
|
||||
|
||||
protected:
|
||||
|
||||
virtual ref<Connection> openConnection() = 0;
|
||||
|
||||
void initConnection(Connection & conn);
|
||||
|
|
@ -139,6 +147,8 @@ protected:
|
|||
|
||||
virtual void narFromPath(const StorePath & path, Sink & sink) override;
|
||||
|
||||
ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path);
|
||||
|
||||
private:
|
||||
|
||||
std::atomic_bool failed{false};
|
||||
|
|
|
|||
|
|
@ -80,7 +80,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
|
|||
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
|
||||
conn->to = FdSink(conn->sshConn->in.get());
|
||||
conn->from = FdSource(conn->sshConn->out.get());
|
||||
initConnection(*conn);
|
||||
return conn;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,9 +8,7 @@
|
|||
#include "json.hh"
|
||||
#include "url.hh"
|
||||
#include "archive.hh"
|
||||
|
||||
#include <future>
|
||||
|
||||
#include "callback.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -1040,38 +1038,26 @@ static bool isNonUriPath(const std::string & spec) {
|
|||
&& spec.find("/") != std::string::npos;
|
||||
}
|
||||
|
||||
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
|
||||
{
|
||||
if (uri == "daemon") {
|
||||
return tDaemon;
|
||||
} else if (uri == "local" || isNonUriPath(uri)) {
|
||||
return tLocal;
|
||||
} else if (uri == "" || uri == "auto") {
|
||||
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
|
||||
return tLocal;
|
||||
else if (pathExists(settings.nixDaemonSocketFile))
|
||||
return tDaemon;
|
||||
else
|
||||
return tLocal;
|
||||
} else {
|
||||
return tOther;
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
|
||||
{
|
||||
switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) {
|
||||
case tDaemon:
|
||||
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
|
||||
case tLocal: {
|
||||
Store::Params params2 = params;
|
||||
if (isNonUriPath(uri)) {
|
||||
params2["root"] = absPath(uri);
|
||||
}
|
||||
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
|
||||
}
|
||||
default:
|
||||
return nullptr;
|
||||
if (uri == "" || uri == "auto") {
|
||||
auto stateDir = get(params, "state").value_or(settings.nixStateDir);
|
||||
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
|
||||
return std::make_shared<LocalStore>(params);
|
||||
else if (pathExists(settings.nixDaemonSocketFile))
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
else
|
||||
return std::make_shared<LocalStore>(params);
|
||||
} else if (uri == "daemon") {
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
} else if (uri == "local") {
|
||||
return std::make_shared<LocalStore>(params);
|
||||
} else if (isNonUriPath(uri)) {
|
||||
Store::Params params2 = params;
|
||||
params2["root"] = absPath(uri);
|
||||
return std::make_shared<LocalStore>(params2);
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -449,7 +449,8 @@ public:
|
|||
/* Like addToStore(), but the contents of the path are contained
|
||||
in `dump', which is either a NAR serialisation (if recursive ==
|
||||
true) or simply the contents of a regular file (if recursive ==
|
||||
false). */
|
||||
false).
|
||||
`dump` may be drained */
|
||||
// FIXME: remove?
|
||||
virtual StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
|
||||
|
|
@ -478,8 +479,38 @@ public:
|
|||
BuildMode buildMode = bmNormal);
|
||||
|
||||
/* Build a single non-materialized derivation (i.e. not from an
|
||||
on-disk .drv file). Note that ‘drvPath’ is only used for
|
||||
informational purposes. */
|
||||
on-disk .drv file).
|
||||
|
||||
‘drvPath’ is used to deduplicate worker goals so it is imperative that
|
||||
is correct. That said, it doesn't literally need to be store path that
|
||||
would be calculated from writing this derivation to the store: it is OK
|
||||
if it instead is that of a Derivation which would resolve to this (by
|
||||
taking the outputs of it's input derivations and adding them as input
|
||||
sources) such that the build time referenceable-paths are the same.
|
||||
|
||||
In the input-addressed case, we usually *do* use an "original"
|
||||
unresolved derivations's path, as that is what will be used in the
|
||||
`buildPaths` case. Also, the input-addressed output paths are verified
|
||||
only by that contents of that specific unresolved derivation, so it is
|
||||
nice to keep that information around so if the original derivation is
|
||||
ever obtained later, it can be verified whether the trusted user in fact
|
||||
used the proper output path.
|
||||
|
||||
In the content-addressed case, we want to always use the
|
||||
resolved drv path calculated from the provided derivation. This serves
|
||||
two purposes:
|
||||
|
||||
- It keeps the operation trustless, by ruling out a maliciously
|
||||
invalid drv path corresponding to a non-resolution-equivalent
|
||||
derivation.
|
||||
|
||||
- For the floating case in particular, it ensures that the derivation
|
||||
to output mapping respects the resolution equivalence relation, so
|
||||
one cannot choose different resolution-equivalent derivations to
|
||||
subvert dependency coherence (i.e. the property that one doesn't end
|
||||
up with multiple different versions of dependencies without
|
||||
explicitly choosing to allow it).
|
||||
*/
|
||||
virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
BuildMode buildMode = bmNormal) = 0;
|
||||
|
||||
|
|
@ -516,7 +547,7 @@ public:
|
|||
- The collector isn't running, or it's just started but hasn't
|
||||
acquired the GC lock yet. In that case we get and release
|
||||
the lock right away, then exit. The collector scans the
|
||||
permanent root and sees our's.
|
||||
permanent root and sees ours.
|
||||
|
||||
In either case the permanent root is seen by the collector. */
|
||||
virtual void syncWithGC() { };
|
||||
|
|
@ -791,16 +822,6 @@ ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
|
|||
const Store::Params & extraParams = Store::Params());
|
||||
|
||||
|
||||
enum StoreType {
|
||||
tDaemon,
|
||||
tLocal,
|
||||
tOther
|
||||
};
|
||||
|
||||
|
||||
StoreType getStoreType(const std::string & uri = settings.storeUri.get(),
|
||||
const std::string & stateDir = settings.nixStateDir);
|
||||
|
||||
/* Return the default substituter stores, defined by the
|
||||
‘substituters’ option and various legacy options. */
|
||||
std::list<ref<Store>> getDefaultSubstituters();
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ namespace nix {
|
|||
#define WORKER_MAGIC_1 0x6e697863
|
||||
#define WORKER_MAGIC_2 0x6478696f
|
||||
|
||||
#define PROTOCOL_VERSION 0x118
|
||||
#define PROTOCOL_VERSION 0x119
|
||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||
|
||||
|
|
@ -18,7 +18,7 @@ typedef enum {
|
|||
wopQueryReferences = 5, // obsolete
|
||||
wopQueryReferrers = 6,
|
||||
wopAddToStore = 7,
|
||||
wopAddTextToStore = 8,
|
||||
wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore
|
||||
wopBuildPaths = 9,
|
||||
wopEnsurePath = 10,
|
||||
wopAddTempRoot = 11,
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ struct ArchiveSettings : Config
|
|||
#endif
|
||||
"use-case-hack",
|
||||
"Whether to enable a Darwin-specific hack for dealing with file name collisions."};
|
||||
Setting<bool> preallocateContents{this, true, "preallocate-contents",
|
||||
"Whether to preallocate files when writing objects with known size."};
|
||||
};
|
||||
|
||||
static ArchiveSettings archiveSettings;
|
||||
|
|
@ -66,9 +68,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
|
|||
{
|
||||
checkInterrupt();
|
||||
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st))
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
auto st = lstat(path);
|
||||
|
||||
sink << "(";
|
||||
|
||||
|
|
@ -325,6 +325,9 @@ struct RestoreSink : ParseSink
|
|||
|
||||
void preallocateContents(uint64_t len)
|
||||
{
|
||||
if (!archiveSettings.preallocateContents)
|
||||
return;
|
||||
|
||||
#if HAVE_POSIX_FALLOCATE
|
||||
if (len) {
|
||||
errno = posix_fallocate(fd.get(), 0, len);
|
||||
|
|
|
|||
|
|
@ -192,7 +192,7 @@ public:
|
|||
{
|
||||
expectArgs({
|
||||
.label = label,
|
||||
.optional = true,
|
||||
.optional = optional,
|
||||
.handler = {dest}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
46
src/libutil/callback.hh
Normal file
46
src/libutil/callback.hh
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
#pragma once
|
||||
|
||||
#include <future>
|
||||
#include <functional>
|
||||
|
||||
namespace nix {
|
||||
|
||||
/* A callback is a wrapper around a lambda that accepts a valid of
|
||||
type T or an exception. (We abuse std::future<T> to pass the value or
|
||||
exception.) */
|
||||
template<typename T>
|
||||
class Callback
|
||||
{
|
||||
std::function<void(std::future<T>)> fun;
|
||||
std::atomic_flag done = ATOMIC_FLAG_INIT;
|
||||
|
||||
public:
|
||||
|
||||
Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
|
||||
|
||||
Callback(Callback && callback) : fun(std::move(callback.fun))
|
||||
{
|
||||
auto prev = callback.done.test_and_set();
|
||||
if (prev) done.test_and_set();
|
||||
}
|
||||
|
||||
void operator()(T && t) noexcept
|
||||
{
|
||||
auto prev = done.test_and_set();
|
||||
assert(!prev);
|
||||
std::promise<T> promise;
|
||||
promise.set_value(std::move(t));
|
||||
fun(promise.get_future());
|
||||
}
|
||||
|
||||
void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
|
||||
{
|
||||
auto prev = done.test_and_set();
|
||||
assert(!prev);
|
||||
std::promise<T> promise;
|
||||
promise.set_exception(exc);
|
||||
fun(promise.get_future());
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#include "config.hh"
|
||||
#include "args.hh"
|
||||
#include "abstractsettingtojson.hh"
|
||||
#include "abstract-setting-to-json.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
#pragma once
|
||||
|
||||
#include <boost/format.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
#include <string>
|
||||
#include "ansicolor.hh"
|
||||
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ void Source::operator () (unsigned char * data, size_t len)
|
|||
}
|
||||
|
||||
|
||||
std::string Source::drain()
|
||||
void Source::drainInto(Sink & sink)
|
||||
{
|
||||
std::string s;
|
||||
std::vector<unsigned char> buf(8192);
|
||||
|
|
@ -101,12 +101,19 @@ std::string Source::drain()
|
|||
size_t n;
|
||||
try {
|
||||
n = read(buf.data(), buf.size());
|
||||
s.append((char *) buf.data(), n);
|
||||
sink(buf.data(), n);
|
||||
} catch (EndOfFile &) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
std::string Source::drain()
|
||||
{
|
||||
StringSink s;
|
||||
drainInto(s);
|
||||
return *s.s;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -69,6 +69,8 @@ struct Source
|
|||
|
||||
virtual bool good() { return true; }
|
||||
|
||||
void drainInto(Sink & sink);
|
||||
|
||||
std::string drain();
|
||||
};
|
||||
|
||||
|
|
@ -340,7 +342,7 @@ T readNum(Source & source)
|
|||
((uint64_t) buf[6] << 48) |
|
||||
((uint64_t) buf[7] << 56);
|
||||
|
||||
if (n > std::numeric_limits<T>::max())
|
||||
if (n > (uint64_t)std::numeric_limits<T>::max())
|
||||
throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
|
||||
|
||||
return (T) n;
|
||||
|
|
@ -404,4 +406,93 @@ struct StreamToSourceAdapter : Source
|
|||
};
|
||||
|
||||
|
||||
/* A source that reads a distinct format of concatenated chunks back into its
|
||||
logical form, in order to guarantee a known state to the original stream,
|
||||
even in the event of errors.
|
||||
|
||||
Use with FramedSink, which also allows the logical stream to be terminated
|
||||
in the event of an exception.
|
||||
*/
|
||||
struct FramedSource : Source
|
||||
{
|
||||
Source & from;
|
||||
bool eof = false;
|
||||
std::vector<unsigned char> pending;
|
||||
size_t pos = 0;
|
||||
|
||||
FramedSource(Source & from) : from(from)
|
||||
{ }
|
||||
|
||||
~FramedSource()
|
||||
{
|
||||
if (!eof) {
|
||||
while (true) {
|
||||
auto n = readInt(from);
|
||||
if (!n) break;
|
||||
std::vector<unsigned char> data(n);
|
||||
from(data.data(), n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t read(unsigned char * data, size_t len) override
|
||||
{
|
||||
if (eof) throw EndOfFile("reached end of FramedSource");
|
||||
|
||||
if (pos >= pending.size()) {
|
||||
size_t len = readInt(from);
|
||||
if (!len) {
|
||||
eof = true;
|
||||
return 0;
|
||||
}
|
||||
pending = std::vector<unsigned char>(len);
|
||||
pos = 0;
|
||||
from(pending.data(), len);
|
||||
}
|
||||
|
||||
auto n = std::min(len, pending.size() - pos);
|
||||
memcpy(data, pending.data() + pos, n);
|
||||
pos += n;
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
/* Write as chunks in the format expected by FramedSource.
|
||||
|
||||
The exception_ptr reference can be used to terminate the stream when you
|
||||
detect that an error has occurred on the remote end.
|
||||
*/
|
||||
struct FramedSink : nix::BufferedSink
|
||||
{
|
||||
BufferedSink & to;
|
||||
std::exception_ptr & ex;
|
||||
|
||||
FramedSink(BufferedSink & to, std::exception_ptr & ex) : to(to), ex(ex)
|
||||
{ }
|
||||
|
||||
~FramedSink()
|
||||
{
|
||||
try {
|
||||
to << 0;
|
||||
to.flush();
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
|
||||
void write(const unsigned char * data, size_t len) override
|
||||
{
|
||||
/* Don't send more data if the remote has
|
||||
encountered an error. */
|
||||
if (ex) {
|
||||
auto ex2 = ex;
|
||||
ex = nullptr;
|
||||
std::rethrow_exception(ex2);
|
||||
}
|
||||
to << len;
|
||||
to(data, len);
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ namespace nix {
|
|||
|
||||
// If `separator` is found, we return the portion of the string before the
|
||||
// separator, and modify the string argument to contain only the part after the
|
||||
// separator. Otherwise, wer return `std::nullopt`, and we leave the argument
|
||||
// separator. Otherwise, we return `std::nullopt`, and we leave the argument
|
||||
// string alone.
|
||||
static inline std::optional<std::string_view> splitPrefixTo(std::string_view & string, char separator) {
|
||||
auto sepInstance = string.find(separator);
|
||||
|
|
|
|||
44
src/libutil/url-parts.hh
Normal file
44
src/libutil/url-parts.hh
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <regex>
|
||||
|
||||
namespace nix {
|
||||
|
||||
// URI stuff.
|
||||
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
|
||||
const static std::string schemeRegex = "(?:[a-z+.-]+)";
|
||||
const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
|
||||
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
|
||||
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
|
||||
const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
|
||||
const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
|
||||
const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
|
||||
const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
|
||||
const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
|
||||
const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
|
||||
const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
|
||||
const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
|
||||
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
|
||||
|
||||
// A Git ref (i.e. branch or tag name).
|
||||
const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
|
||||
extern std::regex refRegex;
|
||||
|
||||
// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
|
||||
// This is because of the definition of a ref in refs.c in https://github.com/git/git
|
||||
// See tests/fetchGitRefs.sh for the full definition
|
||||
const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
|
||||
extern std::regex badGitRefRegex;
|
||||
|
||||
// A Git revision (a SHA-1 commit hash).
|
||||
const static std::string revRegexS = "[0-9a-fA-F]{40}";
|
||||
extern std::regex revRegex;
|
||||
|
||||
// A ref or revision, or a ref followed by a revision.
|
||||
const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
|
||||
|
||||
const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
|
||||
extern std::regex flakeIdRegex;
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
#include "url.hh"
|
||||
#include "url-parts.hh"
|
||||
#include "util.hh"
|
||||
|
||||
namespace nix {
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@
|
|||
|
||||
#include "error.hh"
|
||||
|
||||
#include <regex>
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct ParsedURL
|
||||
|
|
@ -29,40 +27,4 @@ std::map<std::string, std::string> decodeQuery(const std::string & query);
|
|||
|
||||
ParsedURL parseURL(const std::string & url);
|
||||
|
||||
// URI stuff.
|
||||
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
|
||||
const static std::string schemeRegex = "(?:[a-z+.-]+)";
|
||||
const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
|
||||
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
|
||||
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
|
||||
const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
|
||||
const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
|
||||
const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
|
||||
const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
|
||||
const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
|
||||
const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
|
||||
const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
|
||||
const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
|
||||
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
|
||||
|
||||
// A Git ref (i.e. branch or tag name).
|
||||
const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
|
||||
extern std::regex refRegex;
|
||||
|
||||
// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
|
||||
// This is because of the definition of a ref in refs.c in https://github.com/git/git
|
||||
// See tests/fetchGitRefs.sh for the full definition
|
||||
const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
|
||||
extern std::regex badGitRefRegex;
|
||||
|
||||
// A Git revision (a SHA-1 commit hash).
|
||||
const static std::string revRegexS = "[0-9a-fA-F]{40}";
|
||||
extern std::regex revRegex;
|
||||
|
||||
// A ref or revision, or a ref followed by a revision.
|
||||
const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
|
||||
|
||||
const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
|
||||
extern std::regex flakeIdRegex;
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,13 +12,9 @@
|
|||
#include <signal.h>
|
||||
|
||||
#include <functional>
|
||||
#include <limits>
|
||||
#include <cstdio>
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
#include <optional>
|
||||
#include <future>
|
||||
#include <iterator>
|
||||
|
||||
#ifndef HAVE_STRUCT_DIRENT_D_TYPE
|
||||
#define DT_UNKNOWN 0
|
||||
|
|
@ -480,43 +476,8 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_
|
|||
}
|
||||
|
||||
|
||||
/* A callback is a wrapper around a lambda that accepts a valid of
|
||||
type T or an exception. (We abuse std::future<T> to pass the value or
|
||||
exception.) */
|
||||
template<typename T>
|
||||
class Callback
|
||||
{
|
||||
std::function<void(std::future<T>)> fun;
|
||||
std::atomic_flag done = ATOMIC_FLAG_INIT;
|
||||
|
||||
public:
|
||||
|
||||
Callback(std::function<void(std::future<T>)> fun) : fun(fun) { }
|
||||
|
||||
Callback(Callback && callback) : fun(std::move(callback.fun))
|
||||
{
|
||||
auto prev = callback.done.test_and_set();
|
||||
if (prev) done.test_and_set();
|
||||
}
|
||||
|
||||
void operator()(T && t) noexcept
|
||||
{
|
||||
auto prev = done.test_and_set();
|
||||
assert(!prev);
|
||||
std::promise<T> promise;
|
||||
promise.set_value(std::move(t));
|
||||
fun(promise.get_future());
|
||||
}
|
||||
|
||||
void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept
|
||||
{
|
||||
auto prev = done.test_and_set();
|
||||
assert(!prev);
|
||||
std::promise<T> promise;
|
||||
promise.set_exception(exc);
|
||||
fun(promise.get_future());
|
||||
}
|
||||
};
|
||||
class Callback;
|
||||
|
||||
|
||||
/* Start a thread that handles various signals. Also block those signals
|
||||
|
|
|
|||
|
|
@ -76,6 +76,13 @@ static void update(const StringSet & channelNames)
|
|||
|
||||
auto store = openStore();
|
||||
|
||||
auto [fd, unpackChannelPath] = createTempFile();
|
||||
writeFull(fd.get(),
|
||||
#include "unpack-channel.nix.gen.hh"
|
||||
);
|
||||
fd = -1;
|
||||
AutoDelete del(unpackChannelPath, false);
|
||||
|
||||
// Download each channel.
|
||||
Strings exprs;
|
||||
for (const auto & channel : channels) {
|
||||
|
|
@ -104,7 +111,7 @@ static void update(const StringSet & channelNames)
|
|||
|
||||
bool unpacked = false;
|
||||
if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) {
|
||||
runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> "
|
||||
runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath +
|
||||
"{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" });
|
||||
unpacked = true;
|
||||
}
|
||||
|
|
@ -125,7 +132,7 @@ static void update(const StringSet & channelNames)
|
|||
// Unpack the channel tarballs into the Nix store and install them
|
||||
// into the channels profile.
|
||||
std::cerr << "unpacking channels...\n";
|
||||
Strings envArgs{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" };
|
||||
Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--from-expression" };
|
||||
for (auto & expr : exprs)
|
||||
envArgs.push_back(std::move(expr));
|
||||
envArgs.push_back("--quiet");
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#include "shared.hh"
|
||||
#include "local-store.hh"
|
||||
#include "remote-store.hh"
|
||||
#include "util.hh"
|
||||
#include "serialise.hh"
|
||||
#include "archive.hh"
|
||||
|
|
@ -285,44 +286,28 @@ static int _main(int argc, char * * argv)
|
|||
initPlugins();
|
||||
|
||||
if (stdio) {
|
||||
if (getStoreType() == tDaemon) {
|
||||
// Forward on this connection to the real daemon
|
||||
auto socketPath = settings.nixDaemonSocketFile;
|
||||
auto s = socket(PF_UNIX, SOCK_STREAM, 0);
|
||||
if (s == -1)
|
||||
throw SysError("creating Unix domain socket");
|
||||
if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) {
|
||||
auto conn = store->openConnectionWrapper();
|
||||
int from = conn->from.fd;
|
||||
int to = conn->to.fd;
|
||||
|
||||
auto socketDir = dirOf(socketPath);
|
||||
if (chdir(socketDir.c_str()) == -1)
|
||||
throw SysError("changing to socket directory '%1%'", socketDir);
|
||||
|
||||
auto socketName = std::string(baseNameOf(socketPath));
|
||||
auto addr = sockaddr_un{};
|
||||
addr.sun_family = AF_UNIX;
|
||||
if (socketName.size() + 1 >= sizeof(addr.sun_path))
|
||||
throw Error("socket name %1% is too long", socketName);
|
||||
strcpy(addr.sun_path, socketName.c_str());
|
||||
|
||||
if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1)
|
||||
throw SysError("cannot connect to daemon at %1%", socketPath);
|
||||
|
||||
auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1;
|
||||
auto nfds = std::max(from, STDIN_FILENO) + 1;
|
||||
while (true) {
|
||||
fd_set fds;
|
||||
FD_ZERO(&fds);
|
||||
FD_SET(s, &fds);
|
||||
FD_SET(from, &fds);
|
||||
FD_SET(STDIN_FILENO, &fds);
|
||||
if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
|
||||
throw SysError("waiting for data from client or server");
|
||||
if (FD_ISSET(s, &fds)) {
|
||||
auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
|
||||
if (FD_ISSET(from, &fds)) {
|
||||
auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
|
||||
if (res == -1)
|
||||
throw SysError("splicing data from daemon socket to stdout");
|
||||
else if (res == 0)
|
||||
throw EndOfFile("unexpected EOF from daemon socket");
|
||||
}
|
||||
if (FD_ISSET(STDIN_FILENO, &fds)) {
|
||||
auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
|
||||
auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
|
||||
if (res == -1)
|
||||
throw SysError("splicing data from stdin to daemon socket");
|
||||
else if (res == 0)
|
||||
|
|
|
|||
|
|
@ -230,7 +230,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
|
|||
{
|
||||
DrvNames selectors = drvNamesFromArgs(args);
|
||||
if (selectors.empty())
|
||||
selectors.push_back(DrvName("*"));
|
||||
selectors.emplace_back("*");
|
||||
|
||||
DrvInfos elems;
|
||||
set<unsigned int> done;
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ static int _main(int argc, char * * argv)
|
|||
bool fromExpr = false;
|
||||
string attrPath;
|
||||
bool unpack = false;
|
||||
bool executable = false;
|
||||
string name;
|
||||
|
||||
struct MyArgs : LegacyArgs, MixEvalArgs
|
||||
|
|
@ -81,6 +82,8 @@ static int _main(int argc, char * * argv)
|
|||
}
|
||||
else if (*arg == "--unpack")
|
||||
unpack = true;
|
||||
else if (*arg == "--executable")
|
||||
executable = true;
|
||||
else if (*arg == "--name")
|
||||
name = getArg(*arg, arg, end);
|
||||
else if (*arg != "" && arg->at(0) == '-')
|
||||
|
|
@ -175,7 +178,11 @@ static int _main(int argc, char * * argv)
|
|||
|
||||
/* Download the file. */
|
||||
{
|
||||
AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
|
||||
auto mode = 0600;
|
||||
if (executable)
|
||||
mode = 0700;
|
||||
|
||||
AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode);
|
||||
if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
|
||||
|
||||
FdSink sink(fd.get());
|
||||
|
|
@ -201,7 +208,7 @@ static int _main(int argc, char * * argv)
|
|||
tmpFile = unpacked;
|
||||
}
|
||||
|
||||
const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
|
||||
const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
|
||||
|
||||
auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
|
||||
storePath = info.path;
|
||||
|
|
|
|||
|
|
@ -98,14 +98,14 @@ struct CmdBundle : InstallableCommand
|
|||
if (!evalState->isDerivation(*vRes))
|
||||
throw Error("the bundler '%s' does not produce a derivation", bundler.what());
|
||||
|
||||
auto attr1 = vRes->attrs->find(evalState->sDrvPath);
|
||||
auto attr1 = vRes->attrs->get(evalState->sDrvPath);
|
||||
if (!attr1)
|
||||
throw Error("the bundler '%s' does not produce a derivation", bundler.what());
|
||||
|
||||
PathSet context2;
|
||||
StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2));
|
||||
|
||||
auto attr2 = vRes->attrs->find(evalState->sOutPath);
|
||||
auto attr2 = vRes->attrs->get(evalState->sOutPath);
|
||||
if (!attr2)
|
||||
throw Error("the bundler '%s' does not produce a derivation", bundler.what());
|
||||
|
||||
|
|
|
|||
|
|
@ -49,9 +49,7 @@ struct CmdDoctor : StoreCommand
|
|||
{
|
||||
logger->log("Running checks against store uri: " + store->getUri());
|
||||
|
||||
auto type = getStoreType();
|
||||
|
||||
if (type < tOther) {
|
||||
if (store.dynamic_pointer_cast<LocalFSStore>()) {
|
||||
success &= checkNixInPath();
|
||||
success &= checkProfileRoots(store);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ struct CmdHash : Command
|
|||
switch (mode) {
|
||||
case FileIngestionMethod::Flat:
|
||||
d = "print cryptographic hash of a regular file";
|
||||
break;
|
||||
case FileIngestionMethod::Recursive:
|
||||
d = "print cryptographic hash of the NAR serialisation of a path";
|
||||
};
|
||||
|
|
|
|||
|
|
@ -29,3 +29,5 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
|
|||
src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh
|
||||
|
||||
src/nix/develop.cc: src/nix/get-env.sh.gen.hh
|
||||
|
||||
src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh
|
||||
|
|
|
|||
|
|
@ -111,11 +111,7 @@ std::set<std::string> runResolver(const Path & filename)
|
|||
|
||||
bool isSymlink(const Path & path)
|
||||
{
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st) == -1)
|
||||
throw SysError("getting attributes of path '%1%'", path);
|
||||
|
||||
return S_ISLNK(st.st_mode);
|
||||
return S_ISLNK(lstat(path).st_mode);
|
||||
}
|
||||
|
||||
Path resolveSymlink(const Path & path)
|
||||
|
|
|
|||
|
|
@ -15,3 +15,6 @@ out1=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 1)
|
|||
out2=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 2)
|
||||
|
||||
test $out1 == $out2
|
||||
|
||||
nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5
|
||||
nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ nix_tests = \
|
|||
linux-sandbox.sh \
|
||||
build-dry.sh \
|
||||
build-remote-input-addressed.sh \
|
||||
ssh-relay.sh \
|
||||
nar-access.sh \
|
||||
structured-attrs.sh \
|
||||
fetchGit.sh \
|
||||
|
|
|
|||
|
|
@ -9,9 +9,8 @@ rm -f $TEST_ROOT/result
|
|||
|
||||
export unreachable=$(nix add-to-store ./recursive.sh)
|
||||
|
||||
nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
|
||||
NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
|
||||
with import ./config.nix;
|
||||
with import <nix/config.nix>;
|
||||
mkDerivation {
|
||||
name = "recursive";
|
||||
dummy = builtins.toFile "dummy" "bla bla";
|
||||
|
|
@ -24,9 +23,10 @@ nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/resu
|
|||
|
||||
buildCommand = '\'\''
|
||||
mkdir $out
|
||||
PATH=${nixBinDir}:$PATH
|
||||
opts="--experimental-features nix-command"
|
||||
|
||||
PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH
|
||||
|
||||
# Check that we can query/build paths in our input closure.
|
||||
nix $opts path-info $dummy
|
||||
nix $opts build $dummy
|
||||
|
|
|
|||
|
|
@ -13,14 +13,14 @@ hash=$(nix-hash $path2)
|
|||
chmod u+w $path2
|
||||
touch $path2/bad
|
||||
|
||||
if nix-store --verify --check-contents -v; then
|
||||
echo "nix-store --verify succeeded unexpectedly" >&2
|
||||
exit 1
|
||||
fi
|
||||
(! nix-store --verify --check-contents -v)
|
||||
|
||||
# The path can be repaired by rebuilding the derivation.
|
||||
nix-store --verify --check-contents --repair
|
||||
|
||||
(! [ -e $path2/bad ])
|
||||
(! [ -w $path2 ])
|
||||
|
||||
nix-store --verify-path $path2
|
||||
|
||||
# Re-corrupt and delete the deriver. Now --verify --repair should
|
||||
|
|
@ -30,10 +30,7 @@ touch $path2/bad
|
|||
|
||||
nix-store --delete $(nix-store -qd $path2)
|
||||
|
||||
if nix-store --verify --check-contents --repair; then
|
||||
echo "nix-store --verify --repair succeeded unexpectedly" >&2
|
||||
exit 1
|
||||
fi
|
||||
(! nix-store --verify --check-contents --repair)
|
||||
|
||||
nix-build dependencies.nix -o $TEST_ROOT/result --repair
|
||||
|
||||
|
|
|
|||
|
|
@ -10,13 +10,15 @@ outPath=$(nix-store -rvv "$drvPath")
|
|||
|
||||
echo "output path is $outPath"
|
||||
|
||||
(! [ -w $outPath ])
|
||||
|
||||
text=$(cat "$outPath"/hello)
|
||||
if test "$text" != "Hello World!"; then exit 1; fi
|
||||
|
||||
# Directed delete: $outPath is not reachable from a root, so it should
|
||||
# be deleteable.
|
||||
nix-store --delete $outPath
|
||||
if test -e $outPath/hello; then false; fi
|
||||
(! [ -e $outPath/hello ])
|
||||
|
||||
outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)"
|
||||
if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then
|
||||
|
|
|
|||
16
tests/ssh-relay.sh
Normal file
16
tests/ssh-relay.sh
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
source common.sh
|
||||
|
||||
echo foo > $TEST_ROOT/hello.sh
|
||||
|
||||
ssh_localhost=ssh://localhost
|
||||
remote_store=?remote-store=$ssh_localhost
|
||||
|
||||
store=$ssh_localhost
|
||||
|
||||
store+=$remote_store
|
||||
store+=$remote_store
|
||||
store+=$remote_store
|
||||
|
||||
out=$(nix add-to-store --store "$store" $TEST_ROOT/hello.sh)
|
||||
|
||||
[ foo = $(< $out) ]
|
||||
Loading…
Reference in a new issue