max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2024-11-26 07:46:21 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/master' into substitute-other-storedir

Browse source
This commit is contained in:
Matthew Bauer 2020-06-19 14:22:06 -04:00
commit 2796b01e86
25 changed files with 327 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
doc/manual/advanced-topics/diff-hook.xml
View file

@ -70,7 +70,7 @@ path just built.</para>
<screen>
$ nix-build ./deterministic.nix -A stable
these derivations will be built:
this derivation will be built:
/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
@ -85,7 +85,7 @@ checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
<screen>
$ nix-build ./deterministic.nix -A unstable
these derivations will be built:
this derivation will be built:
/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
@ -193,7 +193,7 @@ repeat = 1
An example output of this configuration:
<screen>
$ nix-build ./test.nix -A unstable
these derivations will be built:
this derivation will be built:
/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...

2
doc/manual/advanced-topics/post-build-hook.xml
View file

@ -122,7 +122,7 @@ post-build-hook = /etc/nix/upload-to-cache.sh
<screen>
$ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
these derivations will be built:
this derivation will be built:
/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...

2
doc/manual/command-ref/nix-env.xml
View file

@ -516,7 +516,7 @@ source:
$ nix-env -f '&lt;nixpkgs>' -iA hello --dry-run
(dry run; not doing anything)
installing ‘hello-2.10’
these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
<replaceable>...</replaceable></screen>

2
doc/manual/installation/env-variables.xml
View file

@ -39,7 +39,7 @@ bundle.</para>
<step><para>Set the environment variable and install Nix</para>
<screen>
$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
$ sh &lt;(curl https://nixos.org/nix/install)
$ sh &lt;(curl -L https://nixos.org/nix/install)
</screen></step>
<step><para>In the shell profile and rc files (for example,

4
doc/manual/installation/installing-binary.xml
View file

@ -12,7 +12,7 @@
</para>
<screen>
$ sh &lt;(curl https://nixos.org/nix/install)
$ sh &lt;(curl -L https://nixos.org/nix/install)
</screen>
<para>
@ -39,7 +39,7 @@
To explicitly select a single-user installation on your system:
<screen>
sh &lt;(curl https://nixos.org/nix/install) --no-daemon
sh &lt;(curl -L https://nixos.org/nix/install) --no-daemon
</screen>
</para>

2
doc/manual/introduction/quick-start.xml
View file

@ -15,7 +15,7 @@ to subsequent chapters.</para>
<step><para>Install single-user Nix by running the following:
<screen>
$ bash &lt;(curl https://nixos.org/nix/install)
$ bash &lt;(curl -L https://nixos.org/nix/install)
</screen>
This will install Nix in <filename>/nix</filename>. The install script

13
src/libexpr/attr-path.cc
View file

@ -6,11 +6,11 @@
namespace nix {
static Strings parseAttrPath(const string & s)
static Strings parseAttrPath(std::string_view s)
{
Strings res;
string cur;
string::const_iterator i = s.begin();
auto i = s.begin();
while (i != s.end()) {
if (*i == '.') {
res.push_back(cur);
@ -32,6 +32,15 @@ static Strings parseAttrPath(const string & s)
}
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
{
std::vector<Symbol> res;
for (auto & a : parseAttrPath(s))
res.push_back(state.symbols.create(a));
return res;
}
std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Value & vIn)
{

2
src/libexpr/attr-path.hh
View file

@ -16,4 +16,6 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
/* Heuristic to find the filename and lineno or a nix value. */
Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
}

39
src/libexpr/eval.cc
View file

@ -161,12 +161,12 @@ const Value *getPrimOp(const Value &v) {
}
string showType(const Value & v)
string showType(ValueType type)
{
switch (v.type) {
switch (type) {
case tInt: return "an integer";
case tBool: return "a boolean";
case tString: return v.string.context ? "a string with context" : "a string";
case tBool: return "a Boolean";
case tString: return "a string";
case tPath: return "a path";
case tNull: return "null";
case tAttrs: return "a set";
@ -175,14 +175,27 @@ string showType(const Value & v)
case tApp: return "a function application";
case tLambda: return "a function";
case tBlackhole: return "a black hole";
case tPrimOp: return "a built-in function";
case tPrimOpApp: return "a partially applied built-in function";
case tExternal: return "an external value";
case tFloat: return "a float";
}
abort();
}
string showType(const Value & v)
{
switch (v.type) {
case tString: return v.string.context ? "a string with context" : "a string";
case tPrimOp:
return fmt("the built-in function '%s'", string(v.primOp->name));
case tPrimOpApp:
return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
case tExternal: return v.external->showType();
case tFloat: return "a float";
default:
return showType(v.type);
}
abort();
}
@ -323,6 +336,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
, sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))
, sRecurseForDerivations(symbols.create("recurseForDerivations"))
, repair(NoRepair)
, store(store)
, baseEnv(allocEnv(128))
@ -471,14 +485,21 @@ Value * EvalState::addConstant(const string & name, Value & v)
Value * EvalState::addPrimOp(const string & name,
size_t arity, PrimOpFun primOp)
{
auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
Symbol sym = symbols.create(name2);
/* Hack to make constants lazy: turn them into a application of
the primop to a dummy value. */
if (arity == 0) {
auto vPrimOp = allocValue();
vPrimOp->type = tPrimOp;
vPrimOp->primOp = new PrimOp(primOp, 1, sym);
Value v;
primOp(*this, noPos, nullptr, v);
mkApp(v, *vPrimOp, *vPrimOp);
return addConstant(name, v);
}
Value * v = allocValue();
string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
Symbol sym = symbols.create(name2);
v->type = tPrimOp;
v->primOp = new PrimOp(primOp, arity, sym);
staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;

6
src/libexpr/eval.hh
View file

@ -18,7 +18,7 @@ namespace nix {
class Store;
class EvalState;
struct StorePath;
class StorePath;
enum RepairFlag : bool;
@ -74,7 +74,8 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sOutputHash, sOutputHashAlgo, sOutputHashMode;
sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations;
Symbol sDerivationNix;
/* If set, force copying files to the Nix store even if they
@ -324,6 +325,7 @@ private:
/* Return a string representing the type of the value `v'. */
string showType(ValueType type);
string showType(const Value & v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path,

2
src/libexpr/get-drvs.cc
View file

@ -348,7 +348,7 @@ static void getDerivations(EvalState & state, Value & vIn,
should we recurse into it? => Only if it has a
`recurseForDerivations = true' attribute. */
if (i->value->type == tAttrs) {
Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
}

42
src/libexpr/primops.cc
View file

@ -50,20 +50,20 @@ void EvalState::realiseContext(const PathSet & context)
std::vector<StorePathWithOutputs> drvs;
for (auto & i : context) {
std::pair<string, string> decoded = decodeContext(i);
auto ctx = store->parseStorePath(decoded.first);
auto [ctxS, outputName] = decodeContext(i);
auto ctx = store->parseStorePath(ctxS);
if (!store->isValidPath(ctx))
throw InvalidPathError(store->printStorePath(ctx));
if (!decoded.second.empty() && ctx.isDerivation()) {
drvs.push_back(StorePathWithOutputs{ctx, {decoded.second}});
if (!outputName.empty() && ctx.isDerivation()) {
drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
/* Add the output of this derivation to the allowed
paths. */
if (allowedPaths) {
auto drv = store->derivationFromPath(store->parseStorePath(decoded.first));
DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
auto drv = store->derivationFromPath(ctx);
DerivationOutputs::iterator i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
allowedPaths->insert(store->printStorePath(i->second.path));
}
}
@ -79,6 +79,7 @@ void EvalState::realiseContext(const PathSet & context)
StorePathSet willBuild, willSubstitute, unknown;
unsigned long long downloadSize, narSize;
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs);
}
@ -775,10 +776,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput {
std::move(outPath),
(ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "")
+ printHashType(h.type),
h.to_string(Base16, false),
.path = std::move(outPath),
.hash = DerivationOutputHash {
.method = ingestionMethod,
.hash = std::move(h),
},
});
}
@ -792,7 +794,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
for (auto & i : outputs) {
if (!jsonObject) drv.env[i] = "";
drv.outputs.insert_or_assign(i,
DerivationOutput { StorePath::dummy, "", "" });
DerivationOutput {
.path = StorePath::dummy,
.hash = std::optional<DerivationOutputHash> {},
});
}
Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
@ -801,7 +806,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeOutputPath(i, h, drvName);
if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput { std::move(outPath), "", "" });
DerivationOutput {
.path = std::move(outPath),
.hash = std::optional<DerivationOutputHash>(),
});
}
}
@ -2201,10 +2209,11 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun,
std::optional<std::string> requiredFeature)
{
if (!primOps) primOps = new PrimOps;
primOps->emplace_back(name, arity, fun);
primOps->push_back({name, arity, fun, requiredFeature});
}
@ -2396,7 +2405,8 @@ void EvalState::createBaseEnv()
if (RegisterPrimOp::primOps)
for (auto & primOp : *RegisterPrimOp::primOps)
addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp));
if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature))
addPrimOp(primOp.name, primOp.arity, primOp.primOp);
/* Now that we've added all primops, sort the `builtins' set,
because attribute lookups expect it to be sorted. */

17
src/libexpr/primops.hh
View file

@ -7,12 +7,25 @@ namespace nix {
struct RegisterPrimOp
{
typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
struct Info
{
std::string name;
size_t arity;
PrimOpFun primOp;
std::optional<std::string> requiredFeature;
};
typedef std::vector<Info> PrimOps;
static PrimOps * primOps;
/* You can register a constant by passing an arity of 0. fun
will get called during EvalState initialization, so there
may be primops not yet added and builtins is not yet sorted. */
RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
RegisterPrimOp(
std::string name,
size_t arity,
PrimOpFun fun,
std::optional<std::string> requiredFeature = {});
};
/* These primops are disabled without enableNativeCode, but plugins

20
src/libmain/shared.cc
View file

@ -48,7 +48,10 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
{
if (!willBuild.empty()) {
printMsg(lvl, "these derivations will be built:");
if (willBuild.size() == 1)
printMsg(lvl, fmt("this derivation will be built:"));
else
printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size()));
auto sorted = store->topoSortPaths(willBuild);
reverse(sorted.begin(), sorted.end());
for (auto & i : sorted)
@ -56,9 +59,18 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
}
if (!willSubstitute.empty()) {
printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
downloadSize / (1024.0 * 1024.0),
narSize / (1024.0 * 1024.0)));
const float downloadSizeMiB = downloadSize / (1024.f * 1024.f);
const float narSizeMiB = narSize / (1024.f * 1024.f);
if (willSubstitute.size() == 1) {
printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):",
downloadSizeMiB,
narSizeMiB));
} else {
printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
willSubstitute.size(),
downloadSizeMiB,
narSizeMiB));
}
for (auto & i : willSubstitute)
printMsg(lvl, fmt(" %s", store->printStorePath(i)));
}

2
src/libstore/binary-cache-store.cc
View file

@ -388,8 +388,6 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
narInfo->sigs.insert(sigs.begin(), sigs.end());
auto narInfoFile = narInfoFileFor(narInfo->path);
writeNarInfo(narInfo);
}

27
src/libstore/build.cc
View file

@ -86,7 +86,7 @@ struct HookInstance;
/* A pointer to a goal. */
class Goal;
struct Goal;
class DerivationGoal;
typedef std::shared_ptr<Goal> GoalPtr;
typedef std::weak_ptr<Goal> WeakGoalPtr;
@ -1196,8 +1196,8 @@ void DerivationGoal::haveDerivation()
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (parsedDrv->contentAddressed()) {
settings.requireExperimentalFeature("content-addressed-paths");
throw Error("content-addressed-paths isn't implemented yet");
settings.requireExperimentalFeature("ca-derivations");
throw Error("ca-derivations isn't implemented yet");
}
@ -3718,10 +3718,7 @@ void DerivationGoal::registerOutputs()
if (fixedOutput) {
FileIngestionMethod outputHashMode; Hash h;
i.second.parseHashInfo(outputHashMode, h);
if (outputHashMode == FileIngestionMethod::Flat) {
if (i.second.hash->method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
@ -3732,20 +3729,22 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = outputHashMode == FileIngestionMethod::Recursive
? hashPath(h.type, actualPath).first
: hashFile(h.type, actualPath);
Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
? hashPath(i.second.hash->hash.type, actualPath).first
: hashFile(i.second.hash->hash.type, actualPath);
auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name());
auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
if (h != h2) {
if (i.second.hash->hash != h2) {
/* Throw an error after registering the path as
valid. */
worker.hashMismatch = true;
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true)));
worker.store.printStorePath(dest),
i.second.hash->hash.to_string(SRI, true),
h2.to_string(SRI, true)));
Path actualDest = worker.store.Store::toRealPath(dest);
@ -3765,7 +3764,7 @@ void DerivationGoal::registerOutputs()
else
assert(worker.store.parseStorePath(path) == dest);
ca = makeFixedOutputCA(outputHashMode, h2);
ca = makeFixedOutputCA(i.second.hash->method, h2);
}
/* Get rid of all weird permissions. This also checks that

112
src/libstore/derivations.cc
View file

@ -8,22 +8,8 @@
namespace nix {
void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
{
recursive = FileIngestionMethod::Flat;
string algo = hashAlgo;
if (string(algo, 0, 2) == "r:") {
recursive = FileIngestionMethod::Recursive;
algo = string(algo, 2);
}
HashType hashType = parseHashType(algo);
if (hashType == htUnknown)
throw Error("unknown hash algorithm '%s'", algo);
hash = Hash(this->hash, hashType);
std::string DerivationOutputHash::printMethodAlgo() const {
return makeFileIngestionPrefix(method) + printHashType(hash.type);
}
@ -120,6 +106,36 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
std::optional<DerivationOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
if (hashType == htUnknown)
throw Error("unknown hash hashAlgorithm '%s'", hashAlgo);
fsh = DerivationOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
}
static Derivation parseDerivation(const Store & store, const string & s)
{
Derivation drv;
@ -129,15 +145,8 @@ static Derivation parseDerivation(const Store & store, const string & s)
/* Parse the list of outputs. */
while (!endOfList(str)) {
expect(str, "("); std::string id = parseString(str);
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); auto hash = parseString(str);
expect(str, ")");
drv.outputs.emplace(id, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
auto output = parseDerivationOutput(store, str);
drv.outputs.emplace(std::move(id), std::move(output));
}
/* Parse the list of input derivations. */
@ -263,8 +272,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
s += ','; printUnquotedString(s, i.second.hashAlgo);
s += ','; printUnquotedString(s, i.second.hash);
s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
s += ','; printUnquotedString(s,
i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
s += ')';
}
@ -320,7 +330,7 @@ bool BasicDerivation::isFixedOutput() const
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
outputs.begin()->second.hash != "";
outputs.begin()->second.hash;
}
@ -353,8 +363,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
return hashString(htSHA256, "fixed:out:"
+ i->second.hashAlgo + ":"
+ i->second.hash + ":"
+ i->second.hash->printMethodAlgo() + ":"
+ i->second.hash->hash.to_string(Base16, false) + ":"
+ store.printStorePath(i->second.path));
}
@ -397,6 +407,33 @@ StorePathSet BasicDerivation::outputPaths() const
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
const auto hash = readString(in);
std::optional<DerivationOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
if (hashType == htUnknown)
throw Error("unknown hash hashAlgorithm '%s'", hashAlgo);
fsh = DerivationOutputHash {
.method = std::move(method),
.hash = Hash(hash, hashType),
};
}
return DerivationOutput {
.path = std::move(path),
.hash = std::move(fsh),
};
}
StringSet BasicDerivation::outputNames() const
{
@ -413,14 +450,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto name = readString(in);
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
auto hash = readString(in);
drv.outputs.emplace(name, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
auto output = readDerivationOutput(in, store);
drv.outputs.emplace(std::move(name), std::move(output));
}
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
@ -442,7 +473,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
{
out << drv.outputs.size();
for (auto & i : drv.outputs)
out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
out << i.first
<< store.printStorePath(i.second.path)
<< i.second.hash->printMethodAlgo()
<< i.second.hash->hash.to_string(Base16, false);
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();

10
src/libstore/derivations.hh
View file

@ -12,11 +12,17 @@ namespace nix {
/* Abstract syntax of derivations. */
/// Pair of a hash, and how the file system was ingested
struct DerivationOutputHash {
FileIngestionMethod method;
Hash hash;
std::string printMethodAlgo() const;
};
struct DerivationOutput
{
StorePath path;
std::string hashAlgo; /* hash used for expected hash computation */
std::string hash; /* expected hash, may be null */
std::optional<DerivationOutputHash> hash; /* hash used for expected hash computation */
void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
};

6
src/libstore/filetransfer.cc
View file

@ -72,6 +72,7 @@ struct curlFileTransfer : public FileTransfer
curl_off_t writtenToSink = 0;
inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
/* Get the HTTP status code, or 0 for other protocols. */
long getHTTPStatus()
{
@ -98,7 +99,7 @@ struct curlFileTransfer : public FileTransfer
/* Only write data to the sink if this is a
successful response. */
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
if (successfulStatuses.count(httpStatus)) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
}
@ -352,8 +353,7 @@ struct curlFileTransfer : public FileTransfer
if (writeException)
failEx(writeException);
else if (code == CURLE_OK &&
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */))
else if (code == CURLE_OK && successfulStatuses.count(httpStatus))
{
result.cached = httpStatus == 304;
act.progress(result.bodySize, result.bodySize);

10
src/libstore/local-store.cc
View file

@ -561,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
FileIngestionMethod method; Hash h;
out->second.parseHashInfo(method, h);
check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
check(
makeFixedOutputPath(
out->second.hash->method,
out->second.hash->hash,
drvName),
out->second.path, "out");
}
else {

21
src/libstore/store-api.cc
View file

@ -173,20 +173,20 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath(
FileIngestionMethod recursive,
FileIngestionMethod method,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{
if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
} else {
assert(references.empty());
return makeStorePath("output:out",
hashString(htSHA256,
"fixed:out:"
+ (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+ makeFileIngestionPrefix(method)
+ hash.to_string(Base16, true) + ":"),
name);
}
@ -864,10 +864,21 @@ Strings ValidPathInfo::shortRefs() const
}
std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
switch (m) {
case FileIngestionMethod::Flat:
return "";
case FileIngestionMethod::Recursive:
return "r:";
default:
throw Error("impossible, caught both cases");
}
}
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
{
return "fixed:"
+ (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
+ makeFileIngestionPrefix(method)
+ hash.to_string(Base32, true);
}

8
src/libstore/store-api.hh
View file

@ -189,9 +189,10 @@ struct ValidPathInfo
Strings shortRefs() const;
ValidPathInfo(const StorePath & path) : path(path) { }
ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path) : path(std::move(path)) { }
ValidPathInfo(StorePath && path) : path(std::move(path)) { };
ValidPathInfo(const StorePath & path) : path(path) { };
virtual ~ValidPathInfo() { }
};
@ -842,6 +843,9 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
std::istream & str,
bool hashGiven = false);
/* Compute the prefix to the hash algorithm which indicates how the files were
ingested. */
std::string makeFileIngestionPrefix(const FileIngestionMethod m);
/* Compute the content-addressability assertion (ValidPathInfo::ca)
for paths created by makeFixedOutputPath() / addToStore(). */

78
src/libutil/tests/compression.cc Normal file
View file

@ -0,0 +1,78 @@
#include "compression.hh"
#include <gtest/gtest.h>
namespace nix {
/* ----------------------------------------------------------------------------
* compress / decompress
* --------------------------------------------------------------------------*/
TEST(compress, compressWithUnknownMethod) {
ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod);
}
TEST(compress, noneMethodDoesNothingToTheInput) {
ref<std::string> o = compress("none", "this-is-a-test");
ASSERT_EQ(*o, "this-is-a-test");
}
TEST(decompress, decompressXzCompressed) {
auto method = "xz";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, *compress(method, str));
ASSERT_EQ(*o, str);
}
TEST(decompress, decompressBzip2Compressed) {
auto method = "bzip2";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, *compress(method, str));
ASSERT_EQ(*o, str);
}
TEST(decompress, decompressBrCompressed) {
auto method = "br";
auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
ref<std::string> o = decompress(method, *compress(method, str));
ASSERT_EQ(*o, str);
}
TEST(decompress, decompressInvalidInputThrowsCompressionError) {
auto method = "bzip2";
auto str = "this is a string that does not qualify as valid bzip2 data";
ASSERT_THROW(decompress(method, str), CompressionError);
}
/* ----------------------------------------------------------------------------
* compression sinks
* --------------------------------------------------------------------------*/
TEST(makeCompressionSink, noneSinkDoesNothingToInput) {
StringSink strSink;
auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
auto sink = makeCompressionSink("none", strSink);
(*sink)(inputString);
sink->finish();
ASSERT_STREQ((*strSink.s).c_str(), inputString);
}
TEST(makeCompressionSink, compressAndDecompress) {
StringSink strSink;
auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
auto decompressionSink = makeDecompressionSink("bzip2", strSink);
auto sink = makeCompressionSink("bzip2", *decompressionSink);
(*sink)(inputString);
sink->finish();
decompressionSink->finish();
ASSERT_STREQ((*strSink.s).c_str(), inputString);
}
}

8
src/nix/develop.cc
View file

@ -135,7 +135,13 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = hashDerivationModulo(*store, drv, true);
auto shellOutPath = store->makeOutputPath("out", h, drvName);
drv.outputs.insert_or_assign("out", DerivationOutput { shellOutPath, "", "" });
drv.outputs.insert_or_assign("out", DerivationOutput {
.path = shellOutPath,
.hash = DerivationOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash { },
},
});
drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(store, drv, drvName);

6
src/nix/show-derivation.cc
View file

@ -70,9 +70,9 @@ struct CmdShowDerivation : InstallablesCommand
for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first));
outputObj.attr("path", store->printStorePath(output.second.path));
if (output.second.hash != "") {
outputObj.attr("hashAlgo", output.second.hashAlgo);
outputObj.attr("hash", output.second.hash);
if (output.second.hash) {
outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo());
outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false));
}
}
}