mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-10 08:16:15 +02:00
Merge remote-tracking branch 'upstream/master' into drv-outputs-map-allow-missing
This commit is contained in:
John Ericson
commit
fbeb8695fb
12 changed files with 170 additions and 125 deletions
|
|
@ -153,6 +153,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
|
|||
|
||||
auto [fdTemp, fnTemp] = createTempFile();
|
||||
|
||||
AutoDelete autoDelete(fnTemp);
|
||||
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
|
||||
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
|
||||
|
|
@ -167,6 +169,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
|
|||
TeeSource teeSource(narSource, *compressionSink);
|
||||
narAccessor = makeNarAccessor(teeSource);
|
||||
compressionSink->finish();
|
||||
fileSink.flush();
|
||||
}
|
||||
|
||||
auto now2 = std::chrono::steady_clock::now();
|
||||
|
|
@ -280,7 +283,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
|
|||
if (repair || !fileExists(narInfo->url)) {
|
||||
stats.narWrite++;
|
||||
upsertFile(narInfo->url,
|
||||
std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
|
||||
std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
|
||||
"application/x-nix-nar");
|
||||
} else
|
||||
stats.narWriteAverted++;
|
||||
|
|
|
|||
|
|
@ -1042,20 +1042,6 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
Path srcPath(absPath(_srcPath));
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
dumpPath(srcPath, sink, filter);
|
||||
else
|
||||
readFile(srcPath, sink);
|
||||
});
|
||||
return addToStoreFromDump(*source, name, method, hashAlgo, repair);
|
||||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
|
||||
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -145,16 +145,8 @@ public:
|
|||
void addToStore(const ValidPathInfo & info, Source & source,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs) override;
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo,
|
||||
PathFilter & filter, RepairFlag repair) override;
|
||||
|
||||
/* Like addToStore(), but the contents of the path are contained
|
||||
in `dump', which is either a NAR serialisation (if recursive ==
|
||||
true) or simply the contents of a regular file (if recursive ==
|
||||
false). */
|
||||
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
|
||||
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
|
||||
|
||||
StorePath addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair) override;
|
||||
|
|
|
|||
111
src/libstore/path-info.hh
Normal file
111
src/libstore/path-info.hh
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
#pragma once
|
||||
|
||||
#include "path.hh"
|
||||
#include "hash.hh"
|
||||
#include "content-address.hh"
|
||||
|
||||
#include <string>
|
||||
#include <optional>
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
||||
class Store;
|
||||
|
||||
|
||||
struct SubstitutablePathInfo
|
||||
{
|
||||
std::optional<StorePath> deriver;
|
||||
StorePathSet references;
|
||||
uint64_t downloadSize; /* 0 = unknown or inapplicable */
|
||||
uint64_t narSize; /* 0 = unknown */
|
||||
};
|
||||
|
||||
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
|
||||
|
||||
|
||||
struct ValidPathInfo
|
||||
{
|
||||
StorePath path;
|
||||
std::optional<StorePath> deriver;
|
||||
// TODO document this
|
||||
std::optional<Hash> narHash;
|
||||
StorePathSet references;
|
||||
time_t registrationTime = 0;
|
||||
uint64_t narSize = 0; // 0 = unknown
|
||||
uint64_t id; // internal use only
|
||||
|
||||
/* Whether the path is ultimately trusted, that is, it's a
|
||||
derivation output that was built locally. */
|
||||
bool ultimate = false;
|
||||
|
||||
StringSet sigs; // note: not necessarily verified
|
||||
|
||||
/* If non-empty, an assertion that the path is content-addressed,
|
||||
i.e., that the store path is computed from a cryptographic hash
|
||||
of the contents of the path, plus some other bits of data like
|
||||
the "name" part of the path. Such a path doesn't need
|
||||
signatures, since we don't have to trust anybody's claim that
|
||||
the path is the output of a particular derivation. (In the
|
||||
extensional store model, we have to trust that the *contents*
|
||||
of an output path of a derivation were actually produced by
|
||||
that derivation. In the intensional model, we have to trust
|
||||
that a particular output path was produced by a derivation; the
|
||||
path then implies the contents.)
|
||||
|
||||
Ideally, the content-addressability assertion would just be a Boolean,
|
||||
and the store path would be computed from the name component, ‘narHash’
|
||||
and ‘references’. However, we support many types of content addresses.
|
||||
*/
|
||||
std::optional<ContentAddress> ca;
|
||||
|
||||
bool operator == (const ValidPathInfo & i) const
|
||||
{
|
||||
return
|
||||
path == i.path
|
||||
&& narHash == i.narHash
|
||||
&& references == i.references;
|
||||
}
|
||||
|
||||
/* Return a fingerprint of the store path to be used in binary
|
||||
cache signatures. It contains the store path, the base-32
|
||||
SHA-256 hash of the NAR serialisation of the path, the size of
|
||||
the NAR, and the sorted references. The size field is strictly
|
||||
speaking superfluous, but might prevent endless/excessive data
|
||||
attacks. */
|
||||
std::string fingerprint(const Store & store) const;
|
||||
|
||||
void sign(const Store & store, const SecretKey & secretKey);
|
||||
|
||||
/* Return true iff the path is verifiably content-addressed. */
|
||||
bool isContentAddressed(const Store & store) const;
|
||||
|
||||
/* Functions to view references + hasSelfReference as one set, mainly for
|
||||
compatibility's sake. */
|
||||
StorePathSet referencesPossiblyToSelf() const;
|
||||
void insertReferencePossiblyToSelf(StorePath && ref);
|
||||
void setReferencesPossiblyToSelf(StorePathSet && refs);
|
||||
|
||||
static const size_t maxSigs = std::numeric_limits<size_t>::max();
|
||||
|
||||
/* Return the number of signatures on this .narinfo that were
|
||||
produced by one of the specified keys, or maxSigs if the path
|
||||
is content-addressed. */
|
||||
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
|
||||
|
||||
/* Verify a single signature. */
|
||||
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
|
||||
|
||||
Strings shortRefs() const;
|
||||
|
||||
ValidPathInfo(const ValidPathInfo & other) = default;
|
||||
|
||||
ValidPathInfo(StorePath && path) : path(std::move(path)) { };
|
||||
ValidPathInfo(const StorePath & path) : path(path) { };
|
||||
|
||||
virtual ~ValidPathInfo() { }
|
||||
};
|
||||
|
||||
typedef list<ValidPathInfo> ValidPathInfos;
|
||||
|
||||
}
|
||||
|
|
@ -266,6 +266,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
|||
const std::string & mimeType,
|
||||
const std::string & contentEncoding)
|
||||
{
|
||||
istream->seekg(0, istream->end);
|
||||
auto size = istream->tellg();
|
||||
istream->seekg(0, istream->beg);
|
||||
|
||||
auto maxThreads = std::thread::hardware_concurrency();
|
||||
|
||||
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
|
||||
|
|
@ -343,10 +347,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
|
|||
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
|
||||
.count();
|
||||
|
||||
printInfo("uploaded 's3://%s/%s' in %d ms",
|
||||
bucketName, path, duration);
|
||||
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
|
||||
bucketName, path, size, duration);
|
||||
|
||||
stats.putTimeMs += duration;
|
||||
stats.putBytes += std::max(size, (decltype(size)) 0);
|
||||
stats.put++;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ public:
|
|||
struct Stats
|
||||
{
|
||||
std::atomic<uint64_t> put{0};
|
||||
std::atomic<uint64_t> putBytes{0};
|
||||
std::atomic<uint64_t> putTimeMs{0};
|
||||
std::atomic<uint64_t> get{0};
|
||||
std::atomic<uint64_t> getBytes{0};
|
||||
|
|
|
|||
|
|
@ -239,6 +239,20 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
|
|||
}
|
||||
|
||||
|
||||
StorePath Store::addToStore(const string & name, const Path & _srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
Path srcPath(absPath(_srcPath));
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
dumpPath(srcPath, sink, filter);
|
||||
else
|
||||
readFile(srcPath, sink);
|
||||
});
|
||||
return addToStoreFromDump(*source, name, method, hashAlgo, repair);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
The aim of this function is to compute in one pass the correct ValidPathInfo for
|
||||
the files that we are trying to add to the store. To accomplish that in one
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include "globals.hh"
|
||||
#include "config.hh"
|
||||
#include "derivations.hh"
|
||||
#include "path-info.hh"
|
||||
|
||||
#include <atomic>
|
||||
#include <limits>
|
||||
|
|
@ -101,95 +102,6 @@ struct GCResults
|
|||
};
|
||||
|
||||
|
||||
struct SubstitutablePathInfo
|
||||
{
|
||||
std::optional<StorePath> deriver;
|
||||
StorePathSet references;
|
||||
uint64_t downloadSize; /* 0 = unknown or inapplicable */
|
||||
uint64_t narSize; /* 0 = unknown */
|
||||
};
|
||||
|
||||
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
|
||||
|
||||
struct ValidPathInfo
|
||||
{
|
||||
StorePath path;
|
||||
std::optional<StorePath> deriver;
|
||||
// TODO document this
|
||||
std::optional<Hash> narHash;
|
||||
StorePathSet references;
|
||||
time_t registrationTime = 0;
|
||||
uint64_t narSize = 0; // 0 = unknown
|
||||
uint64_t id; // internal use only
|
||||
|
||||
/* Whether the path is ultimately trusted, that is, it's a
|
||||
derivation output that was built locally. */
|
||||
bool ultimate = false;
|
||||
|
||||
StringSet sigs; // note: not necessarily verified
|
||||
|
||||
/* If non-empty, an assertion that the path is content-addressed,
|
||||
i.e., that the store path is computed from a cryptographic hash
|
||||
of the contents of the path, plus some other bits of data like
|
||||
the "name" part of the path. Such a path doesn't need
|
||||
signatures, since we don't have to trust anybody's claim that
|
||||
the path is the output of a particular derivation. (In the
|
||||
extensional store model, we have to trust that the *contents*
|
||||
of an output path of a derivation were actually produced by
|
||||
that derivation. In the intensional model, we have to trust
|
||||
that a particular output path was produced by a derivation; the
|
||||
path then implies the contents.)
|
||||
|
||||
Ideally, the content-addressability assertion would just be a Boolean,
|
||||
and the store path would be computed from the name component, ‘narHash’
|
||||
and ‘references’. However, we support many types of content addresses.
|
||||
*/
|
||||
std::optional<ContentAddress> ca;
|
||||
|
||||
bool operator == (const ValidPathInfo & i) const
|
||||
{
|
||||
return
|
||||
path == i.path
|
||||
&& narHash == i.narHash
|
||||
&& references == i.references;
|
||||
}
|
||||
|
||||
/* Return a fingerprint of the store path to be used in binary
|
||||
cache signatures. It contains the store path, the base-32
|
||||
SHA-256 hash of the NAR serialisation of the path, the size of
|
||||
the NAR, and the sorted references. The size field is strictly
|
||||
speaking superfluous, but might prevent endless/excessive data
|
||||
attacks. */
|
||||
std::string fingerprint(const Store & store) const;
|
||||
|
||||
void sign(const Store & store, const SecretKey & secretKey);
|
||||
|
||||
/* Return true iff the path is verifiably content-addressed. */
|
||||
bool isContentAddressed(const Store & store) const;
|
||||
|
||||
static const size_t maxSigs = std::numeric_limits<size_t>::max();
|
||||
|
||||
/* Return the number of signatures on this .narinfo that were
|
||||
produced by one of the specified keys, or maxSigs if the path
|
||||
is content-addressed. */
|
||||
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
|
||||
|
||||
/* Verify a single signature. */
|
||||
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
|
||||
|
||||
Strings shortRefs() const;
|
||||
|
||||
ValidPathInfo(const ValidPathInfo & other) = default;
|
||||
|
||||
ValidPathInfo(StorePath && path) : path(std::move(path)) { };
|
||||
ValidPathInfo(const StorePath & path) : path(path) { };
|
||||
|
||||
virtual ~ValidPathInfo() { }
|
||||
};
|
||||
|
||||
typedef list<ValidPathInfo> ValidPathInfos;
|
||||
|
||||
|
||||
enum BuildMode { bmNormal, bmRepair, bmCheck };
|
||||
|
||||
|
||||
|
|
@ -462,7 +374,7 @@ public:
|
|||
libutil/archive.hh). */
|
||||
virtual StorePath addToStore(const string & name, const Path & srcPath,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
|
||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
|
||||
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair);
|
||||
|
||||
/* Copy the contents of a path to the store and register the
|
||||
validity the resulting path, using a constant amount of
|
||||
|
|
@ -471,6 +383,10 @@ public:
|
|||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
|
||||
std::optional<Hash> expectedCAHash = {});
|
||||
|
||||
/* Like addToStore(), but the contents of the path are contained
|
||||
in `dump', which is either a NAR serialisation (if recursive ==
|
||||
true) or simply the contents of a regular file (if recursive ==
|
||||
false). */
|
||||
// FIXME: remove?
|
||||
virtual StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
|
||||
|
|
|
|||
|
|
@ -494,6 +494,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
|
|||
{
|
||||
Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
|
||||
// Strictly speaking, this is UB, but who cares...
|
||||
// FIXME: use O_TMPFILE.
|
||||
AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
|
||||
if (!fd)
|
||||
throw SysError("creating temporary file '%s'", tmpl);
|
||||
|
|
|
|||
|
|
@ -437,7 +437,7 @@ ref<eval_cache::EvalCache> openEvalCache(
|
|||
std::shared_ptr<flake::LockedFlake> lockedFlake,
|
||||
bool useEvalCache)
|
||||
{
|
||||
auto fingerprint = lockedFlake->getFingerprint();
|
||||
auto fingerprint = lockedFlake->getFingerprint();
|
||||
return make_ref<nix::eval_cache::EvalCache>(
|
||||
useEvalCache && evalSettings.pureEval
|
||||
? std::optional { std::cref(fingerprint) }
|
||||
|
|
|
|||
|
|
@ -218,7 +218,9 @@ outPath=$(nix-build --no-out-link -E '
|
|||
|
||||
nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
|
||||
|
||||
[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]]
|
||||
diff -u \
|
||||
<(jq -S < $cacheDir/$(basename $outPath).ls) \
|
||||
<(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S)
|
||||
|
||||
|
||||
# Test debug info index generation.
|
||||
|
|
@ -234,4 +236,6 @@ outPath=$(nix-build --no-out-link -E '
|
|||
|
||||
nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
|
||||
|
||||
[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]]
|
||||
diff -u \
|
||||
<(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \
|
||||
<(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S)
|
||||
|
|
|
|||
|
|
@ -26,12 +26,24 @@ nix cat-store $storePath/foo/baz > baz.cat-nar
|
|||
diff -u baz.cat-nar $storePath/foo/baz
|
||||
|
||||
# Test --json.
|
||||
[[ $(nix ls-nar --json $narFile /) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]]
|
||||
[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' ]]
|
||||
[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0,"narOffset":368}' ]]
|
||||
[[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]]
|
||||
[[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]]
|
||||
[[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]]
|
||||
diff -u \
|
||||
<(nix ls-nar --json $narFile / | jq -S) \
|
||||
<(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
|
||||
diff -u \
|
||||
<(nix ls-nar --json -R $narFile /foo | jq -S) \
|
||||
<(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S)
|
||||
diff -u \
|
||||
<(nix ls-nar --json -R $narFile /foo/bar | jq -S) \
|
||||
<(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S)
|
||||
diff -u \
|
||||
<(nix ls-store --json $storePath | jq -S) \
|
||||
<(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S)
|
||||
diff -u \
|
||||
<(nix ls-store --json -R $storePath/foo | jq -S) \
|
||||
<(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S)
|
||||
diff -u \
|
||||
<(nix ls-store --json -R $storePath/foo/bar| jq -S) \
|
||||
<(echo '{"type":"regular","size":0}' | jq -S)
|
||||
|
||||
# Test missing files.
|
||||
nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR'
|
||||
|
|
|
|||
Loading…
Reference in a new issue