Factor out commonality between nix-prefetch-url and nix-store --add-fixed

This commit is contained in:
Eelco Dolstra 2020-07-10 13:21:37 +02:00
parent 7f1a86d57c
commit 5dff49f661
6 changed files with 64 additions and 42 deletions

View file

@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
return ca ? renderContentAddress(*ca) : ""; return ca ? renderContentAddress(*ca) : "";
} }
Hash getContentAddressHash(const ContentAddress & ca)
{
return std::visit(overloaded {
[](TextHash th) {
return th.hash;
},
[](FixedOutputHash fsh) {
return fsh.hash;
}
}, ca);
}
} }

View file

@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
Hash getContentAddressHash(const ContentAddress & ca);
} }

View file

@ -7,6 +7,7 @@
#include "json.hh" #include "json.hh"
#include "derivations.hh" #include "derivations.hh"
#include "url.hh" #include "url.hh"
#include "archive.hh"
#include <future> #include <future>
@ -221,6 +222,37 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
} }
ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
std::optional<Hash> expectedCAHash)
{
/* FIXME: inefficient: we're reading/hashing 'tmpFile' three
times. */
auto hash = method == FileIngestionMethod::Recursive
? hashPath(hashAlgo, srcPath).first
: hashFile(hashAlgo, srcPath);
if (expectedCAHash && expectedCAHash != hash)
throw Error("hash mismatch for '%s'", srcPath);
auto [narHash, narSize] = hashPath(htSHA256, srcPath);
ValidPathInfo info(makeFixedOutputPath(method, hash, name));
info.narHash = narHash;
info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & sink) {
dumpPath(srcPath, sink);
});
addToStore(info, *source);
}
return info;
}
Store::Store(const Params & params) Store::Store(const Params & params)
: Config(params) : Config(params)
, state({(size_t) pathInfoCacheSize}) , state({(size_t) pathInfoCacheSize})

View file

@ -450,6 +450,13 @@ public:
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path, using a constant amount of
memory. */
ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
std::optional<Hash> expectedCAHash = {});
// FIXME: remove? // FIXME: remove?
virtual StorePath addToStoreFromDump(const string & dump, const string & name, virtual StorePath addToStoreFromDump(const string & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)

View file

@ -10,7 +10,6 @@
#include "../nix/legacy.hh" #include "../nix/legacy.hh"
#include "progress-bar.hh" #include "progress-bar.hh"
#include "tarfile.hh" #include "tarfile.hh"
#include "archive.hh"
#include <iostream> #include <iostream>
@ -154,14 +153,15 @@ static int _main(int argc, char * * argv)
/* If an expected hash is given, the file may already exist in /* If an expected hash is given, the file may already exist in
the store. */ the store. */
Hash hash, expectedHash(ht); std::optional<Hash> expectedHash;
Hash hash;
std::optional<StorePath> storePath; std::optional<StorePath> storePath;
if (args.size() == 2) { if (args.size() == 2) {
expectedHash = Hash(args[1], ht); expectedHash = Hash(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, expectedHash, name); storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
if (store->isValidPath(*storePath)) if (store->isValidPath(*storePath))
hash = expectedHash; hash = *expectedHash;
else else
storePath.reset(); storePath.reset();
} }
@ -201,28 +201,12 @@ static int _main(int argc, char * * argv)
tmpFile = unpacked; tmpFile = unpacked;
} }
/* FIXME: inefficient: we're reading/hashing 'tmpFile' const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
three times. */
auto [narHash, narSize] = hashPath(htSHA256, tmpFile);
hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
storePath = info.path;
if (expectedHash != Hash(ht) && expectedHash != hash) assert(info.ca);
throw Error("hash mismatch for '%1%'", uri); hash = getContentAddressHash(*info.ca);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
storePath = store->makeFixedOutputPath(recursive, hash, name);
/* Copy the file to the Nix store. */
ValidPathInfo info(*storePath);
info.narHash = narHash;
info.narSize = narSize;
info.ca = FixedOutputHash { .method = recursive, .hash = hash };
auto source = sinkToSource([&](Sink & sink) {
dumpPath(tmpFile, sink);
});
store->addToStore(info, *source);
} }
stopProgressBar(); stopProgressBar();

View file

@ -186,23 +186,8 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
HashType hashAlgo = parseHashType(opArgs.front()); HashType hashAlgo = parseHashType(opArgs.front());
opArgs.pop_front(); opArgs.pop_front();
for (auto & i : opArgs) { for (auto & i : opArgs)
auto hash = method == FileIngestionMethod::Recursive std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
? hashPath(hashAlgo, i).first
: hashFile(hashAlgo, i);
auto [narHash, narSize] = hashPath(htSHA256, i);
ValidPathInfo info(store->makeFixedOutputPath(method, hash, baseNameOf(i)));
info.narHash = narHash;
info.narSize = narSize;
info.ca = FixedOutputHash { .method = method, .hash = hash };
auto source = sinkToSource([&](Sink & sink) {
dumpPath(i, sink);
});
store->addToStore(info, *source);
std::cout << fmt("%s\n", store->printStorePath(info.path));
}
} }