Use "raw pattern" for content address types

We weren't because this ancient PR predated it!

This is actually a new version of the pattern which addresses some
issues identified in #7479.
This commit is contained in:
John Ericson 2023-03-30 17:12:49 -04:00
parent a6d00a7bfb
commit c51d554c93
14 changed files with 118 additions and 87 deletions

View file

@ -2508,7 +2508,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
/* Check wanted hash */ /* Check wanted hash */
const Hash & wanted = dof.hash.hash; const Hash & wanted = dof.hash.hash;
assert(newInfo0.ca); assert(newInfo0.ca);
auto got = getContentAddressHash(*newInfo0.ca); auto got = newInfo0.ca->getHash();
if (wanted != got) { if (wanted != got) {
/* Throw an error after registering the path as /* Throw an error after registering the path as
valid. */ valid. */

View file

@ -97,7 +97,7 @@ void PathSubstitutionGoal::tryNext()
if (ca) { if (ca) {
subPath = sub->makeFixedOutputPathFromCA( subPath = sub->makeFixedOutputPathFromCA(
std::string { storePath.name() }, std::string { storePath.name() },
caWithoutRefs(*ca)); ContentAddressWithReferences::withoutRefs(*ca));
if (sub->storeDir == worker.store.storeDir) if (sub->storeDir == worker.store.storeDir)
assert(subPath == storePath); assert(subPath == storePath);
} else if (sub->storeDir != worker.store.storeDir) { } else if (sub->storeDir != worker.store.storeDir) {

View file

@ -9,7 +9,6 @@ std::string FixedOutputHash::printMethodAlgo() const
return makeFileIngestionPrefix(method) + printHashType(hash.type); return makeFileIngestionPrefix(method) + printHashType(hash.type);
} }
std::string makeFileIngestionPrefix(FileIngestionMethod m) std::string makeFileIngestionPrefix(FileIngestionMethod m)
{ {
switch (m) { switch (m) {
@ -22,35 +21,35 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m)
} }
} }
std::string renderContentAddress(ContentAddress ca) std::string ContentAddress::render() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHash & th) { [](const TextHash & th) {
return "text:" return "text:"
+ th.hash.to_string(Base32, true); + th.hash.to_string(Base32, true);
}, },
[](FixedOutputHash & fsh) { [](const FixedOutputHash & fsh) {
return "fixed:" return "fixed:"
+ makeFileIngestionPrefix(fsh.method) + makeFileIngestionPrefix(fsh.method)
+ fsh.hash.to_string(Base32, true); + fsh.hash.to_string(Base32, true);
} }
}, ca); }, raw);
} }
std::string renderContentAddressMethod(ContentAddressMethod cam) std::string ContentAddressMethod::render() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](TextHashMethod & th) { [](const TextHashMethod & th) {
return std::string{"text:"} + printHashType(htSHA256); return std::string{"text:"} + printHashType(htSHA256);
}, },
[](FixedOutputHashMethod & fshm) { [](const FixedOutputHashMethod & fshm) {
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
} }
}, cam); }, raw);
} }
/* /**
Parses content address strings up to the hash. * Parses content address strings up to the hash.
*/ */
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest)
{ {
@ -94,7 +93,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
} }
ContentAddress parseContentAddress(std::string_view rawCa) { ContentAddress ContentAddress::parse(std::string_view rawCa) {
auto rest = rawCa; auto rest = rawCa;
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest);
@ -112,10 +111,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
}); });
}, },
}, caMethod); }, caMethod.raw);
} }
ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod)
{ {
std::string asPrefix = std::string{caMethod} + ":"; std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference // parseContentAddressMethodPrefix takes its argument by reference
@ -123,26 +122,28 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
return parseContentAddressMethodPrefix(asPrefixView); return parseContentAddressMethodPrefix(asPrefixView);
} }
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) std::optional<ContentAddress> ContentAddress::parseOpt(std::string_view rawCaOpt)
{ {
return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt); return rawCaOpt == ""
? std::nullopt
: std::optional { ContentAddress::parse(rawCaOpt) };
}; };
std::string renderContentAddress(std::optional<ContentAddress> ca) std::string renderContentAddress(std::optional<ContentAddress> ca)
{ {
return ca ? renderContentAddress(*ca) : ""; return ca ? ca->render() : "";
} }
Hash getContentAddressHash(const ContentAddress & ca) const Hash & ContentAddress::getHash() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](const TextHash & th) { [](const TextHash & th) -> auto & {
return th.hash; return th.hash;
}, },
[](const FixedOutputHash & fsh) { [](const FixedOutputHash & fsh) -> auto & {
return fsh.hash; return fsh.hash;
}, },
}, ca); }, raw);
} }
bool StoreReferences::empty() const bool StoreReferences::empty() const
@ -155,7 +156,7 @@ size_t StoreReferences::size() const
return (self ? 1 : 0) + others.size(); return (self ? 1 : 0) + others.size();
} }
ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) {
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextHash & h) -> ContentAddressWithReferences { [&](const TextHash & h) -> ContentAddressWithReferences {
return TextInfo { return TextInfo {
@ -169,7 +170,7 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) {
.references = {}, .references = {},
}; };
}, },
}, ca); }, ca.raw);
} }
} }

View file

@ -39,17 +39,16 @@ enum struct FileIngestionMethod : uint8_t {
Recursive = true Recursive = true
}; };
struct FixedOutputHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
};
/** /**
* Compute the prefix to the hash algorithm which indicates how the * Compute the prefix to the hash algorithm which indicates how the
* files were ingested. * files were ingested.
*/ */
std::string makeFileIngestionPrefix(FileIngestionMethod m); std::string makeFileIngestionPrefix(FileIngestionMethod m);
struct FixedOutputHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
};
/** /**
* An enumeration of all the ways we can serialize file system objects. * An enumeration of all the ways we can serialize file system objects.
@ -59,14 +58,25 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m);
* with info on references, and we have `ContentAddressWithReferences`, * with info on references, and we have `ContentAddressWithReferences`,
* as defined further below. * as defined further below.
*/ */
struct ContentAddressMethod
{
typedef std::variant< typedef std::variant<
TextHashMethod, TextHashMethod,
FixedOutputHashMethod FixedOutputHashMethod
> ContentAddressMethod; > Raw;
ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); Raw raw;
/* The moral equivalent of `using Raw::Raw;` */
ContentAddressMethod(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
static ContentAddressMethod parse(std::string_view rawCaMethod);
std::string render() const;
};
std::string renderContentAddressMethod(ContentAddressMethod caMethod);
/* /*
* Mini content address * Mini content address
@ -115,25 +125,41 @@ struct FixedOutputHash {
* - fixed:<r?>:<ht>:<h>: For paths computed by * - fixed:<r?>:<ht>:<h>: For paths computed by
* Store::makeFixedOutputPath() / Store::addToStore(). * Store::makeFixedOutputPath() / Store::addToStore().
*/ */
struct ContentAddress
{
typedef std::variant< typedef std::variant<
TextHash, TextHash,
FixedOutputHash FixedOutputHash
> ContentAddress; > Raw;
Raw raw;
/* The moral equivalent of `using Raw::Raw;` */
ContentAddress(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
/** /**
* Compute the content-addressability assertion (ValidPathInfo::ca) for * Compute the content-addressability assertion (ValidPathInfo::ca) for
* paths created by Store::makeFixedOutputPath() / Store::addToStore(). * paths created by Store::makeFixedOutputPath() / Store::addToStore().
*/ */
std::string renderContentAddress(ContentAddress ca); std::string render() const;
static ContentAddress parse(std::string_view rawCa);
static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
const Hash & getHash() const;
};
std::string renderContentAddress(std::optional<ContentAddress> ca); std::string renderContentAddress(std::optional<ContentAddress> ca);
ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
Hash getContentAddressHash(const ContentAddress & ca);
/*
* Full content address
*
* See the schema for store paths in store-api.cc
*/
/** /**
* A set of references to other store objects. * A set of references to other store objects.
@ -167,12 +193,6 @@ struct StoreReferences {
GENERATE_CMP(StoreReferences, me->self, me->others); GENERATE_CMP(StoreReferences, me->self, me->others);
}; };
/*
* Full content address
*
* See the schema for store paths in store-api.cc
*/
// This matches the additional info that we need for makeTextPath // This matches the additional info that we need for makeTextPath
struct TextInfo { struct TextInfo {
TextHash hash; TextHash hash;
@ -200,15 +220,25 @@ struct FixedOutputInfo {
* *
* A ContentAddress without a Hash. * A ContentAddress without a Hash.
*/ */
struct ContentAddressWithReferences
{
typedef std::variant< typedef std::variant<
TextInfo, TextInfo,
FixedOutputInfo FixedOutputInfo
> ContentAddressWithReferences; > Raw;
Raw raw;
/* The moral equivalent of `using Raw::Raw;` */
ContentAddressWithReferences(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
/** /**
* Create a ContentAddressWithReferences from a mere ContentAddress, by * Create a ContentAddressWithReferences from a mere ContentAddress, by
* assuming no references in all cases. * assuming no references in all cases.
*/ */
ContentAddressWithReferences caWithoutRefs(const ContentAddress &); static ContentAddressWithReferences withoutRefs(const ContentAddress &);
};
} }

View file

@ -401,21 +401,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
auto pathInfo = [&]() { auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork(); // NB: FramedSource must be out of scope before logger->stopWork();
ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr);
FramedSource source(from); FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded { return std::visit(overloaded {
[&](TextHashMethod &) { [&](const TextHashMethod &) {
// We could stream this by changing Store // We could stream this by changing Store
std::string contents = source.drain(); std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair); auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
[&](FixedOutputHashMethod & fohm) { [&](const FixedOutputHashMethod & fohm) {
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
}, contentAddressMethod); }, contentAddressMethod.raw);
}(); }();
logger->stopWork(); logger->stopWork();
@ -880,7 +880,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
from >> info.registrationTime >> info.narSize >> info.ultimate; from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from); info.sigs = readStrings<StringSet>(from);
info.ca = parseContentAddressOpt(readString(from)); info.ca = ContentAddress::parseOpt(readString(from));
from >> repair >> dontCheckSigs; from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs) if (!trusted && dontCheckSigs)
dontCheckSigs = false; dontCheckSigs = false;

View file

@ -156,7 +156,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
throw Error("NAR hash is now mandatory"); throw Error("NAR hash is now mandatory");
info->narHash = Hash::parseAnyPrefixed(s); info->narHash = Hash::parseAnyPrefixed(s);
} }
info->ca = parseContentAddressOpt(readString(conn->from)); info->ca = ContentAddress::parseOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from); info->sigs = readStrings<StringSet>(conn->from);
auto s = readString(conn->from); auto s = readString(conn->from);

View file

@ -944,7 +944,7 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s
if (s) info->sigs = tokenizeString<StringSet>(s, " "); if (s) info->sigs = tokenizeString<StringSet>(s, " ");
s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7);
if (s) info->ca = parseContentAddressOpt(s); if (s) info->ca = ContentAddress::parseOpt(s);
/* Get the references. */ /* Get the references. */
auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
@ -1150,7 +1150,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst
if (path.second) { if (path.second) {
subPath = makeFixedOutputPathFromCA( subPath = makeFixedOutputPathFromCA(
path.first.name(), path.first.name(),
caWithoutRefs(*path.second)); ContentAddressWithReferences::withoutRefs(*path.second));
if (sub->storeDir == storeDir) if (sub->storeDir == storeDir)
assert(subPath == path.first); assert(subPath == path.first);
if (subPath != path.first) if (subPath != path.first)
@ -1329,7 +1329,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
printStorePath(info.path), info.narSize, hashResult.second); printStorePath(info.path), info.narSize, hashResult.second);
if (info.ca) { if (info.ca) {
if (auto foHash = std::get_if<FixedOutputHash>(&*info.ca)) { if (auto foHash = std::get_if<FixedOutputHash>(&info.ca->raw)) {
auto actualFoHash = hashCAPath( auto actualFoHash = hashCAPath(
foHash->method, foHash->method,
foHash->hash.type, foHash->hash.type,
@ -1342,7 +1342,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
actualFoHash.hash.to_string(Base32, true)); actualFoHash.hash.to_string(Base32, true));
} }
} }
if (auto textHash = std::get_if<TextHash>(&*info.ca)) { if (auto textHash = std::get_if<TextHash>(&info.ca->raw)) {
auto actualTextHash = hashString(htSHA256, readFile(realPath)); auto actualTextHash = hashString(htSHA256, readFile(realPath));
if (textHash->hash != actualTextHash) { if (textHash->hash != actualTextHash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",

View file

@ -273,7 +273,7 @@ public:
narInfo->deriver = StorePath(queryNAR.getStr(9)); narInfo->deriver = StorePath(queryNAR.getStr(9));
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig); narInfo->sigs.insert(sig);
narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11));
return {oValid, narInfo}; return {oValid, narInfo};
}); });

View file

@ -74,7 +74,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "CA") { else if (name == "CA") {
if (ca) throw corrupt(); if (ca) throw corrupt();
// FIXME: allow blank ca or require skipping field? // FIXME: allow blank ca or require skipping field?
ca = parseContentAddressOpt(value); ca = ContentAddress::parseOpt(value);
} }
pos = eol + 1; pos = eol + 1;

View file

@ -49,7 +49,7 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef
}, },
}; };
}, },
}, *ca); }, ca->raw);
} }
bool ValidPathInfo::isContentAddressed(const Store & store) const bool ValidPathInfo::isContentAddressed(const Store & store) const
@ -116,7 +116,7 @@ ValidPathInfo::ValidPathInfo(
this->references.insert(path); this->references.insert(path);
this->ca = std::move((FixedOutputHash &&) foi); this->ca = std::move((FixedOutputHash &&) foi);
}, },
}, std::move(ca)); }, std::move(ca).raw);
} }
@ -136,7 +136,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
if (format >= 16) { if (format >= 16) {
source >> info.ultimate; source >> info.ultimate;
info.sigs = readStrings<StringSet>(source); info.sigs = readStrings<StringSet>(source);
info.ca = parseContentAddressOpt(readString(source)); info.ca = ContentAddress::parseOpt(readString(source));
} }
return info; return info;
} }

View file

@ -44,7 +44,7 @@ void write(const Store & store, Sink & out, const StorePath & storePath)
ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _) ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _)
{ {
return parseContentAddress(readString(from)); return ContentAddress::parse(readString(from));
} }
void write(const Store & store, Sink & out, const ContentAddress & ca) void write(const Store & store, Sink & out, const ContentAddress & ca)
@ -134,7 +134,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto
std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _) std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _)
{ {
return parseContentAddressOpt(readString(from)); return ContentAddress::parseOpt(readString(from));
} }
void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt) void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt)
@ -545,7 +545,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
conn->to conn->to
<< wopAddToStore << wopAddToStore
<< name << name
<< renderContentAddressMethod(caMethod); << caMethod.render();
worker_proto::write(*this, conn->to, references); worker_proto::write(*this, conn->to, references);
conn->to << repair; conn->to << repair;
@ -603,7 +603,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
} }
} }
}, caMethod); }, caMethod.raw);
auto path = parseStorePath(readString(conn->from)); auto path = parseStorePath(readString(conn->from));
// Release our connection to prevent a deadlock in queryPathInfo(). // Release our connection to prevent a deadlock in queryPathInfo().
conn_.reset(); conn_.reset();

View file

@ -221,7 +221,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
[&](const FixedOutputInfo & foi) { [&](const FixedOutputInfo & foi) {
return makeFixedOutputPath(name, foi); return makeFixedOutputPath(name, foi);
} }
}, ca); }, ca.raw);
} }

View file

@ -970,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs)
info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {});
in >> info.registrationTime >> info.narSize >> info.ultimate; in >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(in); info.sigs = readStrings<StringSet>(in);
info.ca = parseContentAddressOpt(readString(in)); info.ca = ContentAddress::parseOpt(readString(in));
if (info.narSize == 0) if (info.narSize == 0)
throw Error("narInfo is too old and missing the narSize field"); throw Error("narInfo is too old and missing the narSize field");

View file

@ -124,7 +124,7 @@ std::tuple<StorePath, Hash> prefetchFile(
auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
storePath = info.path; storePath = info.path;
assert(info.ca); assert(info.ca);
hash = getContentAddressHash(*info.ca); hash = info.ca->getHash();
} }
return {storePath.value(), hash.value()}; return {storePath.value(), hash.value()};