max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2024-11-22 05:56:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #9815 from NixOS/nix-hash-path

Browse source 
`nix hash path`, text hashing for `nix store add`, and preparatory refactors
This commit is contained in:
Théophane Hufschmitt 2024-02-22 17:15:34 +01:00 committed by GitHub
commit 6a5210f48e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 245 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

121
src/libcmd/misc-store-flags.cc Normal file
View file

@ -0,0 +1,121 @@
#include "misc-store-flags.hh"
namespace nix::flag
{
static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
{
for (auto & format : hashFormats) {
if (hasPrefix(format, prefix)) {
completions.add(format);
}
}
}
Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf)
{
assert(*hf == nix::HashFormat::SRI);
return Args::Flag {
.longName = std::move(longName),
.description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.",
.labels = {"hash-format"},
.handler = {[hf](std::string s) {
*hf = parseHashFormat(s);
}},
.completer = hashFormatCompleter,
};
}
Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf)
{
return Args::Flag {
.longName = std::move(longName),
.description = "Hash format (`base16`, `nix32`, `base64`, `sri`).",
.labels = {"hash-format"},
.handler = {[ohf](std::string s) {
*ohf = std::optional<HashFormat>{parseHashFormat(s)};
}},
.completer = hashFormatCompleter,
};
}
static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
{
for (auto & algo : hashAlgorithms)
if (hasPrefix(algo, prefix))
completions.add(algo);
}
Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha)
{
return Args::Flag {
.longName = std::move(longName),
.description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
.labels = {"hash-algo"},
.handler = {[ha](std::string s) {
*ha = parseHashAlgo(s);
}},
.completer = hashAlgoCompleter,
};
}
Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha)
{
return Args::Flag {
.longName = std::move(longName),
.description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
.labels = {"hash-algo"},
.handler = {[oha](std::string s) {
*oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
}},
.completer = hashAlgoCompleter,
};
}
Args::Flag fileIngestionMethod(FileIngestionMethod * method)
{
return Args::Flag {
.longName = "mode",
// FIXME indentation carefully made for context, this is messed up.
.description = R"(
How to compute the hash of the input.
One of:
- `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
- `flat`: Assumes that the input is a single file and directly passes it to the hash function;
)",
.labels = {"file-ingestion-method"},
.handler = {[method](std::string s) {
*method = parseFileIngestionMethod(s);
}},
};
}
Args::Flag contentAddressMethod(ContentAddressMethod * method)
{
return Args::Flag {
.longName = "mode",
// FIXME indentation carefully made for context, this is messed up.
.description = R"(
How to compute the content-address of the store object.
One of:
- `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
- `flat`: Assumes that the input is a single file and directly passes it to the hash function;
- `text`: Like `flat`, but used for
[derivations](@docroot@/glossary.md#store-derivation) serialized in store object and
[`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
For advanced use-cases only;
for regular usage prefer `nar` and `flat.
)",
.labels = {"content-address-method"},
.handler = {[method](std::string s) {
*method = ContentAddressMethod::parse(s);
}},
};
}
}

21
src/libcmd/misc-store-flags.hh Normal file
View file

@ -0,0 +1,21 @@
#include "args.hh"
#include "content-address.hh"
namespace nix::flag {
Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha);
static inline Args::Flag hashAlgo(HashAlgorithm * ha)
{
return hashAlgo("hash-algo", ha);
}
Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha);
Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf);
Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf);
static inline Args::Flag hashAlgoOpt(std::optional<HashAlgorithm> * oha)
{
return hashAlgoOpt("hash-algo", oha);
}
Args::Flag fileIngestionMethod(FileIngestionMethod * method);
Args::Flag contentAddressMethod(ContentAddressMethod * method);
}

67
src/libutil/args.cc
View file

@ -544,73 +544,6 @@ nlohmann::json Args::toJSON()
return res;
}
static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
{
for (auto & format : hashFormats) {
if (hasPrefix(format, prefix)) {
completions.add(format);
}
}
}
Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) {
assert(*hf == nix::HashFormat::SRI);
return Flag{
.longName = std::move(longName),
.description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.",
.labels = {"hash-format"},
.handler = {[hf](std::string s) {
*hf = parseHashFormat(s);
}},
.completer = hashFormatCompleter,
};
}
Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf) {
return Flag{
.longName = std::move(longName),
.description = "Hash format (`base16`, `nix32`, `base64`, `sri`).",
.labels = {"hash-format"},
.handler = {[ohf](std::string s) {
*ohf = std::optional<HashFormat>{parseHashFormat(s)};
}},
.completer = hashFormatCompleter,
};
}
static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
{
for (auto & algo : hashAlgorithms)
if (hasPrefix(algo, prefix))
completions.add(algo);
}
Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha)
{
return Flag{
.longName = std::move(longName),
.description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
.labels = {"hash-algo"},
.handler = {[ha](std::string s) {
*ha = parseHashAlgo(s);
}},
.completer = hashAlgoCompleter,
};
}
Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
{
return Flag{
.longName = std::move(longName),
.description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
.labels = {"hash-algo"},
.handler = {[oha](std::string s) {
*oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
}},
.completer = hashAlgoCompleter,
};
}
static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs)
{
completions.setType(Completions::Type::Filenames);

19
src/libutil/args.hh
View file

@ -155,6 +155,8 @@ protected:
*/
using CompleterClosure = std::function<CompleterFun>;
public:
/**
* Description of flags / options
*
@ -175,19 +177,10 @@ protected:
CompleterClosure completer;
std::optional<ExperimentalFeature> experimentalFeature;
static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha);
static Flag mkHashAlgoFlag(HashAlgorithm * ha) {
return mkHashAlgoFlag("hash-algo", ha);
}
static Flag mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
static Flag mkHashAlgoOptFlag(std::optional<HashAlgorithm> * oha) {
return mkHashAlgoOptFlag("hash-algo", oha);
}
static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf);
static Flag mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf);
};
protected:
/**
* Index of all registered "long" flag descriptions (flags like
* `--long`).
@ -206,6 +199,8 @@ protected:
*/
virtual bool processFlag(Strings::iterator & pos, Strings::iterator end);
public:
/**
* Description of positional arguments
*
@ -220,6 +215,8 @@ protected:
CompleterClosure completer;
};
protected:
/**
* Queue of expected positional argument forms.
*

20
src/nix/add-to-store.cc
View file

@ -3,6 +3,7 @@
#include "store-api.hh"
#include "archive.hh"
#include "posix-source-accessor.hh"
#include "misc-store-flags.hh"
using namespace nix;
@ -26,23 +27,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand
.handler = {&namePart},
});
addFlag({
.longName = "mode",
.description = R"(
How to compute the hash of the input.
One of:
addFlag(flag::contentAddressMethod(&caMethod));
- `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
- `flat`: Assumes that the input is a single file and directly passes it to the hash function;
)",
.labels = {"hash-mode"},
.handler = {[this](std::string s) {
this->caMethod = parseFileIngestionMethod(s);
}},
});
addFlag(Flag::mkHashAlgoFlag(&hashAlgo));
addFlag(flag::hashAlgo(&hashAlgo));
}
void run(ref<Store> store) override
@ -63,7 +50,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand
struct CmdAdd : CmdAddToStore
{
std::string description() override
{
return "Add a file or directory to the Nix store";

99
src/nix/hash.cc
View file

@ -6,11 +6,12 @@
#include "references.hh"
#include "archive.hh"
#include "posix-source-accessor.hh"
#include "misc-store-flags.hh"
using namespace nix;
/**
* Base for `nix hash file` (deprecated), `nix hash path` and `nix-hash` (legacy).
* Base for `nix hash path`, `nix hash file` (deprecated), and `nix-hash` (legacy).
*
* Deprecation Issue: https://github.com/NixOS/nix/issues/8876
*/
@ -19,12 +20,21 @@ struct CmdHashBase : Command
FileIngestionMethod mode;
HashFormat hashFormat = HashFormat::SRI;
bool truncate = false;
HashAlgorithm ha = HashAlgorithm::SHA256;
HashAlgorithm hashAlgo = HashAlgorithm::SHA256;
std::vector<std::string> paths;
std::optional<std::string> modulus;
explicit CmdHashBase(FileIngestionMethod mode) : mode(mode)
{
expectArgs({
.label = "paths",
.handler = {&paths},
.completer = completePath
});
// FIXME The following flags should be deprecated, but we don't
// yet have a mechanism for that.
addFlag({
.longName = "sri",
.description = "Print the hash in SRI format.",
@ -49,22 +59,7 @@ struct CmdHashBase : Command
.handler = {&hashFormat, HashFormat::Base16},
});
addFlag(Flag::mkHashAlgoFlag("type", &ha));
#if 0
addFlag({
.longName = "modulo",
.description = "Compute the hash modulo the specified string.",
.labels = {"modulus"},
.handler = {&modulus},
});
#endif\
expectArgs({
.label = "paths",
.handler = {&paths},
.completer = completePath
});
addFlag(flag::hashAlgo("type", &hashAlgo));
}
std::string description() override
@ -85,9 +80,9 @@ struct CmdHashBase : Command
std::unique_ptr<AbstractHashSink> hashSink;
if (modulus)
hashSink = std::make_unique<HashModuloSink>(ha, *modulus);
hashSink = std::make_unique<HashModuloSink>(hashAlgo, *modulus);
else
hashSink = std::make_unique<HashSink>(ha);
hashSink = std::make_unique<HashSink>(hashAlgo);
auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path);
dumpPath(accessor, canonPath, *hashSink, mode);
@ -99,15 +94,53 @@ struct CmdHashBase : Command
}
};
/**
* `nix hash path`
*/
struct CmdHashPath : CmdHashBase
{
CmdHashPath()
: CmdHashBase(FileIngestionMethod::Recursive)
{
addFlag(flag::hashAlgo("algo", &hashAlgo));
addFlag(flag::fileIngestionMethod(&mode));
addFlag(flag::hashFormatWithDefault("format", &hashFormat));
#if 0
addFlag({
.longName = "modulo",
.description = "Compute the hash modulo the specified string.",
.labels = {"modulus"},
.handler = {&modulus},
});
#endif
}
};
/**
* For deprecated `nix hash file`
*
* Deprecation Issue: https://github.com/NixOS/nix/issues/8876
*/
struct CmdHashFile : CmdHashBase
{
CmdHashFile()
: CmdHashBase(FileIngestionMethod::Flat)
{
}
};
/**
* For deprecated `nix hash to-*`
*/
struct CmdToBase : Command
{
HashFormat hashFormat;
std::optional<HashAlgorithm> ht;
std::optional<HashAlgorithm> hashAlgo;
std::vector<std::string> args;
CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat)
{
addFlag(Flag::mkHashAlgoOptFlag("type", &ht));
addFlag(flag::hashAlgoOpt("type", &hashAlgo));
expectArgs("strings", &args);
}
@ -124,7 +157,7 @@ struct CmdToBase : Command
{
warn("The old format conversion sub commands of `nix hash` where deprecated in favor of `nix hash convert`.");
for (auto s : args)
logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI));
logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI));
}
};
@ -139,9 +172,9 @@ struct CmdHashConvert : Command
std::vector<std::string> hashStrings;
CmdHashConvert(): to(HashFormat::SRI) {
addFlag(Args::Flag::mkHashFormatOptFlag("from", &from));
addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to));
addFlag(Args::Flag::mkHashAlgoOptFlag(&algo));
addFlag(flag::hashFormatOpt("from", &from));
addFlag(flag::hashFormatWithDefault("to", &to));
addFlag(flag::hashAlgoOpt(&algo));
expectArgs({
.label = "hashes",
.handler = {&hashStrings},
@ -181,8 +214,8 @@ struct CmdHash : NixMultiCommand
"hash",
{
{"convert", []() { return make_ref<CmdHashConvert>();}},
{"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
{"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
{"path", []() { return make_ref<CmdHashPath>(); }},
{"file", []() { return make_ref<CmdHashFile>(); }},
{"to-base16", []() { return make_ref<CmdToBase>(HashFormat::Base16); }},
{"to-base32", []() { return make_ref<CmdToBase>(HashFormat::Nix32); }},
{"to-base64", []() { return make_ref<CmdToBase>(HashFormat::Base64); }},
@ -206,7 +239,7 @@ static int compatNixHash(int argc, char * * argv)
// Wait until `nix hash convert` is not hidden behind experimental flags anymore.
// warn("`nix-hash` has been deprecated in favor of `nix hash convert`.");
std::optional<HashAlgorithm> ha;
std::optional<HashAlgorithm> hashAlgo;
bool flat = false;
HashFormat hashFormat = HashFormat::Base16;
bool truncate = false;
@ -226,7 +259,7 @@ static int compatNixHash(int argc, char * * argv)
else if (*arg == "--truncate") truncate = true;
else if (*arg == "--type") {
std::string s = getArg(*arg, arg, end);
ha = parseHashAlgo(s);
hashAlgo = parseHashAlgo(s);
}
else if (*arg == "--to-base16") {
op = opTo;
@ -253,8 +286,8 @@ static int compatNixHash(int argc, char * * argv)
if (op == opHash) {
CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
if (!ha.has_value()) ha = HashAlgorithm::MD5;
cmd.ha = ha.value();
if (!hashAlgo.has_value()) hashAlgo = HashAlgorithm::MD5;
cmd.hashAlgo = hashAlgo.value();
cmd.hashFormat = hashFormat;
cmd.truncate = truncate;
cmd.paths = ss;
@ -264,7 +297,7 @@ static int compatNixHash(int argc, char * * argv)
else {
CmdToBase cmd(hashFormat);
cmd.args = ss;
if (ha.has_value()) cmd.ht = ha;
if (hashAlgo.has_value()) cmd.hashAlgo = hashAlgo;
cmd.run();
}

3
src/nix/prefetch.cc
View file

@ -10,6 +10,7 @@
#include "eval-inline.hh"
#include "legacy.hh"
#include "posix-source-accessor.hh"
#include "misc-store-flags.hh"
#include <nlohmann/json.hpp>
@ -284,7 +285,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
}}
});
addFlag(Flag::mkHashAlgoFlag("hash-type", &hashAlgo));
addFlag(flag::hashAlgo("hash-type", &hashAlgo));
addFlag({
.longName = "executable",

5
tests/functional/add.sh
View file

@ -45,3 +45,8 @@ clearStore
[[ "$path1" == "$path2" ]]
path4=$(nix store add --mode flat --hash-algo sha1 ./dummy)
)
(
path1=$(nix store add --mode text ./dummy)
path2=$(nix eval --impure --raw --expr 'builtins.toFile "dummy" (builtins.readFile ./dummy)')
[[ "$path1" == "$path2" ]]
)

28
tests/functional/hash-path.sh
View file

@ -2,19 +2,24 @@ source common.sh
try () {
printf "%s" "$2" > $TEST_ROOT/vector
hash="$(nix-hash --flat ${FORMAT_FLAG-} --type "$1" "$TEST_ROOT/vector")"
hash="$(nix-hash --flat ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")"
if ! (( "${NO_TEST_CLASSIC-}" )) && test "$hash" != "$3"; then
echo "try nix-hash: hash $1, expected $3, got $hash"
exit 1
fi
hash="$(nix hash file ${FORMAT_FLAG-} --type "$1" "$TEST_ROOT/vector")"
hash="$(nix hash file ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")"
if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then
echo "try nix hash: hash $1, expected $3, got $hash"
exit 1
fi
hash="$(nix hash path --mode flat ${FORMAT+--format $FORMAT} --algo "$1" "$TEST_ROOT/vector")"
if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then
echo "try nix hash: hash $1, expected $3, got $hash"
exit 1
fi
}
FORMAT_FLAG=--base16
FORMAT=base16
try md5 "" "d41d8cd98f00b204e9800998ecf8427e"
try md5 "a" "0cc175b9c0f1b6a831c399e269772661"
try md5 "abc" "900150983cd24fb0d6963f7d28e17f72"
@ -34,18 +39,18 @@ try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "248d6a61d
try sha512 "" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"
try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"
try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445"
unset FORMAT_FLAG
unset FORMAT
FORMAT_FLAG=--base32
FORMAT=base32
try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
unset FORMAT_FLAG
unset FORMAT
FORMAT_FLAG=--sri
FORMAT=sri
try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="
try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw=="
try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ=="
try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE="
unset FORMAT_FLAG
unset FORMAT
# nix-hash [--flat] defaults to the Base16 format
NO_TEST_NIX_COMMAND=1 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"
@ -56,7 +61,12 @@ NO_TEST_CLASSIC=1 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5
try2 () {
hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path)
if test "$hash" != "$2"; then
echo "hash $1, expected $2, got $hash"
echo "try nix-hash; hash $1, expected $2, got $hash"
exit 1
fi
hash="$(nix hash path --mode nar --format base16 --algo "$1" "$TEST_ROOT/hash-path")"
if test "$hash" != "$2"; then
echo "try nix hash: hash $1, expected $2, got $hash"
exit 1
fi
}