nix-super/tests/functional/binary-cache.sh

306 lines
9.1 KiB
Bash
Raw Normal View History

#!/usr/bin/env bash
source common.sh
TODO_NixOS
needLocalStore "'--no-require-sigs' cant be used with the daemon"
# We can produce drvs directly into the binary cache
clearStore
clearCacheCache
nix-instantiate --store "file://$cacheDir" dependencies.nix
# Create the binary cache.
clearStore
clearCache
outPath=$(nix-build dependencies.nix --no-out-link)
nix copy --to "file://$cacheDir" "$outPath"
readarray -t paths < <(nix path-info --all --json --store "file://$cacheDir" | jq 'keys|sort|.[]' -r)
[[ "${#paths[@]}" -eq 3 ]]
for path in "${paths[@]}"; do
[[ "$path" =~ -dependencies-input-0$ ]] \
|| [[ "$path" =~ -dependencies-input-2$ ]] \
|| [[ "$path" =~ -dependencies-top$ ]]
done
# Test copying build logs to the binary cache.
expect 1 nix log --store "file://$cacheDir" "$outPath" 2>&1 | grep 'is not available'
nix store copy-log --to "file://$cacheDir" "$outPath"
nix log --store "file://$cacheDir" "$outPath" | grep FOO
rm -rf "$TEST_ROOT/var/log/nix"
expect 1 nix log "$outPath" 2>&1 | grep 'is not available'
nix log --substituters "file://$cacheDir" "$outPath" | grep FOO
# Test copying build logs from the binary cache.
nix store copy-log --from "file://$cacheDir" "$(nix-store -qd "$outPath")"^'*'
nix log "$outPath" | grep FOO
basicDownloadTests() {
# No uploading tests bcause upload with force HTTP doesn't work.
# By default, a binary cache doesn't support "nix-env -qas", but does
# support installation.
clearStore
clearCacheCache
2013-04-23 13:43:28 +03:00
nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath"
[ -x "$outPath/program" ]
2014-02-26 19:59:01 +02:00
# But with the right configuration, "nix-env -qas" should also work.
clearStore
clearCacheCache
echo "WantMassQuery: 1" >> "$cacheDir/nix-cache-info"
nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S"
x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only)
[ -z "$x" ]
nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath"
nix-store --check-validity "$outPath"
nix-store -qR "$outPath" | grep input-2
echo "WantMassQuery: 0" >> "$cacheDir/nix-cache-info"
}
# Test LocalBinaryCacheStore.
basicDownloadTests
# Test HttpBinaryCacheStore.
2020-03-24 15:26:13 +02:00
export _NIX_FORCE_HTTP=1
basicDownloadTests
2012-12-03 22:02:06 +02:00
# Test whether Nix notices if the NAR doesn't match the hash in the NAR info.
clearStore
nar=$(find "$cacheDir/nar/" -type f -name "*.nar.xz" | head -n1)
mv "$nar" "$nar".good
mkdir -p "$TEST_ROOT/empty"
nix-store --dump "$TEST_ROOT/empty" | xz > "$nar"
expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log"
grepQuiet "hash mismatch" "$TEST_ROOT/log"
mv "$nar".good "$nar"
Support cryptographically signed binary caches NAR info files in binary caches can now have a cryptographic signature that Nix will verify before using the corresponding NAR file. To create a private/public key pair for signing and verifying a binary cache, do: $ openssl genrsa -out ./cache-key.sec 2048 $ openssl rsa -in ./cache-key.sec -pubout > ./cache-key.pub You should also come up with a symbolic name for the key, such as "cache.example.org-1". This will be used by clients to look up the public key. (It's a good idea to number keys, in case you ever need to revoke/replace one.) To create a binary cache signed with the private key: $ nix-push --dest /path/to/binary-cache --key ./cache-key.sec --key-name cache.example.org-1 The public key (cache-key.pub) should be distributed to the clients. They should have a nix.conf should contain something like: signed-binary-caches = * binary-cache-public-key-cache.example.org-1 = /path/to/cache-key.pub If all works well, then if Nix fetches something from the signed binary cache, you will see a message like: *** Downloading ‘http://cache.example.org/nar/7dppcj5sc1nda7l54rjc0g5l1hamj09j-subversion-1.7.11’ (signed by ‘cache.example.org-1’) to ‘/nix/store/7dppcj5sc1nda7l54rjc0g5l1hamj09j-subversion-1.7.11’... On the other hand, if the signature is wrong, you get a message like NAR info file `http://cache.example.org/7dppcj5sc1nda7l54rjc0g5l1hamj09j.narinfo' has an invalid signature; ignoring Signatures are implemented as a single line appended to the NAR info file, which looks like this: Signature: 1;cache.example.org-1;HQ9Xzyanq9iV...muQ== Thus the signature has 3 fields: a version (currently "1"), the ID of key, and the base64-encoded signature of the SHA-256 hash of the contents of the NAR info file up to but not including the Signature line. Issue #75.
2014-01-08 16:23:41 +02:00
# Test whether this unsigned cache is rejected if the user requires signed caches.
clearStore
2016-05-30 15:53:57 +03:00
clearCacheCache
Support cryptographically signed binary caches NAR info files in binary caches can now have a cryptographic signature that Nix will verify before using the corresponding NAR file. To create a private/public key pair for signing and verifying a binary cache, do: $ openssl genrsa -out ./cache-key.sec 2048 $ openssl rsa -in ./cache-key.sec -pubout > ./cache-key.pub You should also come up with a symbolic name for the key, such as "cache.example.org-1". This will be used by clients to look up the public key. (It's a good idea to number keys, in case you ever need to revoke/replace one.) To create a binary cache signed with the private key: $ nix-push --dest /path/to/binary-cache --key ./cache-key.sec --key-name cache.example.org-1 The public key (cache-key.pub) should be distributed to the clients. They should have a nix.conf should contain something like: signed-binary-caches = * binary-cache-public-key-cache.example.org-1 = /path/to/cache-key.pub If all works well, then if Nix fetches something from the signed binary cache, you will see a message like: *** Downloading ‘http://cache.example.org/nar/7dppcj5sc1nda7l54rjc0g5l1hamj09j-subversion-1.7.11’ (signed by ‘cache.example.org-1’) to ‘/nix/store/7dppcj5sc1nda7l54rjc0g5l1hamj09j-subversion-1.7.11’... On the other hand, if the signature is wrong, you get a message like NAR info file `http://cache.example.org/7dppcj5sc1nda7l54rjc0g5l1hamj09j.narinfo' has an invalid signature; ignoring Signatures are implemented as a single line appended to the NAR info file, which looks like this: Signature: 1;cache.example.org-1;HQ9Xzyanq9iV...muQ== Thus the signature has 3 fields: a version (currently "1"), the ID of key, and the base64-encoded signature of the SHA-256 hash of the contents of the NAR info file up to but not including the Signature line. Issue #75.
2014-01-08 16:23:41 +02:00
if nix-store --substituters "file://$cacheDir" -r "$outPath"; then
2014-01-08 18:56:58 +02:00
echo "unsigned binary cache incorrectly accepted"
exit 1
fi
Support cryptographically signed binary caches NAR info files in binary caches can now have a cryptographic signature that Nix will verify before using the corresponding NAR file. To create a private/public key pair for signing and verifying a binary cache, do: $ openssl genrsa -out ./cache-key.sec 2048 $ openssl rsa -in ./cache-key.sec -pubout > ./cache-key.pub You should also come up with a symbolic name for the key, such as "cache.example.org-1". This will be used by clients to look up the public key. (It's a good idea to number keys, in case you ever need to revoke/replace one.) To create a binary cache signed with the private key: $ nix-push --dest /path/to/binary-cache --key ./cache-key.sec --key-name cache.example.org-1 The public key (cache-key.pub) should be distributed to the clients. They should have a nix.conf should contain something like: signed-binary-caches = * binary-cache-public-key-cache.example.org-1 = /path/to/cache-key.pub If all works well, then if Nix fetches something from the signed binary cache, you will see a message like: *** Downloading ‘http://cache.example.org/nar/7dppcj5sc1nda7l54rjc0g5l1hamj09j-subversion-1.7.11’ (signed by ‘cache.example.org-1’) to ‘/nix/store/7dppcj5sc1nda7l54rjc0g5l1hamj09j-subversion-1.7.11’... On the other hand, if the signature is wrong, you get a message like NAR info file `http://cache.example.org/7dppcj5sc1nda7l54rjc0g5l1hamj09j.narinfo' has an invalid signature; ignoring Signatures are implemented as a single line appended to the NAR info file, which looks like this: Signature: 1;cache.example.org-1;HQ9Xzyanq9iV...muQ== Thus the signature has 3 fields: a version (currently "1"), the ID of key, and the base64-encoded signature of the SHA-256 hash of the contents of the NAR info file up to but not including the Signature line. Issue #75.
2014-01-08 16:23:41 +02:00
# Test whether fallback works if a NAR has disappeared. This does not require --fallback.
clearStore
mv "$cacheDir/nar" "$cacheDir/nar2"
nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result"
mv "$cacheDir/nar2" "$cacheDir/nar"
# Test whether fallback works if a NAR is corrupted. This does require --fallback.
clearStore
mv "$cacheDir/nar" "$cacheDir/nar2"
mkdir "$cacheDir/nar"
for i in $(cd "$cacheDir/nar2" && echo *); do touch "$cacheDir"/nar/"$i"; done
(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result")
nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" --fallback
2018-06-05 15:44:26 +03:00
rm -rf "$cacheDir/nar"
mv "$cacheDir/nar2" "$cacheDir/nar"
# Test whether building works if the binary cache contains an
# incomplete closure.
clearStore
rm -v "$(grep -l "StorePath:.*dependencies-input-2" "$cacheDir"/*.narinfo)"
nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log"
grepQuiet "copying path.*input-0" "$TEST_ROOT/log"
grepQuiet "copying path.*input-2" "$TEST_ROOT/log"
grepQuiet "copying path.*top" "$TEST_ROOT/log"
# Idem, but without cached .narinfo.
clearStore
clearCacheCache
nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log"
grepQuiet "don't know how to build" "$TEST_ROOT/log"
grepQuiet "building.*input-1" "$TEST_ROOT/log"
grepQuiet "building.*input-2" "$TEST_ROOT/log"
grepQuiet "copying path.*input-0" "$TEST_ROOT/log"
grepQuiet "copying path.*top" "$TEST_ROOT/log"
# Create a signed binary cache.
clearCache
2018-06-05 15:44:26 +03:00
clearCacheCache
nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk1"
publicKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk1")
nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk2"
badKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk2")
nix key generate-secret --key-name foo.nixos.org-1 > "$TEST_ROOT/sk3"
otherKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk3")
_NIX_FORCE_HTTP='' nix copy --to "file://$cacheDir"?secret-key="$TEST_ROOT"/sk1 "$outPath"
# Downloading should fail if we don't provide a key.
clearStore
2016-05-30 15:53:57 +03:00
clearCacheCache
(! nix-store -r "$outPath" --substituters "file://$cacheDir")
# And it should fail if we provide an incorrect key.
clearStore
2016-05-30 15:53:57 +03:00
clearCacheCache
(! nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$badKey")
# It should succeed if we provide the correct key.
nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey"
# It should fail if we corrupt the .narinfo.
clearStore
cacheDir2=$TEST_ROOT/binary-cache-2
rm -rf "$cacheDir2"
cp -r "$cacheDir" "$cacheDir2"
for i in "$cacheDir2"/*.narinfo; do
grep -v References "$i" > "$i".tmp
mv "$i".tmp "$i"
done
2016-05-30 15:53:57 +03:00
clearCacheCache
(! nix-store -r "$outPath" --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey")
# If we provide a bad and a good binary cache, it should succeed.
nix-store -r "$outPath" --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey"
2015-02-10 12:54:06 +02:00
2020-07-13 14:32:33 +03:00
unset _NIX_FORCE_HTTP
# Test 'nix verify --all' on a binary cache.
nix store verify -vvvvv --all --store "file://$cacheDir" --no-trust
2020-07-13 17:01:00 +03:00
# Test local NAR caching.
narCache=$TEST_ROOT/nar-cache
rm -rf "$narCache"
mkdir "$narCache"
2020-07-13 17:01:00 +03:00
[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]]
2020-07-13 17:01:00 +03:00
rm -rfv "$cacheDir/nar"
[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]]
2020-07-13 17:01:00 +03:00
(! nix store cat --store "file://$cacheDir" "$outPath/foobar")
2020-07-13 17:01:00 +03:00
2020-07-13 14:32:33 +03:00
# Test NAR listing generation.
clearCache
# preserve quotes variables in the single-quoted string
# shellcheck disable=SC2016
2020-07-13 14:32:33 +03:00
outPath=$(nix-build --no-out-link -E '
with import ./config.nix;
mkDerivation {
name = "nar-listing";
buildCommand = "mkdir $out; echo foo > $out/bar; ln -s xyzzy $out/link";
}
')
nix copy --to "file://$cacheDir"?write-nar-listing=1 "$outPath"
2020-07-13 14:32:33 +03:00
diff -u \
<(jq -S < "$cacheDir/$(basename "$outPath" | cut -c1-32).ls") \
<(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S)
# Test debug info index generation.
clearCache
# preserve quotes variables in the single-quoted string
# shellcheck disable=SC2016
outPath=$(nix-build --no-out-link -E '
with import ./config.nix;
mkDerivation {
name = "debug-info";
buildCommand = "mkdir -p $out/lib/debug/.build-id/02; echo foo > $out/lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug";
}
')
nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" "$outPath"
diff -u \
<(jq -S < "$cacheDir"/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) \
<(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S)
2020-10-18 01:43:52 +03:00
# Test against issue https://github.com/NixOS/nix/issues/3964
# preserve quotes variables in the single-quoted string
# shellcheck disable=SC2016
2020-10-18 01:43:52 +03:00
expr='
with import ./config.nix;
mkDerivation {
name = "multi-output";
buildCommand = "mkdir -p $out; echo foo > $doc; echo $doc > $out/docref";
outputs = ["out" "doc"];
}
'
outPath=$(nix-build --no-out-link -E "$expr")
docPath=$(nix-store -q --references "$outPath")
2020-10-18 01:43:52 +03:00
# $ nix-store -q --tree $outPath
# ...-multi-output
# +---...-multi-output-doc
nix copy --to "file://$cacheDir" "$outPath"
2020-10-18 01:43:52 +03:00
hashpart() {
basename "$1" | cut -c1-32
}
# break the closure of out by removing doc
rm "$cacheDir/$(hashpart "$docPath")".narinfo
2020-10-18 01:43:52 +03:00
nix-store --delete "$outPath" "$docPath"
2020-10-18 01:43:52 +03:00
# -vvv is the level that logs during the loop
2020-10-26 18:59:32 +02:00
timeout 60 nix-build --no-out-link -E "$expr" --option substituters "file://$cacheDir" \
2020-10-18 01:43:52 +03:00
--option trusted-binary-caches "file://$cacheDir" --no-require-sigs