Factor out a bit more language testings infra

Will be used in a second test after `lang.sh`.
This commit is contained in:
John Ericson 2024-06-23 15:33:45 -04:00
parent df068734ac
commit 490ca93cf8
7 changed files with 82 additions and 67 deletions

View file

@ -522,6 +522,7 @@
''^tests/functional/ca/repl\.sh$''
''^tests/functional/ca/selfref-gc\.sh$''
''^tests/functional/ca/why-depends\.sh$''
''^tests/functional/characterisation-test-infra\.sh$''
''^tests/functional/check\.sh$''
''^tests/functional/common/vars-and-functions\.sh$''
''^tests/functional/completions\.sh$''
@ -579,9 +580,7 @@
''^tests/functional/impure-env\.sh$''
''^tests/functional/impure-eval\.sh$''
''^tests/functional/install-darwin\.sh$''
''^tests/functional/lang-test-infra\.sh$''
''^tests/functional/lang\.sh$''
''^tests/functional/lang/framework\.sh$''
''^tests/functional/legacy-ssh-store\.sh$''
''^tests/functional/linux-sandbox\.sh$''
''^tests/functional/local-overlay-store/add-lower-inner\.sh$''

View file

@ -3,7 +3,7 @@
# Test the function for lang.sh
source common.sh
source lang/framework.sh
source characterisation/framework.sh
# We are testing this, so don't want outside world to affect us.
unset _NIX_TEST_ACCEPT

View file

@ -0,0 +1,77 @@
# shellcheck shell=bash
# Golden test support
#
# Test that the output of the given test matches what is expected. If
# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so
# that next time the test succeeds.
function diffAndAcceptInner() {
local -r testName=$1
local -r got="$2"
local -r expected="$3"
# Absence of expected file indicates empty output expected.
if test -e "$expected"; then
local -r expectedOrEmpty="$expected"
else
local -r expectedOrEmpty=characterisation/empty
fi
# Diff so we get a nice message
if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then
echo "FAIL: evaluation result of $testName not as expected"
# shellcheck disable=SC2034
badDiff=1
fi
# Update expected if `_NIX_TEST_ACCEPT` is non-empty.
if test -n "${_NIX_TEST_ACCEPT-}"; then
cp "$got" "$expected"
# Delete empty expected files to avoid bloating the repo with
# empty files.
if ! test -s "$expected"; then
rm "$expected"
fi
fi
}
function characterisationTestExit() {
# Make sure shellcheck knows all these will be defined by the caller
: "${badDiff?} ${badExitCode?}"
if test -n "${_NIX_TEST_ACCEPT-}"; then
if (( "$badDiff" )); then
set +x
echo 'Output did mot match, but accepted output as the persisted expected output.'
echo 'That means the next time the tests are run, they should pass.'
set -x
else
set +x
echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,'
echo 'indicating the unexpected output should be accepted as the expected output going forward,'
echo 'but no tests had unexpected output so there was no expected output to update.'
set -x
fi
if (( "$badExitCode" )); then
exit "$badExitCode"
else
skipTest "regenerating golden masters"
fi
else
if (( "$badDiff" )); then
set +x
echo ''
echo 'You can rerun this test with:'
echo ''
echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.test"
echo ''
echo 'to regenerate the files containing the expected output,'
echo 'and then view the git diff to decide whether a change is'
echo 'good/intentional or bad/unintentional.'
echo 'If the diff contains arbitrary or impure information,'
echo 'please improve the normalization that the test applies to the output.'
set -x
fi
exit $(( "$badExitCode" + "$badDiff" ))
fi
}

View file

@ -4,7 +4,7 @@ source common.sh
set -o pipefail
source lang/framework.sh
source characterisation/framework.sh
# specialize function a bit
function diffAndAccept() {
@ -138,32 +138,4 @@ for i in lang/eval-okay-*.nix; do
fi
done
if test -n "${_NIX_TEST_ACCEPT-}"; then
if (( "$badDiff" )); then
echo 'Output did mot match, but accepted output as the persisted expected output.'
echo 'That means the next time the tests are run, they should pass.'
else
echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,'
echo 'indicating the unexpected output should be accepted as the expected output going forward,'
echo 'but no tests had unexpected output so there was no expected output to update.'
fi
if (( "$badExitCode" )); then
exit "$badExitCode"
else
skipTest "regenerating golden masters"
fi
else
if (( "$badDiff" )); then
echo ''
echo 'You can rerun this test with:'
echo ''
echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test'
echo ''
echo 'to regenerate the files containing the expected output,'
echo 'and then view the git diff to decide whether a change is'
echo 'good/intentional or bad/unintentional.'
echo 'If the diff contains arbitrary or impure information,'
echo 'please improve the normalization that the test applies to the output.'
fi
exit $(( "$badExitCode" + "$badDiff" ))
fi
characterisationTestExit

View file

@ -1,33 +0,0 @@
# Golden test support
#
# Test that the output of the given test matches what is expected. If
# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so
# that next time the test succeeds.
function diffAndAcceptInner() {
local -r testName=$1
local -r got="$2"
local -r expected="$3"
# Absence of expected file indicates empty output expected.
if test -e "$expected"; then
local -r expectedOrEmpty="$expected"
else
local -r expectedOrEmpty=lang/empty.exp
fi
# Diff so we get a nice message
if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then
echo "FAIL: evaluation result of $testName not as expected"
badDiff=1
fi
# Update expected if `_NIX_TEST_ACCEPT` is non-empty.
if test -n "${_NIX_TEST_ACCEPT-}"; then
cp "$got" "$expected"
# Delete empty expected files to avoid bloating the repo with
# empty files.
if ! test -s "$expected"; then
rm "$expected"
fi
fi
}

View file

@ -23,7 +23,7 @@ nix_tests = \
remote-store.sh \
legacy-ssh-store.sh \
lang.sh \
lang-test-infra.sh \
characterisation-test-infra.sh \
experimental-features.sh \
fetchMercurial.sh \
gc-auto.sh \