mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-12 17:26:19 +02:00
Make ./mk/run-test.sh
work by itself; add mk/debug-test.sh
First, logic is consolidated in the shell script instead of being spread between them and makefiles. That makes understanding what is going on a little easier. This would not be super interesting by itself, but it gives us a way to debug tests more easily. *That* in turn I hope is much more compelling. See the updated manual for details. Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io> Co-authored-by: Eelco Dolstra <edolstra@gmail.com> Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
This commit is contained in:
parent
1437582ccd
commit
0251d44cc2
6 changed files with 111 additions and 11 deletions
|
@ -99,8 +99,79 @@ You can run the whole testsuite with `make check`, or the tests for a specific c
|
||||||
### Functional tests
|
### Functional tests
|
||||||
|
|
||||||
The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
|
The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
|
||||||
The whole testsuite can be run with `make install && make installcheck`.
|
Each test is a bash script.
|
||||||
Individual tests can be run with `make tests/{testName}.sh.test`.
|
|
||||||
|
The whole testsuite can be run with:
|
||||||
|
|
||||||
|
```shell-session
|
||||||
|
$ make install && make installcheck
|
||||||
|
ran test tests/foo.sh... [PASS]
|
||||||
|
ran test tests/bar.sh... [PASS]
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
Individual tests can be run with `make`:
|
||||||
|
|
||||||
|
```shell-session
|
||||||
|
$ make tests/${testName}.sh.test
|
||||||
|
ran test tests/${testName}.sh... [PASS]
|
||||||
|
```
|
||||||
|
|
||||||
|
or without `make`:
|
||||||
|
|
||||||
|
```shell-session
|
||||||
|
$ ./mk/run-test.sh tests/${testName}.sh
|
||||||
|
ran test tests/${testName}.sh... [PASS]
|
||||||
|
```
|
||||||
|
|
||||||
|
To see the complet eoutput, one can also run:
|
||||||
|
|
||||||
|
```shell-session
|
||||||
|
$ ./mk/debug-test.sh tests/${testName}.sh
|
||||||
|
+ foo
|
||||||
|
output from foo
|
||||||
|
+ bar
|
||||||
|
output from bar
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
The test script will be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails.
|
||||||
|
|
||||||
|
#### Debugging failing functional tests
|
||||||
|
|
||||||
|
When a functional test fails, it usually does so somewhere in the middle of the script.
|
||||||
|
|
||||||
|
To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB.
|
||||||
|
|
||||||
|
For example, if the script looks like:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
foo
|
||||||
|
nix blah blub
|
||||||
|
bar
|
||||||
|
```
|
||||||
|
one would could edit it like so:
|
||||||
|
|
||||||
|
```diff
|
||||||
|
foo
|
||||||
|
-nix blah blub
|
||||||
|
+gdb --args nix blah blub
|
||||||
|
bar
|
||||||
|
```
|
||||||
|
|
||||||
|
Then, when one runs the script with `./mk/debug-test.sh`, it will drop them into GDB once the script reaches that point:
|
||||||
|
|
||||||
|
```shell-session
|
||||||
|
$ ./mk/debug-test.sh tests/${testName}.sh
|
||||||
|
...
|
||||||
|
+ gdb blash blub
|
||||||
|
GNU gdb (GDB) 12.1
|
||||||
|
...
|
||||||
|
(gdb)
|
||||||
|
```
|
||||||
|
|
||||||
|
One can debug the Nix invocation in all the usual ways.
|
||||||
|
(For exampling running `run` will run the Nix invocation.)
|
||||||
|
|
||||||
### Integration tests
|
### Integration tests
|
||||||
|
|
||||||
|
|
11
mk/common-test.sh
Normal file
11
mk/common-test.sh
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=')
|
||||||
|
|
||||||
|
: ${BASH:=/usr/bin/env bash}
|
||||||
|
|
||||||
|
init_test () {
|
||||||
|
cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
run_test_proper () {
|
||||||
|
cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
|
||||||
|
}
|
11
mk/debug-test.sh
Executable file
11
mk/debug-test.sh
Executable file
|
@ -0,0 +1,11 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
test=$1
|
||||||
|
|
||||||
|
dir="$(dirname "${BASH_SOURCE[0]}")"
|
||||||
|
source "$dir/common-test.sh"
|
||||||
|
|
||||||
|
(init_test)
|
||||||
|
run_test_proper
|
|
@ -1,4 +1,4 @@
|
||||||
#!/bin/sh
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
set -u
|
set -u
|
||||||
|
|
||||||
|
@ -7,7 +7,12 @@ green=""
|
||||||
yellow=""
|
yellow=""
|
||||||
normal=""
|
normal=""
|
||||||
|
|
||||||
post_run_msg="ran test $1..."
|
test=$1
|
||||||
|
|
||||||
|
dir="$(dirname "${BASH_SOURCE[0]}")"
|
||||||
|
source "$dir/common-test.sh"
|
||||||
|
|
||||||
|
post_run_msg="ran test $test..."
|
||||||
if [ -t 1 ]; then
|
if [ -t 1 ]; then
|
||||||
red="[31;1m"
|
red="[31;1m"
|
||||||
green="[32;1m"
|
green="[32;1m"
|
||||||
|
@ -16,12 +21,12 @@ if [ -t 1 ]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
run_test () {
|
run_test () {
|
||||||
(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
|
(init_test 2>/dev/null > /dev/null)
|
||||||
log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
|
log="$(run_test_proper 2>&1)"
|
||||||
status=$?
|
status=$?
|
||||||
}
|
}
|
||||||
|
|
||||||
run_test "$1"
|
run_test
|
||||||
|
|
||||||
# Hack: Retry the test if it fails with “unexpected EOF reading a line” as these
|
# Hack: Retry the test if it fails with “unexpected EOF reading a line” as these
|
||||||
# appear randomly without anyone knowing why.
|
# appear randomly without anyone knowing why.
|
||||||
|
@ -32,7 +37,7 @@ if [[ $status -ne 0 && $status -ne 99 && \
|
||||||
]]; then
|
]]; then
|
||||||
echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)"
|
echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)"
|
||||||
echo "$log" | sed 's/^/ /'
|
echo "$log" | sed 's/^/ /'
|
||||||
run_test "$1"
|
run_test
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ $status -eq 0 ]; then
|
if [ $status -eq 0 ]; then
|
|
@ -8,7 +8,11 @@ define run-install-test
|
||||||
|
|
||||||
.PHONY: $1.test
|
.PHONY: $1.test
|
||||||
$1.test: $1 $(test-deps)
|
$1.test: $1 $(test-deps)
|
||||||
@env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
|
@env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
|
||||||
|
|
||||||
|
.PHONY: $1.test-debug
|
||||||
|
$1.test-debug: $1 $(test-deps)
|
||||||
|
@env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null
|
||||||
|
|
||||||
endef
|
endef
|
||||||
|
|
||||||
|
|
|
@ -121,8 +121,6 @@ endif
|
||||||
|
|
||||||
install-tests += $(foreach x, $(nix_tests), tests/$(x))
|
install-tests += $(foreach x, $(nix_tests), tests/$(x))
|
||||||
|
|
||||||
tests-environment = NIX_REMOTE= $(bash) -e
|
|
||||||
|
|
||||||
clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix
|
clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix
|
||||||
|
|
||||||
test-deps += tests/common.sh tests/config.nix tests/ca/config.nix
|
test-deps += tests/common.sh tests/config.nix tests/ca/config.nix
|
||||||
|
|
Loading…
Reference in a new issue