Make ./mk/run-test.sh work by itself; add mk/debug-test.sh

First, logic is consolidated in the shell script instead of being spread
between them and makefiles. That makes understanding what is going on a
little easier.

This would not be super interesting by itself, but it gives us a way to
debug tests more easily. *That* in turn I hope is much more compelling.
See the updated manual for details.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>

doc/manual/src/contributing/hacking.md

@@ -99,8 +99,79 @@ You can run the whole testsuite with `make check`, or the tests for a specific c
 ### Functional tests
 
 The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
-The whole testsuite can be run with `make install && make installcheck`.
-Individual tests can be run with `make tests/{testName}.sh.test`.
+Each test is a bash script.
+
+The whole testsuite can be run with:
+
+```shell-session
+$ make install && make installcheck
+ran test tests/foo.sh... [PASS]
+ran test tests/bar.sh... [PASS]
+...
+```
+
+Individual tests can be run with `make`:
+
+```shell-session
+$ make tests/${testName}.sh.test
+ran test tests/${testName}.sh... [PASS]
+```
+
+or without `make`:
+
+```shell-session
+$ ./mk/run-test.sh tests/${testName}.sh
+ran test tests/${testName}.sh... [PASS]
+```
+
+To see the complet eoutput, one can also run:
+
+```shell-session
+$ ./mk/debug-test.sh tests/${testName}.sh
++ foo
+output from foo
++ bar
+output from bar
+...
+```
+
+The test script will be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails.
+
+#### Debugging failing functional tests
+
+When a functional test fails, it usually does so somewhere in the middle of the script.
+
+To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB.
+
+For example, if the script looks like:
+
+```bash
+foo
+nix blah blub
+bar
+```
+one would could edit it like so:
+
+```diff
+ foo
+-nix blah blub
++gdb --args nix blah blub
+ bar
+```
+
+Then, when one runs the script with `./mk/debug-test.sh`, it will drop them into GDB once the script reaches that point:
+
+```shell-session
+$ ./mk/debug-test.sh tests/${testName}.sh
+...
++ gdb blash blub
+GNU gdb (GDB) 12.1
+...
+(gdb)
+```
+
+One can debug the Nix invocation in all the usual ways.
+(For exampling running `run` will run the Nix invocation.)
 
 ### Integration tests
 

mk/common-test.sh

@@ -0,0 +1,11 @@
+TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=')
+
+: ${BASH:=/usr/bin/env bash}
+
+init_test () {
+   cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
+}
+
+run_test_proper () {
+   cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
+}

mk/debug-test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eu
+
+test=$1
+
+dir="$(dirname "${BASH_SOURCE[0]}")"
+source "$dir/common-test.sh"
+
+(init_test)
+run_test_proper

mk/run-test.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -u
 
@@ -7,7 +7,12 @@ green=""
 yellow=""
 normal=""
 
-post_run_msg="ran test $1..."
+test=$1
+
+dir="$(dirname "${BASH_SOURCE[0]}")"
+source "$dir/common-test.sh"
+
+post_run_msg="ran test $test..."
 if [ -t 1 ]; then
     red=""
     green=""
@@ -16,12 +21,12 @@ if [ -t 1 ]; then
 fi
 
 run_test () {
-    (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
-    log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+    (init_test 2>/dev/null > /dev/null)
+    log="$(run_test_proper 2>&1)"
     status=$?
 }
 
-run_test "$1"
+run_test
 
 # Hack: Retry the test if it fails with “unexpected EOF reading a line” as these
 # appear randomly without anyone knowing why.
@@ -32,7 +37,7 @@ if [[ $status -ne 0 && $status -ne 99 && \
 ]]; then
     echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)"
     echo "$log" | sed 's/^/    /'
-    run_test "$1"
+    run_test
 fi
 
 if [ $status -eq 0 ]; then

mk/tests.mk

@@ -8,7 +8,11 @@ define run-install-test
 
   .PHONY: $1.test
   $1.test: $1 $(test-deps)
-	@env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null
+	@env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
+
+  .PHONY: $1.test-debug
+  $1.test-debug: $1 $(test-deps)
+	@env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null
 
 endef
 

tests/local.mk

@@ -121,8 +121,6 @@ endif
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
 
-tests-environment = NIX_REMOTE= $(bash) -e
-
 clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix
 
 test-deps += tests/common.sh tests/config.nix tests/ca/config.nix