merge-check.sh runs and vets the integration level with the integration tag and times each level; the unit bound is not enforced yet (issue 895)

Makefile

@@ -110,14 +110,18 @@ proxy-image:
 # after formatting never ran through it. The script is the one list of steps; this target only gives it
 # a database (MESH_TEST_POSTGRES, exported above).
 #
+# **Two levels** (issue 895): the unit level, `go test ./...` untagged, timed and held to a bound; and the integration
+# level, `-tags integration`, the tests that start a bus, a store or another process. Each is said with its time.
+#
 # **Packages in parallel, under the race detector, each test on a bus of its own** (internal/testbus),
 # as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
 # order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
 check: postgres
 	@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
 
-# Without a database the store's tests skip rather than fail, so this is the honest subset and not
-# the gate.
+# The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
+# that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
+# store's tests would skip rather than fail anyway.
 test:
 	go test -timeout 15m ./...
 

merge-check.sh

@@ -9,15 +9,31 @@
 # seat's first layer (`mesh/merge-gate`), run because the mesh's module graph builds `mesh-controller`
 # from this repository; it is not repeated here. This is the code's own quality and its suite:
 #
-#   1. formatted and vetted;
-#   2. the whole suite, packages in parallel, against the throwaway store the build seat raised at the
-#      release the mesh runs; every live test on a bus of its own (internal/testbus), so nothing is
-#      serialised. Under the race detector when the toolchain has a C compiler, and said when it has not.
+#   1. formatted and vetted, with and without the build tag `integration`;
+#   2. the unit level: `go test ./...` untagged and without the race detector, timed and held to a bound
+#      (unit_bound below): a test with no database, bus, running service or other process (novox/hq ADR 0391,
+#      how-we-build section 5). With a bound set, over it the check fails and names the slowest tests (issue 895);
+#   3. the integration level: the whole suite with `-tags integration`, packages in parallel, against the
+#      throwaway store the build seat raised at the release the mesh runs; every live test on a bus of its own
+#      (internal/testbus), so nothing is serialised. Under the race detector when the toolchain has a C
+#      compiler, and said when it has not.
+#
+# **A test that starts a bus, a store or another process, or composes the whole mesh, carries
+# `//go:build integration`** as the first line of its file; a file that held both kinds is two files
+# (`<name>_integration_test.go` beside `<name>_test.go`). Each level says its test count and its time.
 #
 # Exit 0 is a pass; anything else fails `mesh/repo-check` with the last line printed.
 set -eu
 export GOFLAGS=-mod=vendor GOPROXY=off
 
+# **The unit level's bound, in seconds** (issue 895): the slowest package's own test time of the untagged run, the
+# compile not counted. It is NOT enforced yet: while this script is main's it judges the pull requests that put the
+# `integration` tag on the tests, and on main the untagged run still holds them (about a minute), so the level's time
+# is said and not judged. The release that tags the tests sets this to a number of seconds (and then to 1, how-we-build
+# section 5: the whole unit suite under a second); with a number set, a run over it fails the check and names the
+# slowest tests.
+unit_bound=
+
 # judge_store_tests says each listed test of one package as `go test -v` reported it, and counts in $missing each
 # that did not pass: $1 the package, $2 go test's exit status, $3 the file holding its output, then the names.
 # A result is found anywhere in a line, not only at its start: when a test's last output leaves a line open,
@@ -36,6 +52,44 @@ judge_store_tests() {
 		missing=$((missing + 1))
 	fi
 }
+# judge_unit_run says the unit level as `go test -v` reported it and fails when it was over the bound: $1 the file
+# holding the output of the untagged `go test -v` of every package (packages one after another, each ending in its
+# `ok <package> <time>` line), $2 the bound in seconds. Packages run side by side, so the level's time is that of its
+# slowest package; the five slowest tests of the run are named when it is over, so that the bound is kept by
+# making them cheap or by tagging them `integration`, and never by raising it (issue 895).
+judge_unit_run() {
+	out=$1 bound=$2
+	awk '
+		/^--- (PASS|FAIL|SKIP): / {
+			name = $3; took = $4; gsub(/[()s]/, "", took)
+			pending[++n] = took "\t" name
+			if ($2 == "SKIP:") skipped++
+			next
+		}
+		/^(ok|FAIL)[ \t]/ && NF >= 3 && $3 ~ /^[0-9.]+s$/ {
+			t = $3; sub(/s$/, "", t)
+			for (i = 1; i <= n; i++) print "T\t" pending[i] "\t" $2
+			n = 0
+			print "P\t" t "\t" $2
+		}
+		END { print "S\t" skipped + 0 }
+	' "$out" >"$out.judged"
+	tests=$(grep -c '^T' "$out.judged" || true)
+	skipped=$(awk -F'\t' '$1 == "S" {print $2}' "$out.judged")
+	level=$(awk -F'\t' '$1 == "P" && $2 + 0 > m {m = $2 + 0} END {printf "%.2f", m}' "$out.judged")
+	if [ -z "$bound" ]; then
+		echo "unit level: $tests tests ($skipped skipped), $level s (no bound enforced yet)"
+		return 0
+	fi
+	echo "unit level: $tests tests ($skipped skipped), $level s (bound $bound s)"
+	if awk -v t="$level" -v b="$bound" 'BEGIN {exit !(t + 0 > b + 0)}'; then
+		echo "the unit level took $level s, over its bound of $bound s; the slowest tests:"
+		awk -F'\t' '$1 == "T" {print $2 "\t" $4 "\t" $3}' "$out.judged" | sort -t"$(printf '\t')" -k1,1 -nr | head -n 5 |
+			awk -F'\t' '{printf "  %s s  %s  %s\n", $1, $3, $2}'
+		echo "make them cheap (no wait, no process), or give their files //go:build integration if they start a bus, a store or another process"
+		return 1
+	fi
+}
 # Every Go module of its own below the root, vetted and tested (novox/hq ADR 0420 §5, issue 1025): `./...` stops at a
 # nested go.mod. Its own file, check-nested-modules.sh, which takes $cgo and $race from here.
 nested_modules() {
@@ -43,6 +97,12 @@ nested_modules() {
 }
 # `sh merge-check.sh judge-store-tests <package> <status> <output> <name>…` runs only the judging, to try it on a
 # recorded `go test -v` output.
+# `sh merge-check.sh judge-unit-run <output> <bound>` runs only the judging of the unit level, on a recorded output.
+if [ "${1:-}" = judge-unit-run ]; then
+	shift
+	judge_unit_run "$@"
+	exit
+fi
 if [ "${1:-}" = judge-store-tests ]; then
 	shift
 	missing=0
@@ -61,6 +121,8 @@ if [ -n "$unformatted" ]; then
 	exit 1
 fi
 CGO_ENABLED=0 go vet ./...
+# The files tagged `integration` are vetted too, or nothing would read them.
+CGO_ENABLED=0 go vet -tags integration ./...
 
 # The race detector when the toolchain has a C compiler, and said when it has not.
 if command -v gcc >/dev/null 2>&1; then
@@ -78,7 +140,7 @@ suite() {
 # handed neither and always runs whole; a part that passes is said as a part, never as the check.
 if [ -n "${MESH_CHECK_TESTS:-}" ]; then
 	echo "TRIED IN PART: only the tests matching $MESH_CHECK_TESTS, in every package"
-	suite -run "$MESH_CHECK_TESTS" ./...
+	suite -tags integration -run "$MESH_CHECK_TESTS" ./...
 	exit 0
 fi
 
@@ -107,7 +169,7 @@ shard_package() {
 	bin="$work/$key.test"
 	status=0
 	# shellcheck disable=SC2086 # $race is one flag or none
-	CGO_ENABLED=$cgo go test $race -c -o "$bin" "$dir" >"$work/$key.build" 2>&1 || status=$?
+	CGO_ENABLED=$cgo go test $race -tags integration -c -o "$bin" "$dir" >"$work/$key.build" 2>&1 || status=$?
 	# Every test, example and fuzz target `go test` would run, and nothing a TestMain might print beside them. A
 	# binary that cannot list them, or lists none, fails here rather than pass by running nothing.
 	if [ "$status" -eq 0 ]; then
@@ -213,7 +275,7 @@ if [ -n "$listed" ] && [ -n "${MESH_TEST_POSTGRES:-}" ]; then
 		pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$"
 		(
 			status=0
-			CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$work/store.$key.out" 2>&1 || status=$?
+			CGO_ENABLED=0 go test -tags integration -count=1 -v -run "$pattern" "$pkg" >"$work/store.$key.out" 2>&1 || status=$?
 			echo "$status" >"$work/store.$key.status"
 		) &
 	done
@@ -224,23 +286,42 @@ if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then
 	echo "TRIED IN PART: only the store tests listed in testdata/store-tests"
 	wait
 else
+	# The unit level first and alone, so that nothing else loads the machine while it is timed: every package
+	# untagged, without the race detector (the bound is the plain `go test ./...` a developer runs).
+	status=0
+	CGO_ENABLED=0 go test -count=1 -v ./... >"$work/unit.out" 2>&1 || status=$?
+	if [ "$status" -ne 0 ]; then
+		grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/unit.out"
+		echo "the unit level failed (go test exited $status)"
+		suite_failed=1
+	fi
+	judge_unit_run "$work/unit.out" "$unit_bound" || suite_failed=1
+	# The integration level: the whole suite with the tag, the slowest packages in shards, the race detector when
+	# there is a C compiler. It runs the unit tests again, raced. Said with its test count and its time.
+	integration_start=$(date +%s)
 	module=$(go list -m)
 	skip=$(printf '%s\n' "$sharded" | awk -v m="$module" '{sub(/^\.\//, "", $1); print m "/" $1}')
 	rest=$(go list ./... | grep -vxF "$skip")
 	(
 		status=0
 		# shellcheck disable=SC2086 # the packages are one word each, split on purpose
-		suite $rest >"$work/rest.out" 2>&1 || status=$?
+		suite -tags integration -v $rest >"$work/rest.out" 2>&1 || status=$?
 		echo "$status" >"$work/rest.status"
 	) &
 	for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do
 		shard_package "${entry%%|*}" "${entry#*|}" &
 	done
 	wait
-	cat "$work/rest.out"
+	grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/rest.out" || true
 	[ "$(cat "$work/rest.status")" -eq 0 ] || suite_failed=1
 	say_shards || suite_failed=1
+	shard_tests=$(cat "$work"/*.status 2>/dev/null | awk 'NF == 3 {n += $3} END {print n + 0}')
+	rest_tests=$(grep -cE '^--- (PASS|FAIL|SKIP): ' "$work/rest.out" || true)
+	rest_skipped=$(grep -cE '^--- SKIP: ' "$work/rest.out" || true)
+	echo "integration level (the whole suite, tag integration): $((rest_tests + shard_tests)) tests ($rest_skipped skipped outside the shards), $(($(date +%s) - integration_start)) s"
+	nested_start=$(date +%s)
 	nested_modules || suite_failed=1
+	echo "nested modules: $(($(date +%s) - nested_start)) s"
 fi
 
 if [ -z "$listed" ]; then

testdata/store-tests

@@ -1,6 +1,7 @@
 # The tests that need the throwaway store and must have run against it (novox/hq issue 459): merge-check.sh runs
 # each one named here, by package, and fails unless every one passed — a test that needs the store skips without
 # one, and a skip passes unseen. One per line: <package> <TestName>. Lines starting with # are said nothing of.
+# They run with `-tags integration`: a test file of the integration level (issue 895) is not built without it.
 ./cmd/mesh-controller TestAStopWithoutAMergeWalksTheOthersAgain
 ./cmd/mesh-controller TestAStopWithoutAMergeEndsTheMergesThatContainIt
 ./cmd/mesh-controller TestAStopWalksTheOthersAgainInTheOpenBatch