Merge pull request 'The store-tests step fails on a malformed line, a failed go test and a shortened list (issue 515)' (#238) from fix/515-store-tests-step into main

merge-check.sh

@@ -18,6 +18,34 @@
 set -eu
 export GOFLAGS=-mod=vendor GOPROXY=off
 
+# judge_store_tests says each listed test of one package as `go test -v` reported it, and counts in $missing each
+# that did not pass: $1 the package, $2 go test's exit status, $3 the file holding its output, then the names.
+# A result is found anywhere in a line, not only at its start: when a test's last output leaves a line open,
+# `go test -v` writes `--- PASS: <name> (…)` on that same line (novox/hq issue 515). A package whose go test
+# exits non-zero fails even when each listed test said PASS — a TestMain or a leak checker can fail after them.
+judge_store_tests() {
+	pkg=$1 status=$2 out=$3
+	shift 3
+	for name in "$@"; do
+		line=$(grep -oE -- "--- (PASS|FAIL|SKIP): $name \([^)]*\)" "$out" | tail -n 1 || true)
+		echo "  $pkg ${line:-"--- NOT RUN: $name"}"
+		case "$line" in "--- PASS: "*) ;; *) missing=$((missing + 1)) ;; esac
+	done
+	if [ "$status" -ne 0 ]; then
+		echo "  $pkg FAIL: go test exited $status"
+		missing=$((missing + 1))
+	fi
+}
+# `sh merge-check.sh judge-store-tests <package> <status> <output> <name>…` runs only the judging, to try it on a
+# recorded `go test -v` output.
+if [ "${1:-}" = judge-store-tests ]; then
+	shift
+	missing=0
+	judge_store_tests "$@"
+	[ "$missing" -eq 0 ]
+	exit
+fi
+
 unformatted=$(gofmt -l cmd internal examples)
 if [ -n "$unformatted" ]; then
 	echo "not gofmt'd:"
@@ -36,21 +64,48 @@ fi
 # **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip
 # passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against
 # it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list.
+
+
 listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true)
+# Each line is exactly `<package> <TestName>`: a line of one field ran nothing and said nothing (novox/hq issue 515).
+malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true)
+if [ -n "$malformed" ]; then
+	echo "testdata/store-tests: a line is not <package> <TestName>:"
+	echo "$malformed"
+	exit 1
+fi
+# **A change does not shorten the list it is judged by** (novox/hq issue 515): the list is in the change's own tree,
+# so an entry main lists stays listed while its test function exists; removing the function removes the entry.
+if ! git rev-parse --verify --quiet origin/main^{commit} >/dev/null; then
+	echo "origin/main cannot be read, and its testdata/store-tests says which store tests must stay listed"
+	exit 1
+fi
+dropped=0
+for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v '^[[:space:]]*\(#\|$\)' |
+	awk 'NF == 2 {print $1 "|" $2}'); do
+	pkg=${entry%%|*} name=${entry#*|}
+	printf '%s\n' "$listed" | awk -v p="$pkg" -v n="$name" '$1 == p && $2 == n {f = 1} END {exit !f}' && continue
+	if cat "$pkg"/*_test.go 2>/dev/null | grep -q "^func $name("; then
+		echo "testdata/store-tests: $pkg $name is listed on main and its test still exists, but the change drops it"
+		dropped=$((dropped + 1))
+	fi
+done
+[ "$dropped" -eq 0 ] || exit 1
+
 if [ -z "$listed" ]; then
 	echo "store tests: none listed in testdata/store-tests"
 elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then
 	echo "store tests, each run against the store:"
 	missing=0
+	out=$(mktemp)
+	trap 'rm -f "$out"' EXIT
 	for pkg in $(printf '%s\n' "$listed" | awk '{print $1}' | sort -u); do
 		names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
 		pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$"
-		ran=$(CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" 2>&1 | grep -E '^--- (PASS|FAIL|SKIP)' || true)
+		status=0
-		for name in $names; do
+		CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$out" 2>&1 || status=$?
-			line=$(printf '%s\n' "$ran" | grep -E "^--- [A-Z]+: $name " || true)
+		# shellcheck disable=SC2086 # the names are one word each, split on purpose
-			echo "  $pkg ${line:-"--- NOT RUN: $name"}"
+		judge_store_tests "$pkg" "$status" "$out" $names
-			case "$line" in "--- PASS: "*) ;; *) missing=$((missing + 1)) ;; esac
-		done
 	done
 	if [ "$missing" -ne 0 ]; then
 		echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store"