jschoubben
8db72edd3c
mesh/merge-gate head 8db72edd3c69 merged into trunk d5d54498358e — pass: the change touches no module of the mesh's graph
mesh/repo-check head 8db72edd3c69 merged into trunk d5d54498358e — pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check.sh): main's version judged it; the ch…
mesh/delivery delivered
349 lines
16 KiB
Bash
Executable File
349 lines
16 KiB
Bash
Executable File
#!/bin/sh
|
|
# mesh-check-toolchain: go
|
|
#
|
|
# The controller's own check (novox/hq ADR 0237 as amended): the second layer of a pull request's merge
|
|
# check, `mesh/repo-check`, run by the build seat in the mesh's Go toolchain — and by hand:
|
|
# `MESH_TEST_POSTGRES=… sh merge-check.sh`.
|
|
#
|
|
# The gate — every machine composed with this change's controller, which judges itself — is the build
|
|
# seat's first layer (`mesh/merge-gate`), run because the mesh's module graph builds `mesh-controller`
|
|
# from this repository; it is not repeated here. This is the code's own quality and its suite:
|
|
#
|
|
# 1. formatted and vetted, with and without the build tag `integration`;
|
|
# 2. the unit level: `go test ./...` untagged and without the race detector, timed and held to a bound
|
|
# (unit_bound below): a test with no database, bus, running service or other process (novox/hq ADR 0391,
|
|
# how-we-build section 5). With a bound set, over it the check fails and names the slowest tests (issue 895);
|
|
# 3. the integration level: the whole suite with `-tags integration`, packages in parallel, against the
|
|
# throwaway store the build seat raised at the release the mesh runs; every live test on a bus of its own
|
|
# (internal/testbus), so nothing is serialised. Under the race detector when the toolchain has a C
|
|
# compiler, and said when it has not.
|
|
#
|
|
# **A test that starts a bus, a store or another process, or composes the whole mesh, carries
|
|
# `//go:build integration`** as the first line of its file; a file that held both kinds is two files
|
|
# (`<name>_integration_test.go` beside `<name>_test.go`). Each level says its test count and its time.
|
|
#
|
|
# Exit 0 is a pass; anything else fails `mesh/repo-check` with the last line printed.
|
|
set -eu
|
|
export GOFLAGS=-mod=vendor GOPROXY=off
|
|
|
|
# **The unit level's bound, in seconds** (issue 895): the slowest package's own test time of the untagged run, the
|
|
# compile not counted. It is NOT enforced yet: while this script is main's it judges the pull requests that put the
|
|
# `integration` tag on the tests, and on main the untagged run still holds them (about a minute), so the level's time
|
|
# is said and not judged. The release that tags the tests sets this to a number of seconds (and then to 1, how-we-build
|
|
# section 5: the whole unit suite under a second); with a number set, a run over it fails the check and names the
|
|
# slowest tests.
|
|
unit_bound=
|
|
|
|
# judge_store_tests says each listed test of one package as `go test -v` reported it, and counts in $missing each
|
|
# that did not pass: $1 the package, $2 go test's exit status, $3 the file holding its output, then the names.
|
|
# A result is found anywhere in a line, not only at its start: when a test's last output leaves a line open,
|
|
# `go test -v` writes `--- PASS: <name> (…)` on that same line (novox/hq issue 515). A package whose go test
|
|
# exits non-zero fails even when each listed test said PASS — a TestMain or a leak checker can fail after them.
|
|
judge_store_tests() {
|
|
pkg=$1 status=$2 out=$3
|
|
shift 3
|
|
for name in "$@"; do
|
|
line=$(grep -oE -- "--- (PASS|FAIL|SKIP): $name \([^)]*\)" "$out" | tail -n 1 || true)
|
|
echo " $pkg ${line:-"--- NOT RUN: $name"}"
|
|
case "$line" in "--- PASS: "*) ;; *) missing=$((missing + 1)) ;; esac
|
|
done
|
|
if [ "$status" -ne 0 ]; then
|
|
echo " $pkg FAIL: go test exited $status"
|
|
missing=$((missing + 1))
|
|
fi
|
|
}
|
|
# judge_unit_run says the unit level as `go test -v` reported it and fails when it was over the bound: $1 the file
|
|
# holding the output of the untagged `go test -v` of every package (packages one after another, each ending in its
|
|
# `ok <package> <time>` line), $2 the bound in seconds. Packages run side by side, so the level's time is that of its
|
|
# slowest package; the five slowest tests of the run are named when it is over, so that the bound is kept by
|
|
# making them cheap or by tagging them `integration`, and never by raising it (issue 895).
|
|
judge_unit_run() {
|
|
out=$1 bound=$2
|
|
awk '
|
|
/^--- (PASS|FAIL|SKIP): / {
|
|
name = $3; took = $4; gsub(/[()s]/, "", took)
|
|
pending[++n] = took "\t" name
|
|
if ($2 == "SKIP:") skipped++
|
|
next
|
|
}
|
|
/^(ok|FAIL)[ \t]/ && NF >= 3 && $3 ~ /^[0-9.]+s$/ {
|
|
t = $3; sub(/s$/, "", t)
|
|
for (i = 1; i <= n; i++) print "T\t" pending[i] "\t" $2
|
|
n = 0
|
|
print "P\t" t "\t" $2
|
|
}
|
|
END { print "S\t" skipped + 0 }
|
|
' "$out" >"$out.judged"
|
|
tests=$(grep -c '^T' "$out.judged" || true)
|
|
skipped=$(awk -F'\t' '$1 == "S" {print $2}' "$out.judged")
|
|
level=$(awk -F'\t' '$1 == "P" && $2 + 0 > m {m = $2 + 0} END {printf "%.2f", m}' "$out.judged")
|
|
if [ -z "$bound" ]; then
|
|
echo "unit level: $tests tests ($skipped skipped), $level s (no bound enforced yet)"
|
|
return 0
|
|
fi
|
|
echo "unit level: $tests tests ($skipped skipped), $level s (bound $bound s)"
|
|
if awk -v t="$level" -v b="$bound" 'BEGIN {exit !(t + 0 > b + 0)}'; then
|
|
echo "the unit level took $level s, over its bound of $bound s; the slowest tests:"
|
|
awk -F'\t' '$1 == "T" {print $2 "\t" $4 "\t" $3}' "$out.judged" | sort -t"$(printf '\t')" -k1,1 -nr | head -n 5 |
|
|
awk -F'\t' '{printf " %s s %s %s\n", $1, $3, $2}'
|
|
echo "make them cheap (no wait, no process), or give their files //go:build integration if they start a bus, a store or another process"
|
|
return 1
|
|
fi
|
|
}
|
|
# Every Go module of its own below the root, vetted and tested (novox/hq ADR 0420 §5, issue 1025): `./...` stops at a
|
|
# nested go.mod. Its own file, check-nested-modules.sh, which takes $cgo and $race from here.
|
|
nested_modules() {
|
|
cgo="$cgo" race="$race" sh ./check-nested-modules.sh
|
|
}
|
|
# `sh merge-check.sh judge-store-tests <package> <status> <output> <name>…` runs only the judging, to try it on a
|
|
# recorded `go test -v` output.
|
|
# `sh merge-check.sh judge-unit-run <output> <bound>` runs only the judging of the unit level, on a recorded output.
|
|
if [ "${1:-}" = judge-unit-run ]; then
|
|
shift
|
|
judge_unit_run "$@"
|
|
exit
|
|
fi
|
|
if [ "${1:-}" = judge-store-tests ]; then
|
|
shift
|
|
missing=0
|
|
judge_store_tests "$@"
|
|
[ "$missing" -eq 0 ]
|
|
exit
|
|
fi
|
|
|
|
gofmt_dirs="cmd internal examples"
|
|
[ -d modules ] && gofmt_dirs="$gofmt_dirs modules"
|
|
# shellcheck disable=SC2086 # the directories are one word each, split on purpose
|
|
unformatted=$(gofmt -l $gofmt_dirs)
|
|
if [ -n "$unformatted" ]; then
|
|
echo "not gofmt'd:"
|
|
echo "$unformatted"
|
|
exit 1
|
|
fi
|
|
CGO_ENABLED=0 go vet ./...
|
|
# The files tagged `integration` are vetted too, or nothing would read them.
|
|
CGO_ENABLED=0 go vet -tags integration ./...
|
|
|
|
# The race detector when the toolchain has a C compiler, and said when it has not.
|
|
if command -v gcc >/dev/null 2>&1; then
|
|
cgo=1 race=-race
|
|
else
|
|
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
|
|
cgo=0 race=
|
|
fi
|
|
suite() {
|
|
# shellcheck disable=SC2086 # $race is one flag or none
|
|
CGO_ENABLED=$cgo go test $race -count=1 -timeout 30m "$@"
|
|
}
|
|
# **A branch tried before its pull request may ask for a part** (novox/hq issue 686): the controller's `try` verb
|
|
# hands MESH_CHECK_TESTS, a `go test -run` pattern, or MESH_CHECK_STORE_ONLY=1. A pull request's merge check is
|
|
# handed neither and always runs whole; a part that passes is said as a part, never as the check.
|
|
if [ -n "${MESH_CHECK_TESTS:-}" ]; then
|
|
echo "TRIED IN PART: only the tests matching $MESH_CHECK_TESTS, in every package"
|
|
suite -tags integration -run "$MESH_CHECK_TESTS" ./...
|
|
exit 0
|
|
fi
|
|
|
|
# **The slowest packages run in shards** (novox/hq issue 717). `go test` runs packages beside each other but the
|
|
# tests of one package one after another, in one process, and these packages' tests wait on the store, a bus, a
|
|
# build or a clock: cmd/mesh-controller alone took 3 to 5 minutes of a check whose other packages end in seconds.
|
|
# Each is compiled once, its tests are dealt round-robin into that many processes, and those run beside each other
|
|
# and beside the rest of the suite. Each process is the package's own test binary, run from the package's
|
|
# directory through its TestMain as `go test` runs it, so a test cannot tell, and every test runs exactly once, in
|
|
# one of them. Tests of one package therefore keep what packages run side by side already keep: nothing shared
|
|
# outside their process — no fixed port, file or database name. A package is added here by measuring it
|
|
# (`go test -json` gives each test's time), never to hide a slow test that could wait less.
|
|
sharded='./cmd/mesh-controller 8
|
|
./internal/inventory 4
|
|
./internal/builder 4
|
|
./internal/link 4'
|
|
|
|
work=$(mktemp -d)
|
|
trap 'rm -rf "$work"' EXIT
|
|
|
|
# shard_package compiles one package's tests and runs them in $2 processes, each process's output and
|
|
# "<exit status> <seconds>" left in $work: $1 the package's directory, $2 the number of processes.
|
|
shard_package() {
|
|
dir=$1 shards=$2
|
|
key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _)
|
|
bin="$work/$key.test"
|
|
status=0
|
|
# shellcheck disable=SC2086 # $race is one flag or none
|
|
CGO_ENABLED=$cgo go test $race -tags integration -c -o "$bin" "$dir" >"$work/$key.build" 2>&1 || status=$?
|
|
# Every test, example and fuzz target `go test` would run, and nothing a TestMain might print beside them. A
|
|
# binary that cannot list them, or lists none, fails here rather than pass by running nothing.
|
|
if [ "$status" -eq 0 ]; then
|
|
(cd "$dir" && "$bin" -test.list '.*') >"$work/$key.list" 2>&1 || status=$?
|
|
names=$(grep -E '^(Test|Example|Fuzz)' "$work/$key.list" || true)
|
|
if [ "$status" -ne 0 ] || [ -z "$names" ]; then
|
|
{
|
|
cat "$work/$key.list"
|
|
echo "the test binary of $dir listed no tests to deal into shards (exit status $status)"
|
|
} >>"$work/$key.build"
|
|
status=1
|
|
fi
|
|
fi
|
|
echo "$status" >"$work/$key.built"
|
|
[ "$status" -eq 0 ] || return 0
|
|
i=0
|
|
while [ "$i" -lt "$shards" ]; do
|
|
pattern=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | paste -sd '|' -)
|
|
count=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | wc -l)
|
|
if [ -n "$pattern" ]; then
|
|
(
|
|
start=$(date +%s)
|
|
status=0
|
|
cd "$dir" && "$bin" -test.count=1 -test.timeout=30m -test.paniconexit0 -test.run "^($pattern)\$" \
|
|
>"$work/$key.$i.out" 2>&1 || status=$?
|
|
echo "$status $(($(date +%s) - start)) $count" >"$work/$key.$i.status"
|
|
) &
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
wait
|
|
}
|
|
|
|
# say_shards prints how each sharded package's processes ended, as `go test` says a package, and the output of
|
|
# each that failed; it fails when any did.
|
|
say_shards() {
|
|
failed=0
|
|
module=$(go list -m)
|
|
for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do
|
|
dir=${entry%%|*} shards=${entry#*|}
|
|
key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _)
|
|
pkg="$module/${dir#./}"
|
|
if [ "$(cat "$work/$key.built" 2>/dev/null || echo 1)" -ne 0 ]; then
|
|
cat "$work/$key.build"
|
|
printf 'FAIL\t%s [build failed]\n' "$pkg"
|
|
failed=1
|
|
continue
|
|
fi
|
|
i=0
|
|
while [ "$i" -lt "$shards" ]; do
|
|
if [ -f "$work/$key.$i.status" ]; then
|
|
read -r status took count <"$work/$key.$i.status"
|
|
if [ "$status" -eq 0 ]; then
|
|
printf 'ok \t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count"
|
|
else
|
|
cat "$work/$key.$i.out"
|
|
printf 'FAIL\t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count"
|
|
failed=1
|
|
fi
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
done
|
|
return "$failed"
|
|
}
|
|
|
|
# **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip
|
|
# passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against
|
|
# it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list.
|
|
listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true)
|
|
# Each line is exactly `<package> <TestName>`: a line of one field ran nothing and said nothing (novox/hq issue 515).
|
|
malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true)
|
|
if [ -n "$malformed" ]; then
|
|
echo "testdata/store-tests: a line is not <package> <TestName>:"
|
|
echo "$malformed"
|
|
exit 1
|
|
fi
|
|
# **A change does not shorten the list it is judged by** (novox/hq issue 515): the list is in the change's own tree,
|
|
# so an entry main lists stays listed while its test function exists; removing the function removes the entry.
|
|
if ! git rev-parse --verify --quiet origin/main^{commit} >/dev/null; then
|
|
echo "origin/main cannot be read, and its testdata/store-tests says which store tests must stay listed"
|
|
exit 1
|
|
fi
|
|
dropped=0
|
|
for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v '^[[:space:]]*\(#\|$\)' |
|
|
awk 'NF == 2 {print $1 "|" $2}'); do
|
|
pkg=${entry%%|*} name=${entry#*|}
|
|
printf '%s\n' "$listed" | awk -v p="$pkg" -v n="$name" '$1 == p && $2 == n {f = 1} END {exit !f}' && continue
|
|
if cat "$pkg"/*_test.go 2>/dev/null | grep -q "^func $name("; then
|
|
echo "testdata/store-tests: $pkg $name is listed on main and its test still exists, but the change drops it"
|
|
dropped=$((dropped + 1))
|
|
fi
|
|
done
|
|
[ "$dropped" -eq 0 ] || exit 1
|
|
|
|
# The listed store tests run beside the suite, one process per package (novox/hq issue 717): each package's output
|
|
# and exit status are left in $work and judged once the suite is done.
|
|
store_packages=$(printf '%s\n' "$listed" | awk 'NF {print $1}' | sort -u)
|
|
if [ -n "$listed" ] && [ -n "${MESH_TEST_POSTGRES:-}" ]; then
|
|
for pkg in $store_packages; do
|
|
key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _)
|
|
names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
|
|
pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$"
|
|
(
|
|
status=0
|
|
CGO_ENABLED=0 go test -tags integration -count=1 -v -run "$pattern" "$pkg" >"$work/store.$key.out" 2>&1 || status=$?
|
|
echo "$status" >"$work/store.$key.status"
|
|
) &
|
|
done
|
|
fi
|
|
|
|
suite_failed=0
|
|
if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then
|
|
echo "TRIED IN PART: only the store tests listed in testdata/store-tests"
|
|
wait
|
|
else
|
|
# The unit level first and alone, so that nothing else loads the machine while it is timed: every package
|
|
# untagged, without the race detector (the bound is the plain `go test ./...` a developer runs).
|
|
status=0
|
|
CGO_ENABLED=0 go test -count=1 -v ./... >"$work/unit.out" 2>&1 || status=$?
|
|
if [ "$status" -ne 0 ]; then
|
|
grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/unit.out"
|
|
echo "the unit level failed (go test exited $status)"
|
|
suite_failed=1
|
|
fi
|
|
judge_unit_run "$work/unit.out" "$unit_bound" || suite_failed=1
|
|
# The integration level: the whole suite with the tag, the slowest packages in shards, the race detector when
|
|
# there is a C compiler. It runs the unit tests again, raced. Said with its test count and its time.
|
|
integration_start=$(date +%s)
|
|
module=$(go list -m)
|
|
skip=$(printf '%s\n' "$sharded" | awk -v m="$module" '{sub(/^\.\//, "", $1); print m "/" $1}')
|
|
rest=$(go list ./... | grep -vxF "$skip")
|
|
(
|
|
status=0
|
|
# shellcheck disable=SC2086 # the packages are one word each, split on purpose
|
|
suite -tags integration -v $rest >"$work/rest.out" 2>&1 || status=$?
|
|
echo "$status" >"$work/rest.status"
|
|
) &
|
|
for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do
|
|
shard_package "${entry%%|*}" "${entry#*|}" &
|
|
done
|
|
wait
|
|
grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/rest.out" || true
|
|
[ "$(cat "$work/rest.status")" -eq 0 ] || suite_failed=1
|
|
say_shards || suite_failed=1
|
|
shard_tests=$(cat "$work"/*.status 2>/dev/null | awk 'NF == 3 {n += $3} END {print n + 0}')
|
|
rest_tests=$(grep -cE '^--- (PASS|FAIL|SKIP): ' "$work/rest.out" || true)
|
|
rest_skipped=$(grep -cE '^--- SKIP: ' "$work/rest.out" || true)
|
|
echo "integration level (the whole suite, tag integration): $((rest_tests + shard_tests)) tests ($rest_skipped skipped outside the shards), $(($(date +%s) - integration_start)) s"
|
|
nested_start=$(date +%s)
|
|
nested_modules || suite_failed=1
|
|
echo "nested modules: $(($(date +%s) - nested_start)) s"
|
|
fi
|
|
|
|
if [ -z "$listed" ]; then
|
|
echo "store tests: none listed in testdata/store-tests"
|
|
elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then
|
|
echo "store tests, each run against the store:"
|
|
missing=0
|
|
for pkg in $store_packages; do
|
|
key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _)
|
|
names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
|
|
# shellcheck disable=SC2086 # the names are one word each, split on purpose
|
|
judge_store_tests "$pkg" "$(cat "$work/store.$key.status")" "$work/store.$key.out" $names
|
|
done
|
|
if [ "$missing" -ne 0 ]; then
|
|
echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so the store tests listed in testdata/store-tests skipped"
|
|
fi
|
|
if [ "$suite_failed" -ne 0 ]; then
|
|
echo "the suite failed: each package or shard that failed is said above with its output"
|
|
exit 1
|
|
fi
|