mesh-tools
157d1599b4
The seats test I moved to internal/ownmanifest parsed each catalogue manifest alone, so claude-code's managed_settings and dunst's font-size, which fall back to a predefined key, were refused where the check clones the catalogue beside (ADR 0348 section 7). It reads them with ReadTreeManifest from the catalogue's root, as the tests that issue 914 fixed do. merge-check.sh differed from main in a comment only; it is main's again.
349 lines
16 KiB
Bash
Executable File
349 lines
16 KiB
Bash
Executable File
#!/bin/sh
|
|
# mesh-check-toolchain: go
|
|
#
|
|
# The controller's own check (novox/hq ADR 0237 as amended): the second layer of a pull request's merge
|
|
# check, `mesh/repo-check`, run by the build seat in the mesh's Go toolchain — and by hand:
|
|
# `MESH_TEST_POSTGRES=… sh merge-check.sh`.
|
|
#
|
|
# The gate — every machine composed with this change's controller, which judges itself — is the build
|
|
# seat's first layer (`mesh/merge-gate`), run because the mesh's module graph builds `mesh-controller`
|
|
# from this repository; it is not repeated here. This is the code's own quality and its suite:
|
|
#
|
|
# 1. formatted and vetted, with and without the build tag `integration`;
|
|
# 2. the unit level: `go test ./...` untagged and without the race detector, timed and held to a bound
|
|
# (unit_bound below): a test with no database, bus, running service or other process (novox/hq ADR 0391,
|
|
# how-we-build section 5). With a bound set, over it the check fails and names the slowest tests (issue 895);
|
|
# 3. the integration level: the whole suite with `-tags integration`, packages in parallel, against the
|
|
# throwaway store the build seat raised at the release the mesh runs; every live test on a bus of its own
|
|
# (internal/testbus), so nothing is serialised. Under the race detector when the toolchain has a C
|
|
# compiler, and said when it has not.
|
|
#
|
|
# **A test that starts a bus, a store or another process, or composes the whole mesh, carries
|
|
# `//go:build integration`** as the first line of its file; a file that held both kinds is two files
|
|
# (`<name>_integration_test.go` beside `<name>_test.go`). Each level says its test count and its time.
|
|
#
|
|
# Exit 0 is a pass; anything else fails `mesh/repo-check` with the last line printed.
|
|
set -eu
|
|
export GOFLAGS=-mod=vendor GOPROXY=off
|
|
|
|
# **The unit level's bound, in seconds** (issue 895): the slowest package's own test time of the untagged run, the
|
|
# compile not counted. A run over it fails the check and names the slowest tests. It is 5 s since the tests carry the
|
|
# `integration` tag (design 47 milestone 3): the slowest packages, cmd/mesh-controller and internal/catalogue, measured
|
|
# 2.6-2.8 s on a quiet machine in about 2000 small tests each, and the build node runs the store tests beside this run
|
|
# and other work besides, so twice the measure. How-we-build section 5 asks 1 s for the whole unit suite: issue 1055
|
|
# (a catalogue loaded once per package run, fewer source-scanning tests, the timed run started alone).
|
|
unit_bound=5
|
|
|
|
# judge_store_tests says each listed test of one package as `go test -v` reported it, and counts in $missing each
|
|
# that did not pass: $1 the package, $2 go test's exit status, $3 the file holding its output, then the names.
|
|
# A result is found anywhere in a line, not only at its start: when a test's last output leaves a line open,
|
|
# `go test -v` writes `--- PASS: <name> (…)` on that same line (novox/hq issue 515). A package whose go test
|
|
# exits non-zero fails even when each listed test said PASS — a TestMain or a leak checker can fail after them.
|
|
judge_store_tests() {
|
|
pkg=$1 status=$2 out=$3
|
|
shift 3
|
|
for name in "$@"; do
|
|
line=$(grep -oE -- "--- (PASS|FAIL|SKIP): $name \([^)]*\)" "$out" | tail -n 1 || true)
|
|
echo " $pkg ${line:-"--- NOT RUN: $name"}"
|
|
case "$line" in "--- PASS: "*) ;; *) missing=$((missing + 1)) ;; esac
|
|
done
|
|
if [ "$status" -ne 0 ]; then
|
|
echo " $pkg FAIL: go test exited $status"
|
|
missing=$((missing + 1))
|
|
fi
|
|
}
|
|
# judge_unit_run says the unit level as `go test -v` reported it and fails when it was over the bound: $1 the file
|
|
# holding the output of the untagged `go test -v` of every package (packages one after another, each ending in its
|
|
# `ok <package> <time>` line), $2 the bound in seconds. Packages run side by side, so the level's time is that of its
|
|
# slowest package; the five slowest tests of the run are named when it is over, so that the bound is kept by
|
|
# making them cheap or by tagging them `integration`, and never by raising it (issue 895).
|
|
judge_unit_run() {
|
|
out=$1 bound=$2
|
|
awk '
|
|
/^--- (PASS|FAIL|SKIP): / {
|
|
name = $3; took = $4; gsub(/[()s]/, "", took)
|
|
pending[++n] = took "\t" name
|
|
if ($2 == "SKIP:") skipped++
|
|
next
|
|
}
|
|
/^(ok|FAIL)[ \t]/ && NF >= 3 && $3 ~ /^[0-9.]+s$/ {
|
|
t = $3; sub(/s$/, "", t)
|
|
for (i = 1; i <= n; i++) print "T\t" pending[i] "\t" $2
|
|
n = 0
|
|
print "P\t" t "\t" $2
|
|
}
|
|
END { print "S\t" skipped + 0 }
|
|
' "$out" >"$out.judged"
|
|
tests=$(grep -c '^T' "$out.judged" || true)
|
|
skipped=$(awk -F'\t' '$1 == "S" {print $2}' "$out.judged")
|
|
level=$(awk -F'\t' '$1 == "P" && $2 + 0 > m {m = $2 + 0} END {printf "%.2f", m}' "$out.judged")
|
|
if [ -z "$bound" ]; then
|
|
echo "unit level: $tests tests ($skipped skipped), $level s (no bound enforced yet)"
|
|
return 0
|
|
fi
|
|
echo "unit level: $tests tests ($skipped skipped), $level s (bound $bound s)"
|
|
if awk -v t="$level" -v b="$bound" 'BEGIN {exit !(t + 0 > b + 0)}'; then
|
|
echo "the unit level took $level s, over its bound of $bound s; the slowest tests:"
|
|
awk -F'\t' '$1 == "T" {print $2 "\t" $4 "\t" $3}' "$out.judged" | sort -t"$(printf '\t')" -k1,1 -nr | head -n 5 |
|
|
awk -F'\t' '{printf " %s s %s %s\n", $1, $3, $2}'
|
|
echo "make them cheap (no wait, no process), or give their files //go:build integration if they start a bus, a store or another process"
|
|
return 1
|
|
fi
|
|
}
|
|
# Every Go module of its own below the root, vetted and tested (novox/hq ADR 0420 §5, issue 1025): `./...` stops at a
|
|
# nested go.mod. Its own file, check-nested-modules.sh, which takes $cgo and $race from here.
|
|
nested_modules() {
|
|
cgo="$cgo" race="$race" sh ./check-nested-modules.sh
|
|
}
|
|
# `sh merge-check.sh judge-store-tests <package> <status> <output> <name>…` runs only the judging, to try it on a
|
|
# recorded `go test -v` output.
|
|
# `sh merge-check.sh judge-unit-run <output> <bound>` runs only the judging of the unit level, on a recorded output.
|
|
if [ "${1:-}" = judge-unit-run ]; then
|
|
shift
|
|
judge_unit_run "$@"
|
|
exit
|
|
fi
|
|
if [ "${1:-}" = judge-store-tests ]; then
|
|
shift
|
|
missing=0
|
|
judge_store_tests "$@"
|
|
[ "$missing" -eq 0 ]
|
|
exit
|
|
fi
|
|
|
|
gofmt_dirs="cmd internal examples"
|
|
[ -d modules ] && gofmt_dirs="$gofmt_dirs modules"
|
|
# shellcheck disable=SC2086 # the directories are one word each, split on purpose
|
|
unformatted=$(gofmt -l $gofmt_dirs)
|
|
if [ -n "$unformatted" ]; then
|
|
echo "not gofmt'd:"
|
|
echo "$unformatted"
|
|
exit 1
|
|
fi
|
|
CGO_ENABLED=0 go vet ./...
|
|
# The files tagged `integration` are vetted too, or nothing would read them.
|
|
CGO_ENABLED=0 go vet -tags integration ./...
|
|
|
|
# The race detector when the toolchain has a C compiler, and said when it has not.
|
|
if command -v gcc >/dev/null 2>&1; then
|
|
cgo=1 race=-race
|
|
else
|
|
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
|
|
cgo=0 race=
|
|
fi
|
|
suite() {
|
|
# shellcheck disable=SC2086 # $race is one flag or none
|
|
CGO_ENABLED=$cgo go test $race -count=1 -timeout 30m "$@"
|
|
}
|
|
# **A branch tried before its pull request may ask for a part** (novox/hq issue 686): the controller's `try` verb
|
|
# hands MESH_CHECK_TESTS, a `go test -run` pattern, or MESH_CHECK_STORE_ONLY=1. A pull request's merge check is
|
|
# handed neither and always runs whole; a part that passes is said as a part, never as the check.
|
|
if [ -n "${MESH_CHECK_TESTS:-}" ]; then
|
|
echo "TRIED IN PART: only the tests matching $MESH_CHECK_TESTS, in every package"
|
|
suite -tags integration -run "$MESH_CHECK_TESTS" ./...
|
|
exit 0
|
|
fi
|
|
|
|
# **The slowest packages run in shards** (novox/hq issue 717). `go test` runs packages beside each other but the
|
|
# tests of one package one after another, in one process, and these packages' tests wait on the store, a bus, a
|
|
# build or a clock: cmd/mesh-controller alone took 3 to 5 minutes of a check whose other packages end in seconds.
|
|
# Each is compiled once, its tests are dealt round-robin into that many processes, and those run beside each other
|
|
# and beside the rest of the suite. Each process is the package's own test binary, run from the package's
|
|
# directory through its TestMain as `go test` runs it, so a test cannot tell, and every test runs exactly once, in
|
|
# one of them. Tests of one package therefore keep what packages run side by side already keep: nothing shared
|
|
# outside their process — no fixed port, file or database name. A package is added here by measuring it
|
|
# (`go test -json` gives each test's time), never to hide a slow test that could wait less.
|
|
sharded='./cmd/mesh-controller 8
|
|
./internal/inventory 4
|
|
./internal/builder 4
|
|
./internal/link 4'
|
|
|
|
work=$(mktemp -d)
|
|
trap 'rm -rf "$work"' EXIT
|
|
|
|
# shard_package compiles one package's tests and runs them in $2 processes, each process's output and
|
|
# "<exit status> <seconds>" left in $work: $1 the package's directory, $2 the number of processes.
|
|
shard_package() {
|
|
dir=$1 shards=$2
|
|
key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _)
|
|
bin="$work/$key.test"
|
|
status=0
|
|
# shellcheck disable=SC2086 # $race is one flag or none
|
|
CGO_ENABLED=$cgo go test $race -tags integration -c -o "$bin" "$dir" >"$work/$key.build" 2>&1 || status=$?
|
|
# Every test, example and fuzz target `go test` would run, and nothing a TestMain might print beside them. A
|
|
# binary that cannot list them, or lists none, fails here rather than pass by running nothing.
|
|
if [ "$status" -eq 0 ]; then
|
|
(cd "$dir" && "$bin" -test.list '.*') >"$work/$key.list" 2>&1 || status=$?
|
|
names=$(grep -E '^(Test|Example|Fuzz)' "$work/$key.list" || true)
|
|
if [ "$status" -ne 0 ] || [ -z "$names" ]; then
|
|
{
|
|
cat "$work/$key.list"
|
|
echo "the test binary of $dir listed no tests to deal into shards (exit status $status)"
|
|
} >>"$work/$key.build"
|
|
status=1
|
|
fi
|
|
fi
|
|
echo "$status" >"$work/$key.built"
|
|
[ "$status" -eq 0 ] || return 0
|
|
i=0
|
|
while [ "$i" -lt "$shards" ]; do
|
|
pattern=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | paste -sd '|' -)
|
|
count=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | wc -l)
|
|
if [ -n "$pattern" ]; then
|
|
(
|
|
start=$(date +%s)
|
|
status=0
|
|
cd "$dir" && "$bin" -test.count=1 -test.timeout=30m -test.paniconexit0 -test.run "^($pattern)\$" \
|
|
>"$work/$key.$i.out" 2>&1 || status=$?
|
|
echo "$status $(($(date +%s) - start)) $count" >"$work/$key.$i.status"
|
|
) &
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
wait
|
|
}
|
|
|
|
# say_shards prints how each sharded package's processes ended, as `go test` says a package, and the output of
|
|
# each that failed; it fails when any did.
|
|
say_shards() {
|
|
failed=0
|
|
module=$(go list -m)
|
|
for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do
|
|
dir=${entry%%|*} shards=${entry#*|}
|
|
key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _)
|
|
pkg="$module/${dir#./}"
|
|
if [ "$(cat "$work/$key.built" 2>/dev/null || echo 1)" -ne 0 ]; then
|
|
cat "$work/$key.build"
|
|
printf 'FAIL\t%s [build failed]\n' "$pkg"
|
|
failed=1
|
|
continue
|
|
fi
|
|
i=0
|
|
while [ "$i" -lt "$shards" ]; do
|
|
if [ -f "$work/$key.$i.status" ]; then
|
|
read -r status took count <"$work/$key.$i.status"
|
|
if [ "$status" -eq 0 ]; then
|
|
printf 'ok \t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count"
|
|
else
|
|
cat "$work/$key.$i.out"
|
|
printf 'FAIL\t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count"
|
|
failed=1
|
|
fi
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
done
|
|
return "$failed"
|
|
}
|
|
|
|
# **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip
|
|
# passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against
|
|
# it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list.
|
|
listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true)
|
|
# Each line is exactly `<package> <TestName>`: a line of one field ran nothing and said nothing (novox/hq issue 515).
|
|
malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true)
|
|
if [ -n "$malformed" ]; then
|
|
echo "testdata/store-tests: a line is not <package> <TestName>:"
|
|
echo "$malformed"
|
|
exit 1
|
|
fi
|
|
# **A change does not shorten the list it is judged by** (novox/hq issue 515): the list is in the change's own tree,
|
|
# so an entry main lists stays listed while its test function exists; removing the function removes the entry.
|
|
if ! git rev-parse --verify --quiet origin/main^{commit} >/dev/null; then
|
|
echo "origin/main cannot be read, and its testdata/store-tests says which store tests must stay listed"
|
|
exit 1
|
|
fi
|
|
dropped=0
|
|
for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v '^[[:space:]]*\(#\|$\)' |
|
|
awk 'NF == 2 {print $1 "|" $2}'); do
|
|
pkg=${entry%%|*} name=${entry#*|}
|
|
printf '%s\n' "$listed" | awk -v p="$pkg" -v n="$name" '$1 == p && $2 == n {f = 1} END {exit !f}' && continue
|
|
if cat "$pkg"/*_test.go 2>/dev/null | grep -q "^func $name("; then
|
|
echo "testdata/store-tests: $pkg $name is listed on main and its test still exists, but the change drops it"
|
|
dropped=$((dropped + 1))
|
|
fi
|
|
done
|
|
[ "$dropped" -eq 0 ] || exit 1
|
|
|
|
# The listed store tests run beside the suite, one process per package (novox/hq issue 717): each package's output
|
|
# and exit status are left in $work and judged once the suite is done.
|
|
store_packages=$(printf '%s\n' "$listed" | awk 'NF {print $1}' | sort -u)
|
|
if [ -n "$listed" ] && [ -n "${MESH_TEST_POSTGRES:-}" ]; then
|
|
for pkg in $store_packages; do
|
|
key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _)
|
|
names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
|
|
pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$"
|
|
(
|
|
status=0
|
|
CGO_ENABLED=0 go test -tags integration -count=1 -v -run "$pattern" "$pkg" >"$work/store.$key.out" 2>&1 || status=$?
|
|
echo "$status" >"$work/store.$key.status"
|
|
) &
|
|
done
|
|
fi
|
|
|
|
suite_failed=0
|
|
if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then
|
|
echo "TRIED IN PART: only the store tests listed in testdata/store-tests"
|
|
wait
|
|
else
|
|
# The unit level first and alone, so that nothing else loads the machine while it is timed: every package
|
|
# untagged, without the race detector (the bound is the plain `go test ./...` a developer runs).
|
|
status=0
|
|
CGO_ENABLED=0 go test -count=1 -v ./... >"$work/unit.out" 2>&1 || status=$?
|
|
if [ "$status" -ne 0 ]; then
|
|
grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/unit.out"
|
|
echo "the unit level failed (go test exited $status)"
|
|
suite_failed=1
|
|
fi
|
|
judge_unit_run "$work/unit.out" "$unit_bound" || suite_failed=1
|
|
# The integration level: the whole suite with the tag, the slowest packages in shards, the race detector when
|
|
# there is a C compiler. It runs the unit tests again, raced. Said with its test count and its time.
|
|
integration_start=$(date +%s)
|
|
module=$(go list -m)
|
|
skip=$(printf '%s\n' "$sharded" | awk -v m="$module" '{sub(/^\.\//, "", $1); print m "/" $1}')
|
|
rest=$(go list ./... | grep -vxF "$skip")
|
|
(
|
|
status=0
|
|
# shellcheck disable=SC2086 # the packages are one word each, split on purpose
|
|
suite -tags integration -v $rest >"$work/rest.out" 2>&1 || status=$?
|
|
echo "$status" >"$work/rest.status"
|
|
) &
|
|
for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do
|
|
shard_package "${entry%%|*}" "${entry#*|}" &
|
|
done
|
|
wait
|
|
grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/rest.out" || true
|
|
[ "$(cat "$work/rest.status")" -eq 0 ] || suite_failed=1
|
|
say_shards || suite_failed=1
|
|
shard_tests=$(cat "$work"/*.status 2>/dev/null | awk 'NF == 3 {n += $3} END {print n + 0}')
|
|
rest_tests=$(grep -cE '^--- (PASS|FAIL|SKIP): ' "$work/rest.out" || true)
|
|
rest_skipped=$(grep -cE '^--- SKIP: ' "$work/rest.out" || true)
|
|
echo "integration level (the whole suite, tag integration): $((rest_tests + shard_tests)) tests ($rest_skipped skipped outside the shards), $(($(date +%s) - integration_start)) s"
|
|
nested_start=$(date +%s)
|
|
nested_modules || suite_failed=1
|
|
echo "nested modules: $(($(date +%s) - nested_start)) s"
|
|
fi
|
|
|
|
if [ -z "$listed" ]; then
|
|
echo "store tests: none listed in testdata/store-tests"
|
|
elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then
|
|
echo "store tests, each run against the store:"
|
|
missing=0
|
|
for pkg in $store_packages; do
|
|
key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _)
|
|
names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
|
|
# shellcheck disable=SC2086 # the names are one word each, split on purpose
|
|
judge_store_tests "$pkg" "$(cat "$work/store.$key.status")" "$work/store.$key.out" $names
|
|
done
|
|
if [ "$missing" -ne 0 ]; then
|
|
echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so the store tests listed in testdata/store-tests skipped"
|
|
fi
|
|
if [ "$suite_failed" -ne 0 ]; then
|
|
echo "the suite failed: each package or shard that failed is said above with its output"
|
|
exit 1
|
|
fi
|