jschoubben
3ec4473b29
mesh/merge-gate head 3ec4473b29b2 merged into trunk df8c1ccf1fff — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head 3ec4473b29b2 merged into trunk df8c1ccf1fff — pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check.sh): main's version judged it; the ch…
mesh/delivery delivered
The controller's check spent 3-5 minutes in cmd/mesh-controller, whose few hundred store tests run one after another, while most packages end in seconds. merge-check.sh now compiles each slow package once and deals its tests round-robin into processes that run beside each other and beside the rest of the suite; every test still runs once, under the race detector, through its package's TestMain. The listed store tests run beside the suite instead of after it. internal/builder's judge tests ran a real go build with a fresh Go cache each, compiling the standard library every time: half the suite's CPU. The fake runtime fills an empty cache from one warmed per process. The lease tests run in parallel. The link test of slow work now sets its one-second window after the serving loop asserted the controller's thirty-second one, so it proves the heartbeat (it passed without it) in half the time.
259 lines
11 KiB
Bash
Executable File
259 lines
11 KiB
Bash
Executable File
#!/bin/sh
|
|
# mesh-check-toolchain: go
|
|
#
|
|
# The controller's own check (novox/hq ADR 0237 as amended): the second layer of a pull request's merge
|
|
# check, `mesh/repo-check`, run by the build seat in the mesh's Go toolchain — and by hand:
|
|
# `MESH_TEST_POSTGRES=… sh merge-check.sh`.
|
|
#
|
|
# The gate — every machine composed with this change's controller, which judges itself — is the build
|
|
# seat's first layer (`mesh/merge-gate`), run because the mesh's module graph builds `mesh-controller`
|
|
# from this repository; it is not repeated here. This is the code's own quality and its suite:
|
|
#
|
|
# 1. formatted and vetted;
|
|
# 2. the whole suite, packages in parallel, against the throwaway store the build seat raised at the
|
|
# release the mesh runs; every live test on a bus of its own (internal/testbus), so nothing is
|
|
# serialised. Under the race detector when the toolchain has a C compiler, and said when it has not.
|
|
#
|
|
# Exit 0 is a pass; anything else fails `mesh/repo-check` with the last line printed.
|
|
set -eu
|
|
export GOFLAGS=-mod=vendor GOPROXY=off
|
|
|
|
# judge_store_tests says each listed test of one package as `go test -v` reported it, and counts in $missing each
|
|
# that did not pass: $1 the package, $2 go test's exit status, $3 the file holding its output, then the names.
|
|
# A result is found anywhere in a line, not only at its start: when a test's last output leaves a line open,
|
|
# `go test -v` writes `--- PASS: <name> (…)` on that same line (novox/hq issue 515). A package whose go test
|
|
# exits non-zero fails even when each listed test said PASS — a TestMain or a leak checker can fail after them.
|
|
judge_store_tests() {
|
|
pkg=$1 status=$2 out=$3
|
|
shift 3
|
|
for name in "$@"; do
|
|
line=$(grep -oE -- "--- (PASS|FAIL|SKIP): $name \([^)]*\)" "$out" | tail -n 1 || true)
|
|
echo " $pkg ${line:-"--- NOT RUN: $name"}"
|
|
case "$line" in "--- PASS: "*) ;; *) missing=$((missing + 1)) ;; esac
|
|
done
|
|
if [ "$status" -ne 0 ]; then
|
|
echo " $pkg FAIL: go test exited $status"
|
|
missing=$((missing + 1))
|
|
fi
|
|
}
|
|
# `sh merge-check.sh judge-store-tests <package> <status> <output> <name>…` runs only the judging, to try it on a
|
|
# recorded `go test -v` output.
|
|
if [ "${1:-}" = judge-store-tests ]; then
|
|
shift
|
|
missing=0
|
|
judge_store_tests "$@"
|
|
[ "$missing" -eq 0 ]
|
|
exit
|
|
fi
|
|
|
|
unformatted=$(gofmt -l cmd internal examples)
|
|
if [ -n "$unformatted" ]; then
|
|
echo "not gofmt'd:"
|
|
echo "$unformatted"
|
|
exit 1
|
|
fi
|
|
CGO_ENABLED=0 go vet ./...
|
|
|
|
# The race detector when the toolchain has a C compiler, and said when it has not.
|
|
if command -v gcc >/dev/null 2>&1; then
|
|
cgo=1 race=-race
|
|
else
|
|
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
|
|
cgo=0 race=
|
|
fi
|
|
suite() {
|
|
# shellcheck disable=SC2086 # $race is one flag or none
|
|
CGO_ENABLED=$cgo go test $race -count=1 -timeout 30m "$@"
|
|
}
|
|
# **A branch tried before its pull request may ask for a part** (novox/hq issue 686): the controller's `try` verb
|
|
# hands MESH_CHECK_TESTS, a `go test -run` pattern, or MESH_CHECK_STORE_ONLY=1. A pull request's merge check is
|
|
# handed neither and always runs whole; a part that passes is said as a part, never as the check.
|
|
if [ -n "${MESH_CHECK_TESTS:-}" ]; then
|
|
echo "TRIED IN PART: only the tests matching $MESH_CHECK_TESTS, in every package"
|
|
suite -run "$MESH_CHECK_TESTS" ./...
|
|
exit 0
|
|
fi
|
|
|
|
# **The slowest packages run in shards** (novox/hq issue 717). `go test` runs packages beside each other but the
|
|
# tests of one package one after another, in one process, and these packages' tests wait on the store, a bus, a
|
|
# build or a clock: cmd/mesh-controller alone took 3 to 5 minutes of a check whose other packages end in seconds.
|
|
# Each is compiled once, its tests are dealt round-robin into that many processes, and those run beside each other
|
|
# and beside the rest of the suite. Each process is the package's own test binary, run from the package's
|
|
# directory through its TestMain as `go test` runs it, so a test cannot tell, and every test runs exactly once, in
|
|
# one of them. Tests of one package therefore keep what packages run side by side already keep: nothing shared
|
|
# outside their process — no fixed port, file or database name. A package is added here by measuring it
|
|
# (`go test -json` gives each test's time), never to hide a slow test that could wait less.
|
|
sharded='./cmd/mesh-controller 8
|
|
./internal/inventory 4
|
|
./internal/builder 4
|
|
./internal/link 4'
|
|
|
|
work=$(mktemp -d)
|
|
trap 'rm -rf "$work"' EXIT
|
|
|
|
# shard_package compiles one package's tests and runs them in $2 processes, each process's output and
|
|
# "<exit status> <seconds>" left in $work: $1 the package's directory, $2 the number of processes.
|
|
shard_package() {
|
|
dir=$1 shards=$2
|
|
key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _)
|
|
bin="$work/$key.test"
|
|
status=0
|
|
# shellcheck disable=SC2086 # $race is one flag or none
|
|
CGO_ENABLED=$cgo go test $race -c -o "$bin" "$dir" >"$work/$key.build" 2>&1 || status=$?
|
|
# Every test, example and fuzz target `go test` would run, and nothing a TestMain might print beside them. A
|
|
# binary that cannot list them, or lists none, fails here rather than pass by running nothing.
|
|
if [ "$status" -eq 0 ]; then
|
|
(cd "$dir" && "$bin" -test.list '.*') >"$work/$key.list" 2>&1 || status=$?
|
|
names=$(grep -E '^(Test|Example|Fuzz)' "$work/$key.list" || true)
|
|
if [ "$status" -ne 0 ] || [ -z "$names" ]; then
|
|
{
|
|
cat "$work/$key.list"
|
|
echo "the test binary of $dir listed no tests to deal into shards (exit status $status)"
|
|
} >>"$work/$key.build"
|
|
status=1
|
|
fi
|
|
fi
|
|
echo "$status" >"$work/$key.built"
|
|
[ "$status" -eq 0 ] || return 0
|
|
i=0
|
|
while [ "$i" -lt "$shards" ]; do
|
|
pattern=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | paste -sd '|' -)
|
|
count=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | wc -l)
|
|
if [ -n "$pattern" ]; then
|
|
(
|
|
start=$(date +%s)
|
|
status=0
|
|
cd "$dir" && "$bin" -test.count=1 -test.timeout=30m -test.paniconexit0 -test.run "^($pattern)\$" \
|
|
>"$work/$key.$i.out" 2>&1 || status=$?
|
|
echo "$status $(($(date +%s) - start)) $count" >"$work/$key.$i.status"
|
|
) &
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
wait
|
|
}
|
|
|
|
# say_shards prints how each sharded package's processes ended, as `go test` says a package, and the output of
|
|
# each that failed; it fails when any did.
|
|
say_shards() {
|
|
failed=0
|
|
module=$(go list -m)
|
|
for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do
|
|
dir=${entry%%|*} shards=${entry#*|}
|
|
key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _)
|
|
pkg="$module/${dir#./}"
|
|
if [ "$(cat "$work/$key.built" 2>/dev/null || echo 1)" -ne 0 ]; then
|
|
cat "$work/$key.build"
|
|
printf 'FAIL\t%s [build failed]\n' "$pkg"
|
|
failed=1
|
|
continue
|
|
fi
|
|
i=0
|
|
while [ "$i" -lt "$shards" ]; do
|
|
if [ -f "$work/$key.$i.status" ]; then
|
|
read -r status took count <"$work/$key.$i.status"
|
|
if [ "$status" -eq 0 ]; then
|
|
printf 'ok \t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count"
|
|
else
|
|
cat "$work/$key.$i.out"
|
|
printf 'FAIL\t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count"
|
|
failed=1
|
|
fi
|
|
fi
|
|
i=$((i + 1))
|
|
done
|
|
done
|
|
return "$failed"
|
|
}
|
|
|
|
# **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip
|
|
# passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against
|
|
# it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list.
|
|
listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true)
|
|
# Each line is exactly `<package> <TestName>`: a line of one field ran nothing and said nothing (novox/hq issue 515).
|
|
malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true)
|
|
if [ -n "$malformed" ]; then
|
|
echo "testdata/store-tests: a line is not <package> <TestName>:"
|
|
echo "$malformed"
|
|
exit 1
|
|
fi
|
|
# **A change does not shorten the list it is judged by** (novox/hq issue 515): the list is in the change's own tree,
|
|
# so an entry main lists stays listed while its test function exists; removing the function removes the entry.
|
|
if ! git rev-parse --verify --quiet origin/main^{commit} >/dev/null; then
|
|
echo "origin/main cannot be read, and its testdata/store-tests says which store tests must stay listed"
|
|
exit 1
|
|
fi
|
|
dropped=0
|
|
for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v '^[[:space:]]*\(#\|$\)' |
|
|
awk 'NF == 2 {print $1 "|" $2}'); do
|
|
pkg=${entry%%|*} name=${entry#*|}
|
|
printf '%s\n' "$listed" | awk -v p="$pkg" -v n="$name" '$1 == p && $2 == n {f = 1} END {exit !f}' && continue
|
|
if cat "$pkg"/*_test.go 2>/dev/null | grep -q "^func $name("; then
|
|
echo "testdata/store-tests: $pkg $name is listed on main and its test still exists, but the change drops it"
|
|
dropped=$((dropped + 1))
|
|
fi
|
|
done
|
|
[ "$dropped" -eq 0 ] || exit 1
|
|
|
|
# The listed store tests run beside the suite, one process per package (novox/hq issue 717): each package's output
|
|
# and exit status are left in $work and judged once the suite is done.
|
|
store_packages=$(printf '%s\n' "$listed" | awk 'NF {print $1}' | sort -u)
|
|
if [ -n "$listed" ] && [ -n "${MESH_TEST_POSTGRES:-}" ]; then
|
|
for pkg in $store_packages; do
|
|
key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _)
|
|
names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
|
|
pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$"
|
|
(
|
|
status=0
|
|
CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$work/store.$key.out" 2>&1 || status=$?
|
|
echo "$status" >"$work/store.$key.status"
|
|
) &
|
|
done
|
|
fi
|
|
|
|
suite_failed=0
|
|
if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then
|
|
echo "TRIED IN PART: only the store tests listed in testdata/store-tests"
|
|
wait
|
|
else
|
|
module=$(go list -m)
|
|
skip=$(printf '%s\n' "$sharded" | awk -v m="$module" '{sub(/^\.\//, "", $1); print m "/" $1}')
|
|
rest=$(go list ./... | grep -vxF "$skip")
|
|
(
|
|
status=0
|
|
# shellcheck disable=SC2086 # the packages are one word each, split on purpose
|
|
suite $rest >"$work/rest.out" 2>&1 || status=$?
|
|
echo "$status" >"$work/rest.status"
|
|
) &
|
|
for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do
|
|
shard_package "${entry%%|*}" "${entry#*|}" &
|
|
done
|
|
wait
|
|
cat "$work/rest.out"
|
|
[ "$(cat "$work/rest.status")" -eq 0 ] || suite_failed=1
|
|
say_shards || suite_failed=1
|
|
fi
|
|
|
|
if [ -z "$listed" ]; then
|
|
echo "store tests: none listed in testdata/store-tests"
|
|
elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then
|
|
echo "store tests, each run against the store:"
|
|
missing=0
|
|
for pkg in $store_packages; do
|
|
key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _)
|
|
names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
|
|
# shellcheck disable=SC2086 # the names are one word each, split on purpose
|
|
judge_store_tests "$pkg" "$(cat "$work/store.$key.status")" "$work/store.$key.out" $names
|
|
done
|
|
if [ "$missing" -ne 0 ]; then
|
|
echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so the store tests listed in testdata/store-tests skipped"
|
|
fi
|
|
if [ "$suite_failed" -ne 0 ]; then
|
|
echo "the suite failed: each package or shard that failed is said above with its output"
|
|
exit 1
|
|
fi
|