diff --git a/internal/builder/release_branch_test.go b/internal/builder/release_branch_test.go index 6547ac0e..a0adb5ae 100644 --- a/internal/builder/release_branch_test.go +++ b/internal/builder/release_branch_test.go @@ -156,6 +156,9 @@ run) if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS" ]; then echo "HOME=$HOME GOCACHE=$GOCACHE GOMODCACHE=$GOMODCACHE GOENV=$GOENV GOFLAGS=$GOFLAGS DIR=$workdir" >> "$REPLAYS" fi + if [ "$1" = go ] && [ -n "$WARM_GO_CACHE" ] && [ -n "$GOCACHE" ] && [ -z "$(ls -A "$GOCACHE" 2>/dev/null)" ]; then + mkdir -p "$GOCACHE" && cp -R "$WARM_GO_CACHE/." "$GOCACHE/" + fi cd "$workdir" && exec "$@" ;; esac exit 1 @@ -165,6 +168,8 @@ exit 1 } t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) t.Setenv("GOTOOLCHAIN", "local") + // An empty Go cache the check names is filled from a warm one before `go` runs (warm_go_cache_test.go). + t.Setenv("WARM_GO_CACHE", warmGoCache(t)) return built } @@ -566,6 +571,8 @@ func TestTheBuildSeatNeverReadsMeshLabAtAReleaseBranch(t *testing.T) { // a temporary directory of the host; and no container a check runs for the pull request mounts the judge cache. func TestJudgesAreBuiltWarmFromAJudgeCacheNoCheckContainerMounts(t *testing.T) { built := aRuntimeThatBuildsTheJudge(t) + // Not filled from the test's warm cache: what the first judge's build leaves in the judge cache is what is proved. + t.Setenv("WARM_GO_CACHE", "") runs := filepath.Join(t.TempDir(), "runs") t.Setenv("RUNS", runs) tmp := t.TempDir() diff --git a/internal/builder/warm_go_cache_test.go b/internal/builder/warm_go_cache_test.go new file mode 100644 index 00000000..bc973b83 --- /dev/null +++ b/internal/builder/warm_go_cache_test.go @@ -0,0 +1,80 @@ +package builder + +import ( + "os" + "os/exec" + "path/filepath" + "sync" + "testing" +) + +// **A judge's build in a test starts from a warm Go cache** (novox/hq issue 717). The runtime these tests put on +// PATH (aRuntimeThatBuildsTheJudge) runs the check's `go build` and `go test` on this machine with the Go cache the +// check names — a fresh directory in every test — so each of them compiled the standard library from nothing: some +// ten CPU-seconds a build, under twenty tests, half of the CPU the whole suite spent. The runtime now fills an empty +// Go cache it is given from one this process warmed once, before it runs `go`; what the check names, mounts and +// keeps is unchanged, and so is what each test reads of it. A cache entry is keyed by everything the compile +// depends on, flags included, so a build with a planted -toolexec hook still misses it and still runs its hook. +// +// The warm cache lives beside the system's temporary directory as it was when the package started, never under a +// TMPDIR a test set and then asserts left empty, and is removed when the package's tests end. +var ( + warmCacheBase = os.TempDir() + warmCacheOnce sync.Once + warmCacheDir string +) + +func TestMain(m *testing.M) { + code := m.Run() + if warmCacheDir != "" { + _ = os.RemoveAll(warmCacheDir) + } + os.Exit(code) +} + +// warmGoCache is a Go cache holding what a judge's build and the replays' `go test` compile of the standard +// library, or "" when it cannot be warmed — a test then builds cold, as before, and is only slower. +func warmGoCache(t *testing.T) string { + t.Helper() + warmCacheOnce.Do(func() { + dir, err := os.MkdirTemp(warmCacheBase, "judge-warm-go-cache-") + if err != nil { + return + } + src := filepath.Join(dir, "src") + files := map[string]string{ + "go.mod": "module example.org/warm\n\ngo 1.22\n", + "main.go": "package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n)\n\nfunc main() { fmt.Fprintln(os.Stdout, len(os.Args)) }\n", + "main_test.go": "package main\n\nimport \"testing\"\n\nfunc TestWarm(t *testing.T) {}\n", + } + for name, body := range files { + if err := os.MkdirAll(src, 0o755); err != nil { + return + } + if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil { + return + } + } + cache := filepath.Join(dir, "go-cache") + env := append(os.Environ(), "GOCACHE="+cache, "GOENV=off", "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off", + "GOFLAGS=-buildvcs=false", "TMPDIR="+warmCacheBase) + // As the check builds a judge (CGO off) and runs the replays (CGO as the toolchain has it). + for _, run := range [][]string{ + {"env", "CGO_ENABLED=0", "go", "build", "-o", filepath.Join(dir, "judge"), "."}, + {"go", "test", "-count=1", "."}, + } { + cmd := exec.Command(run[0], run[1:]...) + cmd.Dir, cmd.Env = src, env + if out, err := cmd.CombinedOutput(); err != nil { + t.Logf("the warm Go cache could not be built, so judges build cold: %v\n%s", err, out) + _ = os.RemoveAll(dir) + return + } + } + warmCacheDir = dir + }) + if warmCacheDir == "" { + return "" + } + return filepath.Join(warmCacheDir, "go-cache") +} diff --git a/internal/lease/lease_test.go b/internal/lease/lease_test.go index 2f1d7069..c242b572 100644 --- a/internal/lease/lease_test.go +++ b/internal/lease/lease_test.go @@ -22,6 +22,9 @@ import ( // refused is a loss at once; and a bucket raised again from nothing never issues an epoch twice. // // go test ./internal/lease/ (each test on a bus of its own: internal/testbus) +// +// Each test runs beside the others (novox/hq issue 717): they share nothing but the clock, and wait on it +// most of their time, so one after another they took half a minute of a check. // The bounds in these tests are the design's divided by five, so the cases run in seconds. const ( @@ -95,6 +98,7 @@ func takeWithin(t *testing.T, l *Lease) <-chan uint64 { } func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil) @@ -148,6 +152,7 @@ func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) { } func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil) epochA, err := a.Take(t.Context()) @@ -193,6 +198,7 @@ func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) { } func TestARenewalRefusedIsALossAtOnce(t *testing.T) { + t.Parallel() js, bucket := aBucket(t) a := aController(t, bucket, "a", nil) if _, err := a.Take(t.Context()); err != nil { @@ -226,6 +232,7 @@ func TestARenewalRefusedIsALossAtOnce(t *testing.T) { } func TestKeepingRenewsAcrossManyAges(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a := aController(t, bucket, "a", nil) epoch, err := a.Take(t.Context()) @@ -249,6 +256,7 @@ func TestKeepingRenewsAcrossManyAges(t *testing.T) { } func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) // The mesh has issued epoch 500 before: its store says so. The bucket is new — a bus whose data // was replaced — and its revisions start at one. @@ -275,6 +283,7 @@ func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) { } func TestALeaseBucketWithoutAnAgeIsRefused(t *testing.T) { + t.Parallel() url := testbus.URL(t) conn, err := nats.Connect(url) if err != nil { @@ -339,6 +348,7 @@ func delays(l *Lease) <-chan error { } func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a := aController(t, bucket, "a", nil) told := delays(a) @@ -390,6 +400,7 @@ func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) { } func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a := aController(t, bucket, "a", nil) bus := stallable(a, false) @@ -426,6 +437,7 @@ func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) { } func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a := aController(t, bucket, "a", nil) bus := stallable(a, true) @@ -460,6 +472,7 @@ func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) { } func TestAnotherHolderSeenAfterAStallIsALossAtOnce(t *testing.T) { + t.Parallel() js, bucket := aBucket(t) a := aController(t, bucket, "a", nil) bus := stallable(a, false) diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go index 5ec3b9c1..d3147bda 100644 --- a/internal/link/builds_nats.go +++ b/internal/link/builds_nats.go @@ -332,7 +332,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) // as the controller says it for its own long handlers, so the server neither hands // the ask to a second machine nor counts the wait against its deliveries. working := make(chan struct{}) - go stillWorking(msg, working) + go stillWorking(msg, working, heartbeatWhileWorking) do(ctx, build) close(working) } diff --git a/internal/link/receive_nats.go b/internal/link/receive_nats.go index e8a6cb42..8283b4cf 100644 --- a/internal/link/receive_nats.go +++ b/internal/link/receive_nats.go @@ -220,21 +220,22 @@ func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Con // stops saying so, and the bus redelivers exactly as it should. working := make(chan struct{}) defer close(working) - go stillWorking(msg, working) + go stillWorking(msg, working, heartbeatWhileWorking) } act(ctx, m) } // heartbeatWhileWorking is how often a handler still running tells the bus so — comfortably inside -// the shortest acknowledgement window the mesh gives any of its consumers. -const heartbeatWhileWorking = 10 * time.Second +// the shortest acknowledgement window the mesh gives any of its consumers. A variable only so a test can +// outlive a window of a second in a few (novox/hq issue 717); nothing else sets it. +var heartbeatWhileWorking = 10 * time.Second // stillWorking keeps one message alive until the work on it returns. // // An error is not worth reporting: what the bus does when it is not told is redeliver, which is // exactly what happens if this fails, and the handler's own outcome is the thing worth logging. -func stillWorking(msg *nats.Msg, done <-chan struct{}) { - tick := time.NewTicker(heartbeatWhileWorking) +func stillWorking(msg *nats.Msg, done <-chan struct{}, every time.Duration) { + tick := time.NewTicker(every) defer tick.Stop() for { select { diff --git a/internal/link/receive_nats_test.go b/internal/link/receive_nats_test.go index 16007299..c4c74efc 100644 --- a/internal/link/receive_nats_test.go +++ b/internal/link/receive_nats_test.go @@ -405,16 +405,15 @@ func TestEverySubjectTheControllerFollowsDecodesToAKind(t *testing.T) { // The bus waits a fixed time to be told a message was taken and then redelivers, which is right for // a consumer that died and wrong for one that is busy. Acting on a merge builds modules — minutes // against a thirty-second window — and the same merge was handed over five times while the first -// build was still running (2026-09-28). Here the window is two seconds and the work takes six. +// build was still running (2026-09-28). Here the window is a second and the work takes three. func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { + // Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq + // issue 717: the test once set its window before the serving loop asserted the controller's own of thirty + // seconds, so it waited nine seconds and passed with the heartbeat removed). + before := heartbeatWhileWorking + heartbeatWhileWorking = 250 * time.Millisecond + t.Cleanup(func() { heartbeatWhileWorking = before }) js := aBus(t) - // The controller's own consumer, with a window short enough to outlive in a test. - if err := js.EnsureConsumer(broker.Consumer{ - Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 2, - Why: "a window short enough to outlive in a test", - }); err != nil { - t.Fatal(err) - } var mu sync.Mutex handled := 0 slow := make(chan struct{}) @@ -426,10 +425,32 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { first := handled == 1 mu.Unlock() if first { - time.Sleep(6 * time.Second) + time.Sleep(3 * time.Second) close(slow) } }} + // The controller's own consumer, given a window short enough to outlive in a test once the serving loop has + // asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds, + // which is how this test once waited out a window it never had. + for deadline := time.Now().Add(10 * time.Second); ; { + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + if err == nil && info.PushBound { + break + } + if time.Now().After(deadline) { + t.Fatalf("the serving loop never bound the controller's consumer: %v", err) + } + time.Sleep(10 * time.Millisecond) + } + if err := js.EnsureConsumer(broker.Consumer{ + Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1, + Why: "a window short enough to outlive in a test", + }); err != nil { + t.Fatal(err) + } + if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second { + t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err) + } body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) if err != nil { @@ -444,12 +465,21 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { t.Fatal("the slow work never finished") } // A moment for a redelivery to arrive, if the bus were going to send one. - time.Sleep(3 * time.Second) + time.Sleep(1500 * time.Millisecond) mu.Lock() defer mu.Unlock() if handled != 1 { t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled) } + // Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone + // would pass on a bus that redelivered the work every window; the bus's own count says it did not. + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + if err != nil { + t.Fatal(err) + } + if info.NumRedelivered != 0 { + t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered) + } } // slowly is a listener that runs whatever it was given. diff --git a/merge-check.sh b/merge-check.sh index 36de3711..221d4bba 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -54,13 +54,16 @@ if [ -n "$unformatted" ]; then fi CGO_ENABLED=0 go vet ./... +# The race detector when the toolchain has a C compiler, and said when it has not. +if command -v gcc >/dev/null 2>&1; then + cgo=1 race=-race +else + echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" + cgo=0 race= +fi suite() { - if command -v gcc >/dev/null 2>&1; then - CGO_ENABLED=1 go test -race -count=1 -timeout 30m "$@" - else - echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" - CGO_ENABLED=0 go test -count=1 -timeout 30m "$@" - fi + # shellcheck disable=SC2086 # $race is one flag or none + CGO_ENABLED=$cgo go test $race -count=1 -timeout 30m "$@" } # **A branch tried before its pull request may ask for a part** (novox/hq issue 686): the controller's `try` verb # hands MESH_CHECK_TESTS, a `go test -run` pattern, or MESH_CHECK_STORE_ONLY=1. A pull request's merge check is @@ -70,17 +73,102 @@ if [ -n "${MESH_CHECK_TESTS:-}" ]; then suite -run "$MESH_CHECK_TESTS" ./... exit 0 fi -if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then - echo "TRIED IN PART: only the store tests listed in testdata/store-tests" -else - suite ./... -fi + +# **The slowest packages run in shards** (novox/hq issue 717). `go test` runs packages beside each other but the +# tests of one package one after another, in one process, and these packages' tests wait on the store, a bus, a +# build or a clock: cmd/mesh-controller alone took 3 to 5 minutes of a check whose other packages end in seconds. +# Each is compiled once, its tests are dealt round-robin into that many processes, and those run beside each other +# and beside the rest of the suite. Each process is the package's own test binary, run from the package's +# directory through its TestMain as `go test` runs it, so a test cannot tell, and every test runs exactly once, in +# one of them. Tests of one package therefore keep what packages run side by side already keep: nothing shared +# outside their process — no fixed port, file or database name. A package is added here by measuring it +# (`go test -json` gives each test's time), never to hide a slow test that could wait less. +sharded='./cmd/mesh-controller 8 +./internal/inventory 4 +./internal/builder 4 +./internal/link 4' + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# shard_package compiles one package's tests and runs them in $2 processes, each process's output and +# " " left in $work: $1 the package's directory, $2 the number of processes. +shard_package() { + dir=$1 shards=$2 + key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _) + bin="$work/$key.test" + status=0 + # shellcheck disable=SC2086 # $race is one flag or none + CGO_ENABLED=$cgo go test $race -c -o "$bin" "$dir" >"$work/$key.build" 2>&1 || status=$? + # Every test, example and fuzz target `go test` would run, and nothing a TestMain might print beside them. A + # binary that cannot list them, or lists none, fails here rather than pass by running nothing. + if [ "$status" -eq 0 ]; then + (cd "$dir" && "$bin" -test.list '.*') >"$work/$key.list" 2>&1 || status=$? + names=$(grep -E '^(Test|Example|Fuzz)' "$work/$key.list" || true) + if [ "$status" -ne 0 ] || [ -z "$names" ]; then + { + cat "$work/$key.list" + echo "the test binary of $dir listed no tests to deal into shards (exit status $status)" + } >>"$work/$key.build" + status=1 + fi + fi + echo "$status" >"$work/$key.built" + [ "$status" -eq 0 ] || return 0 + i=0 + while [ "$i" -lt "$shards" ]; do + pattern=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | paste -sd '|' -) + count=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | wc -l) + if [ -n "$pattern" ]; then + ( + start=$(date +%s) + status=0 + cd "$dir" && "$bin" -test.count=1 -test.timeout=30m -test.paniconexit0 -test.run "^($pattern)\$" \ + >"$work/$key.$i.out" 2>&1 || status=$? + echo "$status $(($(date +%s) - start)) $count" >"$work/$key.$i.status" + ) & + fi + i=$((i + 1)) + done + wait +} + +# say_shards prints how each sharded package's processes ended, as `go test` says a package, and the output of +# each that failed; it fails when any did. +say_shards() { + failed=0 + module=$(go list -m) + for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do + dir=${entry%%|*} shards=${entry#*|} + key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _) + pkg="$module/${dir#./}" + if [ "$(cat "$work/$key.built" 2>/dev/null || echo 1)" -ne 0 ]; then + cat "$work/$key.build" + printf 'FAIL\t%s [build failed]\n' "$pkg" + failed=1 + continue + fi + i=0 + while [ "$i" -lt "$shards" ]; do + if [ -f "$work/$key.$i.status" ]; then + read -r status took count <"$work/$key.$i.status" + if [ "$status" -eq 0 ]; then + printf 'ok \t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count" + else + cat "$work/$key.$i.out" + printf 'FAIL\t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count" + failed=1 + fi + fi + i=$((i + 1)) + done + done + return "$failed" +} # **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip # passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against # it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list. - - listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true) # Each line is exactly ` `: a line of one field ran nothing and said nothing (novox/hq issue 515). malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true) @@ -107,20 +195,55 @@ for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v ' done [ "$dropped" -eq 0 ] || exit 1 +# The listed store tests run beside the suite, one process per package (novox/hq issue 717): each package's output +# and exit status are left in $work and judged once the suite is done. +store_packages=$(printf '%s\n' "$listed" | awk 'NF {print $1}' | sort -u) +if [ -n "$listed" ] && [ -n "${MESH_TEST_POSTGRES:-}" ]; then + for pkg in $store_packages; do + key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _) + names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}') + pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$" + ( + status=0 + CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$work/store.$key.out" 2>&1 || status=$? + echo "$status" >"$work/store.$key.status" + ) & + done +fi + +suite_failed=0 +if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then + echo "TRIED IN PART: only the store tests listed in testdata/store-tests" + wait +else + module=$(go list -m) + skip=$(printf '%s\n' "$sharded" | awk -v m="$module" '{sub(/^\.\//, "", $1); print m "/" $1}') + rest=$(go list ./... | grep -vxF "$skip") + ( + status=0 + # shellcheck disable=SC2086 # the packages are one word each, split on purpose + suite $rest >"$work/rest.out" 2>&1 || status=$? + echo "$status" >"$work/rest.status" + ) & + for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do + shard_package "${entry%%|*}" "${entry#*|}" & + done + wait + cat "$work/rest.out" + [ "$(cat "$work/rest.status")" -eq 0 ] || suite_failed=1 + say_shards || suite_failed=1 +fi + if [ -z "$listed" ]; then echo "store tests: none listed in testdata/store-tests" elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then echo "store tests, each run against the store:" missing=0 - out=$(mktemp) - trap 'rm -f "$out"' EXIT - for pkg in $(printf '%s\n' "$listed" | awk '{print $1}' | sort -u); do + for pkg in $store_packages; do + key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _) names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}') - pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$" - status=0 - CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$out" 2>&1 || status=$? # shellcheck disable=SC2086 # the names are one word each, split on purpose - judge_store_tests "$pkg" "$status" "$out" $names + judge_store_tests "$pkg" "$(cat "$work/store.$key.status")" "$work/store.$key.out" $names done if [ "$missing" -ne 0 ]; then echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store" @@ -129,3 +252,7 @@ elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then else echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so the store tests listed in testdata/store-tests skipped" fi +if [ "$suite_failed" -ne 0 ]; then + echo "the suite failed: each package or shard that failed is said above with its output" + exit 1 +fi