From 3ec4473b29b29fc6959788f246d7653d89ede03b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 14 Oct 2026 20:38:35 +0200 Subject: [PATCH] Run the slow test packages in shards and stop compiling the standard library per judge test (hq issue 717) The controller's check spent 3-5 minutes in cmd/mesh-controller, whose few hundred store tests run one after another, while most packages end in seconds. merge-check.sh now compiles each slow package once and deals its tests round-robin into processes that run beside each other and beside the rest of the suite; every test still runs once, under the race detector, through its package's TestMain. The listed store tests run beside the suite instead of after it. internal/builder's judge tests ran a real go build with a fresh Go cache each, compiling the standard library every time: half the suite's CPU. The fake runtime fills an empty cache from one warmed per process. The lease tests run in parallel. The link test of slow work now sets its one-second window after the serving loop asserted the controller's thirty-second one, so it proves the heartbeat (it passed without it) in half the time. --- internal/builder/release_branch_test.go | 7 + internal/builder/warm_go_cache_test.go | 80 ++++++++++++ internal/lease/lease_test.go | 13 ++ internal/link/builds_nats.go | 2 +- internal/link/receive_nats.go | 11 +- internal/link/receive_nats_test.go | 50 +++++-- merge-check.sh | 167 +++++++++++++++++++++--- 7 files changed, 294 insertions(+), 36 deletions(-) create mode 100644 internal/builder/warm_go_cache_test.go diff --git a/internal/builder/release_branch_test.go b/internal/builder/release_branch_test.go index 6547ac0e..a0adb5ae 100644 --- a/internal/builder/release_branch_test.go +++ b/internal/builder/release_branch_test.go @@ -156,6 +156,9 @@ run) if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS" ]; then echo "HOME=$HOME GOCACHE=$GOCACHE GOMODCACHE=$GOMODCACHE GOENV=$GOENV GOFLAGS=$GOFLAGS DIR=$workdir" >> "$REPLAYS" fi + if [ "$1" = go ] && [ -n "$WARM_GO_CACHE" ] && [ -n "$GOCACHE" ] && [ -z "$(ls -A "$GOCACHE" 2>/dev/null)" ]; then + mkdir -p "$GOCACHE" && cp -R "$WARM_GO_CACHE/." "$GOCACHE/" + fi cd "$workdir" && exec "$@" ;; esac exit 1 @@ -165,6 +168,8 @@ exit 1 } t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) t.Setenv("GOTOOLCHAIN", "local") + // An empty Go cache the check names is filled from a warm one before `go` runs (warm_go_cache_test.go). + t.Setenv("WARM_GO_CACHE", warmGoCache(t)) return built } @@ -566,6 +571,8 @@ func TestTheBuildSeatNeverReadsMeshLabAtAReleaseBranch(t *testing.T) { // a temporary directory of the host; and no container a check runs for the pull request mounts the judge cache. func TestJudgesAreBuiltWarmFromAJudgeCacheNoCheckContainerMounts(t *testing.T) { built := aRuntimeThatBuildsTheJudge(t) + // Not filled from the test's warm cache: what the first judge's build leaves in the judge cache is what is proved. + t.Setenv("WARM_GO_CACHE", "") runs := filepath.Join(t.TempDir(), "runs") t.Setenv("RUNS", runs) tmp := t.TempDir() diff --git a/internal/builder/warm_go_cache_test.go b/internal/builder/warm_go_cache_test.go new file mode 100644 index 00000000..bc973b83 --- /dev/null +++ b/internal/builder/warm_go_cache_test.go @@ -0,0 +1,80 @@ +package builder + +import ( + "os" + "os/exec" + "path/filepath" + "sync" + "testing" +) + +// **A judge's build in a test starts from a warm Go cache** (novox/hq issue 717). The runtime these tests put on +// PATH (aRuntimeThatBuildsTheJudge) runs the check's `go build` and `go test` on this machine with the Go cache the +// check names — a fresh directory in every test — so each of them compiled the standard library from nothing: some +// ten CPU-seconds a build, under twenty tests, half of the CPU the whole suite spent. The runtime now fills an empty +// Go cache it is given from one this process warmed once, before it runs `go`; what the check names, mounts and +// keeps is unchanged, and so is what each test reads of it. A cache entry is keyed by everything the compile +// depends on, flags included, so a build with a planted -toolexec hook still misses it and still runs its hook. +// +// The warm cache lives beside the system's temporary directory as it was when the package started, never under a +// TMPDIR a test set and then asserts left empty, and is removed when the package's tests end. +var ( + warmCacheBase = os.TempDir() + warmCacheOnce sync.Once + warmCacheDir string +) + +func TestMain(m *testing.M) { + code := m.Run() + if warmCacheDir != "" { + _ = os.RemoveAll(warmCacheDir) + } + os.Exit(code) +} + +// warmGoCache is a Go cache holding what a judge's build and the replays' `go test` compile of the standard +// library, or "" when it cannot be warmed — a test then builds cold, as before, and is only slower. +func warmGoCache(t *testing.T) string { + t.Helper() + warmCacheOnce.Do(func() { + dir, err := os.MkdirTemp(warmCacheBase, "judge-warm-go-cache-") + if err != nil { + return + } + src := filepath.Join(dir, "src") + files := map[string]string{ + "go.mod": "module example.org/warm\n\ngo 1.22\n", + "main.go": "package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n)\n\nfunc main() { fmt.Fprintln(os.Stdout, len(os.Args)) }\n", + "main_test.go": "package main\n\nimport \"testing\"\n\nfunc TestWarm(t *testing.T) {}\n", + } + for name, body := range files { + if err := os.MkdirAll(src, 0o755); err != nil { + return + } + if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil { + return + } + } + cache := filepath.Join(dir, "go-cache") + env := append(os.Environ(), "GOCACHE="+cache, "GOENV=off", "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off", + "GOFLAGS=-buildvcs=false", "TMPDIR="+warmCacheBase) + // As the check builds a judge (CGO off) and runs the replays (CGO as the toolchain has it). + for _, run := range [][]string{ + {"env", "CGO_ENABLED=0", "go", "build", "-o", filepath.Join(dir, "judge"), "."}, + {"go", "test", "-count=1", "."}, + } { + cmd := exec.Command(run[0], run[1:]...) + cmd.Dir, cmd.Env = src, env + if out, err := cmd.CombinedOutput(); err != nil { + t.Logf("the warm Go cache could not be built, so judges build cold: %v\n%s", err, out) + _ = os.RemoveAll(dir) + return + } + } + warmCacheDir = dir + }) + if warmCacheDir == "" { + return "" + } + return filepath.Join(warmCacheDir, "go-cache") +} diff --git a/internal/lease/lease_test.go b/internal/lease/lease_test.go index 2f1d7069..c242b572 100644 --- a/internal/lease/lease_test.go +++ b/internal/lease/lease_test.go @@ -22,6 +22,9 @@ import ( // refused is a loss at once; and a bucket raised again from nothing never issues an epoch twice. // // go test ./internal/lease/ (each test on a bus of its own: internal/testbus) +// +// Each test runs beside the others (novox/hq issue 717): they share nothing but the clock, and wait on it +// most of their time, so one after another they took half a minute of a check. // The bounds in these tests are the design's divided by five, so the cases run in seconds. const ( @@ -95,6 +98,7 @@ func takeWithin(t *testing.T, l *Lease) <-chan uint64 { } func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil) @@ -148,6 +152,7 @@ func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) { } func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil) epochA, err := a.Take(t.Context()) @@ -193,6 +198,7 @@ func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) { } func TestARenewalRefusedIsALossAtOnce(t *testing.T) { + t.Parallel() js, bucket := aBucket(t) a := aController(t, bucket, "a", nil) if _, err := a.Take(t.Context()); err != nil { @@ -226,6 +232,7 @@ func TestARenewalRefusedIsALossAtOnce(t *testing.T) { } func TestKeepingRenewsAcrossManyAges(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a := aController(t, bucket, "a", nil) epoch, err := a.Take(t.Context()) @@ -249,6 +256,7 @@ func TestKeepingRenewsAcrossManyAges(t *testing.T) { } func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) // The mesh has issued epoch 500 before: its store says so. The bucket is new — a bus whose data // was replaced — and its revisions start at one. @@ -275,6 +283,7 @@ func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) { } func TestALeaseBucketWithoutAnAgeIsRefused(t *testing.T) { + t.Parallel() url := testbus.URL(t) conn, err := nats.Connect(url) if err != nil { @@ -339,6 +348,7 @@ func delays(l *Lease) <-chan error { } func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a := aController(t, bucket, "a", nil) told := delays(a) @@ -390,6 +400,7 @@ func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) { } func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a := aController(t, bucket, "a", nil) bus := stallable(a, false) @@ -426,6 +437,7 @@ func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) { } func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) { + t.Parallel() _, bucket := aBucket(t) a := aController(t, bucket, "a", nil) bus := stallable(a, true) @@ -460,6 +472,7 @@ func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) { } func TestAnotherHolderSeenAfterAStallIsALossAtOnce(t *testing.T) { + t.Parallel() js, bucket := aBucket(t) a := aController(t, bucket, "a", nil) bus := stallable(a, false) diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go index 5ec3b9c1..d3147bda 100644 --- a/internal/link/builds_nats.go +++ b/internal/link/builds_nats.go @@ -332,7 +332,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) // as the controller says it for its own long handlers, so the server neither hands // the ask to a second machine nor counts the wait against its deliveries. working := make(chan struct{}) - go stillWorking(msg, working) + go stillWorking(msg, working, heartbeatWhileWorking) do(ctx, build) close(working) } diff --git a/internal/link/receive_nats.go b/internal/link/receive_nats.go index e8a6cb42..8283b4cf 100644 --- a/internal/link/receive_nats.go +++ b/internal/link/receive_nats.go @@ -220,21 +220,22 @@ func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Con // stops saying so, and the bus redelivers exactly as it should. working := make(chan struct{}) defer close(working) - go stillWorking(msg, working) + go stillWorking(msg, working, heartbeatWhileWorking) } act(ctx, m) } // heartbeatWhileWorking is how often a handler still running tells the bus so — comfortably inside -// the shortest acknowledgement window the mesh gives any of its consumers. -const heartbeatWhileWorking = 10 * time.Second +// the shortest acknowledgement window the mesh gives any of its consumers. A variable only so a test can +// outlive a window of a second in a few (novox/hq issue 717); nothing else sets it. +var heartbeatWhileWorking = 10 * time.Second // stillWorking keeps one message alive until the work on it returns. // // An error is not worth reporting: what the bus does when it is not told is redeliver, which is // exactly what happens if this fails, and the handler's own outcome is the thing worth logging. -func stillWorking(msg *nats.Msg, done <-chan struct{}) { - tick := time.NewTicker(heartbeatWhileWorking) +func stillWorking(msg *nats.Msg, done <-chan struct{}, every time.Duration) { + tick := time.NewTicker(every) defer tick.Stop() for { select { diff --git a/internal/link/receive_nats_test.go b/internal/link/receive_nats_test.go index 16007299..c4c74efc 100644 --- a/internal/link/receive_nats_test.go +++ b/internal/link/receive_nats_test.go @@ -405,16 +405,15 @@ func TestEverySubjectTheControllerFollowsDecodesToAKind(t *testing.T) { // The bus waits a fixed time to be told a message was taken and then redelivers, which is right for // a consumer that died and wrong for one that is busy. Acting on a merge builds modules — minutes // against a thirty-second window — and the same merge was handed over five times while the first -// build was still running (2026-09-28). Here the window is two seconds and the work takes six. +// build was still running (2026-09-28). Here the window is a second and the work takes three. func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { + // Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq + // issue 717: the test once set its window before the serving loop asserted the controller's own of thirty + // seconds, so it waited nine seconds and passed with the heartbeat removed). + before := heartbeatWhileWorking + heartbeatWhileWorking = 250 * time.Millisecond + t.Cleanup(func() { heartbeatWhileWorking = before }) js := aBus(t) - // The controller's own consumer, with a window short enough to outlive in a test. - if err := js.EnsureConsumer(broker.Consumer{ - Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 2, - Why: "a window short enough to outlive in a test", - }); err != nil { - t.Fatal(err) - } var mu sync.Mutex handled := 0 slow := make(chan struct{}) @@ -426,10 +425,32 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { first := handled == 1 mu.Unlock() if first { - time.Sleep(6 * time.Second) + time.Sleep(3 * time.Second) close(slow) } }} + // The controller's own consumer, given a window short enough to outlive in a test once the serving loop has + // asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds, + // which is how this test once waited out a window it never had. + for deadline := time.Now().Add(10 * time.Second); ; { + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + if err == nil && info.PushBound { + break + } + if time.Now().After(deadline) { + t.Fatalf("the serving loop never bound the controller's consumer: %v", err) + } + time.Sleep(10 * time.Millisecond) + } + if err := js.EnsureConsumer(broker.Consumer{ + Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1, + Why: "a window short enough to outlive in a test", + }); err != nil { + t.Fatal(err) + } + if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second { + t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err) + } body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) if err != nil { @@ -444,12 +465,21 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { t.Fatal("the slow work never finished") } // A moment for a redelivery to arrive, if the bus were going to send one. - time.Sleep(3 * time.Second) + time.Sleep(1500 * time.Millisecond) mu.Lock() defer mu.Unlock() if handled != 1 { t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled) } + // Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone + // would pass on a bus that redelivered the work every window; the bus's own count says it did not. + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + if err != nil { + t.Fatal(err) + } + if info.NumRedelivered != 0 { + t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered) + } } // slowly is a listener that runs whatever it was given. diff --git a/merge-check.sh b/merge-check.sh index 36de3711..221d4bba 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -54,13 +54,16 @@ if [ -n "$unformatted" ]; then fi CGO_ENABLED=0 go vet ./... +# The race detector when the toolchain has a C compiler, and said when it has not. +if command -v gcc >/dev/null 2>&1; then + cgo=1 race=-race +else + echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" + cgo=0 race= +fi suite() { - if command -v gcc >/dev/null 2>&1; then - CGO_ENABLED=1 go test -race -count=1 -timeout 30m "$@" - else - echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" - CGO_ENABLED=0 go test -count=1 -timeout 30m "$@" - fi + # shellcheck disable=SC2086 # $race is one flag or none + CGO_ENABLED=$cgo go test $race -count=1 -timeout 30m "$@" } # **A branch tried before its pull request may ask for a part** (novox/hq issue 686): the controller's `try` verb # hands MESH_CHECK_TESTS, a `go test -run` pattern, or MESH_CHECK_STORE_ONLY=1. A pull request's merge check is @@ -70,17 +73,102 @@ if [ -n "${MESH_CHECK_TESTS:-}" ]; then suite -run "$MESH_CHECK_TESTS" ./... exit 0 fi -if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then - echo "TRIED IN PART: only the store tests listed in testdata/store-tests" -else - suite ./... -fi + +# **The slowest packages run in shards** (novox/hq issue 717). `go test` runs packages beside each other but the +# tests of one package one after another, in one process, and these packages' tests wait on the store, a bus, a +# build or a clock: cmd/mesh-controller alone took 3 to 5 minutes of a check whose other packages end in seconds. +# Each is compiled once, its tests are dealt round-robin into that many processes, and those run beside each other +# and beside the rest of the suite. Each process is the package's own test binary, run from the package's +# directory through its TestMain as `go test` runs it, so a test cannot tell, and every test runs exactly once, in +# one of them. Tests of one package therefore keep what packages run side by side already keep: nothing shared +# outside their process — no fixed port, file or database name. A package is added here by measuring it +# (`go test -json` gives each test's time), never to hide a slow test that could wait less. +sharded='./cmd/mesh-controller 8 +./internal/inventory 4 +./internal/builder 4 +./internal/link 4' + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +# shard_package compiles one package's tests and runs them in $2 processes, each process's output and +# " " left in $work: $1 the package's directory, $2 the number of processes. +shard_package() { + dir=$1 shards=$2 + key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _) + bin="$work/$key.test" + status=0 + # shellcheck disable=SC2086 # $race is one flag or none + CGO_ENABLED=$cgo go test $race -c -o "$bin" "$dir" >"$work/$key.build" 2>&1 || status=$? + # Every test, example and fuzz target `go test` would run, and nothing a TestMain might print beside them. A + # binary that cannot list them, or lists none, fails here rather than pass by running nothing. + if [ "$status" -eq 0 ]; then + (cd "$dir" && "$bin" -test.list '.*') >"$work/$key.list" 2>&1 || status=$? + names=$(grep -E '^(Test|Example|Fuzz)' "$work/$key.list" || true) + if [ "$status" -ne 0 ] || [ -z "$names" ]; then + { + cat "$work/$key.list" + echo "the test binary of $dir listed no tests to deal into shards (exit status $status)" + } >>"$work/$key.build" + status=1 + fi + fi + echo "$status" >"$work/$key.built" + [ "$status" -eq 0 ] || return 0 + i=0 + while [ "$i" -lt "$shards" ]; do + pattern=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | paste -sd '|' -) + count=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | wc -l) + if [ -n "$pattern" ]; then + ( + start=$(date +%s) + status=0 + cd "$dir" && "$bin" -test.count=1 -test.timeout=30m -test.paniconexit0 -test.run "^($pattern)\$" \ + >"$work/$key.$i.out" 2>&1 || status=$? + echo "$status $(($(date +%s) - start)) $count" >"$work/$key.$i.status" + ) & + fi + i=$((i + 1)) + done + wait +} + +# say_shards prints how each sharded package's processes ended, as `go test` says a package, and the output of +# each that failed; it fails when any did. +say_shards() { + failed=0 + module=$(go list -m) + for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do + dir=${entry%%|*} shards=${entry#*|} + key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _) + pkg="$module/${dir#./}" + if [ "$(cat "$work/$key.built" 2>/dev/null || echo 1)" -ne 0 ]; then + cat "$work/$key.build" + printf 'FAIL\t%s [build failed]\n' "$pkg" + failed=1 + continue + fi + i=0 + while [ "$i" -lt "$shards" ]; do + if [ -f "$work/$key.$i.status" ]; then + read -r status took count <"$work/$key.$i.status" + if [ "$status" -eq 0 ]; then + printf 'ok \t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count" + else + cat "$work/$key.$i.out" + printf 'FAIL\t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count" + failed=1 + fi + fi + i=$((i + 1)) + done + done + return "$failed" +} # **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip # passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against # it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list. - - listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true) # Each line is exactly ` `: a line of one field ran nothing and said nothing (novox/hq issue 515). malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true) @@ -107,20 +195,55 @@ for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v ' done [ "$dropped" -eq 0 ] || exit 1 +# The listed store tests run beside the suite, one process per package (novox/hq issue 717): each package's output +# and exit status are left in $work and judged once the suite is done. +store_packages=$(printf '%s\n' "$listed" | awk 'NF {print $1}' | sort -u) +if [ -n "$listed" ] && [ -n "${MESH_TEST_POSTGRES:-}" ]; then + for pkg in $store_packages; do + key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _) + names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}') + pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$" + ( + status=0 + CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$work/store.$key.out" 2>&1 || status=$? + echo "$status" >"$work/store.$key.status" + ) & + done +fi + +suite_failed=0 +if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then + echo "TRIED IN PART: only the store tests listed in testdata/store-tests" + wait +else + module=$(go list -m) + skip=$(printf '%s\n' "$sharded" | awk -v m="$module" '{sub(/^\.\//, "", $1); print m "/" $1}') + rest=$(go list ./... | grep -vxF "$skip") + ( + status=0 + # shellcheck disable=SC2086 # the packages are one word each, split on purpose + suite $rest >"$work/rest.out" 2>&1 || status=$? + echo "$status" >"$work/rest.status" + ) & + for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do + shard_package "${entry%%|*}" "${entry#*|}" & + done + wait + cat "$work/rest.out" + [ "$(cat "$work/rest.status")" -eq 0 ] || suite_failed=1 + say_shards || suite_failed=1 +fi + if [ -z "$listed" ]; then echo "store tests: none listed in testdata/store-tests" elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then echo "store tests, each run against the store:" missing=0 - out=$(mktemp) - trap 'rm -f "$out"' EXIT - for pkg in $(printf '%s\n' "$listed" | awk '{print $1}' | sort -u); do + for pkg in $store_packages; do + key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _) names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}') - pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$" - status=0 - CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$out" 2>&1 || status=$? # shellcheck disable=SC2086 # the names are one word each, split on purpose - judge_store_tests "$pkg" "$status" "$out" $names + judge_store_tests "$pkg" "$(cat "$work/store.$key.status")" "$work/store.$key.out" $names done if [ "$missing" -ne 0 ]; then echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store" @@ -129,3 +252,7 @@ elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then else echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so the store tests listed in testdata/store-tests skipped" fi +if [ "$suite_failed" -ne 0 ]; then + echo "the suite failed: each package or shard that failed is said above with its output" + exit 1 +fi