Merge pull request 'Run the slow test packages in shards and build judges from a warm Go cache (hq issue 717)' (#334) from fix/717-the-suite-runs-in-under-two-minutes into main

internal/builder/release_branch_test.go

@@ -156,6 +156,9 @@ run)
   if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS" ]; then
     echo "HOME=$HOME GOCACHE=$GOCACHE GOMODCACHE=$GOMODCACHE GOENV=$GOENV GOFLAGS=$GOFLAGS DIR=$workdir" >> "$REPLAYS"
   fi
+  if [ "$1" = go ] && [ -n "$WARM_GO_CACHE" ] && [ -n "$GOCACHE" ] && [ -z "$(ls -A "$GOCACHE" 2>/dev/null)" ]; then
+    mkdir -p "$GOCACHE" && cp -R "$WARM_GO_CACHE/." "$GOCACHE/"
+  fi
   cd "$workdir" && exec "$@" ;;
 esac
 exit 1
@@ -165,6 +168,8 @@ exit 1
 	}
 	t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
 	t.Setenv("GOTOOLCHAIN", "local")
+	// An empty Go cache the check names is filled from a warm one before `go` runs (warm_go_cache_test.go).
+	t.Setenv("WARM_GO_CACHE", warmGoCache(t))
 	return built
 }
 
@@ -566,6 +571,8 @@ func TestTheBuildSeatNeverReadsMeshLabAtAReleaseBranch(t *testing.T) {
 // a temporary directory of the host; and no container a check runs for the pull request mounts the judge cache.
 func TestJudgesAreBuiltWarmFromAJudgeCacheNoCheckContainerMounts(t *testing.T) {
 	built := aRuntimeThatBuildsTheJudge(t)
+	// Not filled from the test's warm cache: what the first judge's build leaves in the judge cache is what is proved.
+	t.Setenv("WARM_GO_CACHE", "")
 	runs := filepath.Join(t.TempDir(), "runs")
 	t.Setenv("RUNS", runs)
 	tmp := t.TempDir()

internal/builder/warm_go_cache_test.go

@@ -0,0 +1,80 @@
+package builder
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+// **A judge's build in a test starts from a warm Go cache** (novox/hq issue 717). The runtime these tests put on
+// PATH (aRuntimeThatBuildsTheJudge) runs the check's `go build` and `go test` on this machine with the Go cache the
+// check names — a fresh directory in every test — so each of them compiled the standard library from nothing: some
+// ten CPU-seconds a build, under twenty tests, half of the CPU the whole suite spent. The runtime now fills an empty
+// Go cache it is given from one this process warmed once, before it runs `go`; what the check names, mounts and
+// keeps is unchanged, and so is what each test reads of it. A cache entry is keyed by everything the compile
+// depends on, flags included, so a build with a planted -toolexec hook still misses it and still runs its hook.
+//
+// The warm cache lives beside the system's temporary directory as it was when the package started, never under a
+// TMPDIR a test set and then asserts left empty, and is removed when the package's tests end.
+var (
+	warmCacheBase = os.TempDir()
+	warmCacheOnce sync.Once
+	warmCacheDir  string
+)
+
+func TestMain(m *testing.M) {
+	code := m.Run()
+	if warmCacheDir != "" {
+		_ = os.RemoveAll(warmCacheDir)
+	}
+	os.Exit(code)
+}
+
+// warmGoCache is a Go cache holding what a judge's build and the replays' `go test` compile of the standard
+// library, or "" when it cannot be warmed — a test then builds cold, as before, and is only slower.
+func warmGoCache(t *testing.T) string {
+	t.Helper()
+	warmCacheOnce.Do(func() {
+		dir, err := os.MkdirTemp(warmCacheBase, "judge-warm-go-cache-")
+		if err != nil {
+			return
+		}
+		src := filepath.Join(dir, "src")
+		files := map[string]string{
+			"go.mod":       "module example.org/warm\n\ngo 1.22\n",
+			"main.go":      "package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n)\n\nfunc main() { fmt.Fprintln(os.Stdout, len(os.Args)) }\n",
+			"main_test.go": "package main\n\nimport \"testing\"\n\nfunc TestWarm(t *testing.T) {}\n",
+		}
+		for name, body := range files {
+			if err := os.MkdirAll(src, 0o755); err != nil {
+				return
+			}
+			if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil {
+				return
+			}
+		}
+		cache := filepath.Join(dir, "go-cache")
+		env := append(os.Environ(), "GOCACHE="+cache, "GOENV=off", "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off",
+			"GOFLAGS=-buildvcs=false", "TMPDIR="+warmCacheBase)
+		// As the check builds a judge (CGO off) and runs the replays (CGO as the toolchain has it).
+		for _, run := range [][]string{
+			{"env", "CGO_ENABLED=0", "go", "build", "-o", filepath.Join(dir, "judge"), "."},
+			{"go", "test", "-count=1", "."},
+		} {
+			cmd := exec.Command(run[0], run[1:]...)
+			cmd.Dir, cmd.Env = src, env
+			if out, err := cmd.CombinedOutput(); err != nil {
+				t.Logf("the warm Go cache could not be built, so judges build cold: %v\n%s", err, out)
+				_ = os.RemoveAll(dir)
+				return
+			}
+		}
+		warmCacheDir = dir
+	})
+	if warmCacheDir == "" {
+		return ""
+	}
+	return filepath.Join(warmCacheDir, "go-cache")
+}

internal/lease/lease_test.go

@@ -22,6 +22,9 @@ import (
 // refused is a loss at once; and a bucket raised again from nothing never issues an epoch twice.
 //
 //	go test ./internal/lease/   (each test on a bus of its own: internal/testbus)
+//
+// Each test runs beside the others (novox/hq issue 717): they share nothing but the clock, and wait on it
+// most of their time, so one after another they took half a minute of a check.
 
 // The bounds in these tests are the design's divided by five, so the cases run in seconds.
 const (
@@ -95,6 +98,7 @@ func takeWithin(t *testing.T, l *Lease) <-chan uint64 {
 }
 
 func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil)
 
@@ -148,6 +152,7 @@ func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) {
 }
 
 func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil)
 	epochA, err := a.Take(t.Context())
@@ -193,6 +198,7 @@ func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) {
 }
 
 func TestARenewalRefusedIsALossAtOnce(t *testing.T) {
+	t.Parallel()
 	js, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	if _, err := a.Take(t.Context()); err != nil {
@@ -226,6 +232,7 @@ func TestARenewalRefusedIsALossAtOnce(t *testing.T) {
 }
 
 func TestKeepingRenewsAcrossManyAges(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	epoch, err := a.Take(t.Context())
@@ -249,6 +256,7 @@ func TestKeepingRenewsAcrossManyAges(t *testing.T) {
 }
 
 func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	// The mesh has issued epoch 500 before: its store says so. The bucket is new — a bus whose data
 	// was replaced — and its revisions start at one.
@@ -275,6 +283,7 @@ func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) {
 }
 
 func TestALeaseBucketWithoutAnAgeIsRefused(t *testing.T) {
+	t.Parallel()
 	url := testbus.URL(t)
 	conn, err := nats.Connect(url)
 	if err != nil {
@@ -339,6 +348,7 @@ func delays(l *Lease) <-chan error {
 }
 
 func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	told := delays(a)
@@ -390,6 +400,7 @@ func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) {
 }
 
 func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	bus := stallable(a, false)
@@ -426,6 +437,7 @@ func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) {
 }
 
 func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	bus := stallable(a, true)
@@ -460,6 +472,7 @@ func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) {
 }
 
 func TestAnotherHolderSeenAfterAStallIsALossAtOnce(t *testing.T) {
+	t.Parallel()
 	js, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	bus := stallable(a, false)

internal/link/builds_nats.go

@@ -332,7 +332,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 			// as the controller says it for its own long handlers, so the server neither hands
 			// the ask to a second machine nor counts the wait against its deliveries.
 			working := make(chan struct{})
-			go stillWorking(msg, working)
+			go stillWorking(msg, working, heartbeatWhileWorking)
 			do(ctx, build)
 			close(working)
 		}

internal/link/receive_nats.go

@@ -220,21 +220,22 @@ func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Con
 		// stops saying so, and the bus redelivers exactly as it should.
 		working := make(chan struct{})
 		defer close(working)
-		go stillWorking(msg, working)
+		go stillWorking(msg, working, heartbeatWhileWorking)
 	}
 	act(ctx, m)
 }
 
 // heartbeatWhileWorking is how often a handler still running tells the bus so — comfortably inside
-// the shortest acknowledgement window the mesh gives any of its consumers.
-const heartbeatWhileWorking = 10 * time.Second
+// the shortest acknowledgement window the mesh gives any of its consumers. A variable only so a test can
+// outlive a window of a second in a few (novox/hq issue 717); nothing else sets it.
+var heartbeatWhileWorking = 10 * time.Second
 
 // stillWorking keeps one message alive until the work on it returns.
 //
 // An error is not worth reporting: what the bus does when it is not told is redeliver, which is
 // exactly what happens if this fails, and the handler's own outcome is the thing worth logging.
-func stillWorking(msg *nats.Msg, done <-chan struct{}) {
-	tick := time.NewTicker(heartbeatWhileWorking)
+func stillWorking(msg *nats.Msg, done <-chan struct{}, every time.Duration) {
+	tick := time.NewTicker(every)
 	defer tick.Stop()
 	for {
 		select {

internal/link/receive_nats_test.go

@@ -405,16 +405,15 @@ func TestEverySubjectTheControllerFollowsDecodesToAKind(t *testing.T) {
 // The bus waits a fixed time to be told a message was taken and then redelivers, which is right for
 // a consumer that died and wrong for one that is busy. Acting on a merge builds modules — minutes
 // against a thirty-second window — and the same merge was handed over five times while the first
-// build was still running (2026-09-28). Here the window is two seconds and the work takes six.
+// build was still running (2026-09-28). Here the window is a second and the work takes three.
 func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
+	// Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq
+	// issue 717: the test once set its window before the serving loop asserted the controller's own of thirty
+	// seconds, so it waited nine seconds and passed with the heartbeat removed).
+	before := heartbeatWhileWorking
+	heartbeatWhileWorking = 250 * time.Millisecond
+	t.Cleanup(func() { heartbeatWhileWorking = before })
 	js := aBus(t)
-	// The controller's own consumer, with a window short enough to outlive in a test.
-	if err := js.EnsureConsumer(broker.Consumer{
-		Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 2,
-		Why: "a window short enough to outlive in a test",
-	}); err != nil {
-		t.Fatal(err)
-	}
 	var mu sync.Mutex
 	handled := 0
 	slow := make(chan struct{})
@@ -426,10 +425,32 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
 		first := handled == 1
 		mu.Unlock()
 		if first {
-			time.Sleep(6 * time.Second)
+			time.Sleep(3 * time.Second)
 			close(slow)
 		}
 	}}
+	// The controller's own consumer, given a window short enough to outlive in a test once the serving loop has
+	// asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds,
+	// which is how this test once waited out a window it never had.
+	for deadline := time.Now().Add(10 * time.Second); ; {
+		info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
+		if err == nil && info.PushBound {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("the serving loop never bound the controller's consumer: %v", err)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err := js.EnsureConsumer(broker.Consumer{
+		Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1,
+		Why: "a window short enough to outlive in a test",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second {
+		t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err)
+	}
 
 	body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
 	if err != nil {
@@ -444,12 +465,21 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
 		t.Fatal("the slow work never finished")
 	}
 	// A moment for a redelivery to arrive, if the bus were going to send one.
-	time.Sleep(3 * time.Second)
+	time.Sleep(1500 * time.Millisecond)
 	mu.Lock()
 	defer mu.Unlock()
 	if handled != 1 {
 		t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled)
 	}
+	// Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone
+	// would pass on a bus that redelivered the work every window; the bus's own count says it did not.
+	info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.NumRedelivered != 0 {
+		t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered)
+	}
 }
 
 // slowly is a listener that runs whatever it was given.

merge-check.sh

@@ -54,13 +54,16 @@ if [ -n "$unformatted" ]; then
 fi
 CGO_ENABLED=0 go vet ./...
 
+# The race detector when the toolchain has a C compiler, and said when it has not.
+if command -v gcc >/dev/null 2>&1; then
+	cgo=1 race=-race
+else
+	echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
+	cgo=0 race=
+fi
 suite() {
-	if command -v gcc >/dev/null 2>&1; then
-		CGO_ENABLED=1 go test -race -count=1 -timeout 30m "$@"
-	else
-		echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
-		CGO_ENABLED=0 go test -count=1 -timeout 30m "$@"
-	fi
+	# shellcheck disable=SC2086 # $race is one flag or none
+	CGO_ENABLED=$cgo go test $race -count=1 -timeout 30m "$@"
 }
 # **A branch tried before its pull request may ask for a part** (novox/hq issue 686): the controller's `try` verb
 # hands MESH_CHECK_TESTS, a `go test -run` pattern, or MESH_CHECK_STORE_ONLY=1. A pull request's merge check is
@@ -70,17 +73,102 @@ if [ -n "${MESH_CHECK_TESTS:-}" ]; then
 	suite -run "$MESH_CHECK_TESTS" ./...
 	exit 0
 fi
-if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then
-	echo "TRIED IN PART: only the store tests listed in testdata/store-tests"
-else
-	suite ./...
-fi
+
+# **The slowest packages run in shards** (novox/hq issue 717). `go test` runs packages beside each other but the
+# tests of one package one after another, in one process, and these packages' tests wait on the store, a bus, a
+# build or a clock: cmd/mesh-controller alone took 3 to 5 minutes of a check whose other packages end in seconds.
+# Each is compiled once, its tests are dealt round-robin into that many processes, and those run beside each other
+# and beside the rest of the suite. Each process is the package's own test binary, run from the package's
+# directory through its TestMain as `go test` runs it, so a test cannot tell, and every test runs exactly once, in
+# one of them. Tests of one package therefore keep what packages run side by side already keep: nothing shared
+# outside their process — no fixed port, file or database name. A package is added here by measuring it
+# (`go test -json` gives each test's time), never to hide a slow test that could wait less.
+sharded='./cmd/mesh-controller 8
+./internal/inventory 4
+./internal/builder 4
+./internal/link 4'
+
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+
+# shard_package compiles one package's tests and runs them in $2 processes, each process's output and
+# "<exit status> <seconds>" left in $work: $1 the package's directory, $2 the number of processes.
+shard_package() {
+	dir=$1 shards=$2
+	key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _)
+	bin="$work/$key.test"
+	status=0
+	# shellcheck disable=SC2086 # $race is one flag or none
+	CGO_ENABLED=$cgo go test $race -c -o "$bin" "$dir" >"$work/$key.build" 2>&1 || status=$?
+	# Every test, example and fuzz target `go test` would run, and nothing a TestMain might print beside them. A
+	# binary that cannot list them, or lists none, fails here rather than pass by running nothing.
+	if [ "$status" -eq 0 ]; then
+		(cd "$dir" && "$bin" -test.list '.*') >"$work/$key.list" 2>&1 || status=$?
+		names=$(grep -E '^(Test|Example|Fuzz)' "$work/$key.list" || true)
+		if [ "$status" -ne 0 ] || [ -z "$names" ]; then
+			{
+				cat "$work/$key.list"
+				echo "the test binary of $dir listed no tests to deal into shards (exit status $status)"
+			} >>"$work/$key.build"
+			status=1
+		fi
+	fi
+	echo "$status" >"$work/$key.built"
+	[ "$status" -eq 0 ] || return 0
+	i=0
+	while [ "$i" -lt "$shards" ]; do
+		pattern=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | paste -sd '|' -)
+		count=$(printf '%s\n' "$names" | awk -v i="$i" -v k="$shards" 'NF && (NR - 1) % k == i' | wc -l)
+		if [ -n "$pattern" ]; then
+			(
+				start=$(date +%s)
+				status=0
+				cd "$dir" && "$bin" -test.count=1 -test.timeout=30m -test.paniconexit0 -test.run "^($pattern)\$" \
+					>"$work/$key.$i.out" 2>&1 || status=$?
+				echo "$status $(($(date +%s) - start)) $count" >"$work/$key.$i.status"
+			) &
+		fi
+		i=$((i + 1))
+	done
+	wait
+}
+
+# say_shards prints how each sharded package's processes ended, as `go test` says a package, and the output of
+# each that failed; it fails when any did.
+say_shards() {
+	failed=0
+	module=$(go list -m)
+	for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do
+		dir=${entry%%|*} shards=${entry#*|}
+		key=$(printf '%s' "$dir" | tr -c 'A-Za-z0-9' _)
+		pkg="$module/${dir#./}"
+		if [ "$(cat "$work/$key.built" 2>/dev/null || echo 1)" -ne 0 ]; then
+			cat "$work/$key.build"
+			printf 'FAIL\t%s [build failed]\n' "$pkg"
+			failed=1
+			continue
+		fi
+		i=0
+		while [ "$i" -lt "$shards" ]; do
+			if [ -f "$work/$key.$i.status" ]; then
+				read -r status took count <"$work/$key.$i.status"
+				if [ "$status" -eq 0 ]; then
+					printf 'ok  \t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count"
+				else
+					cat "$work/$key.$i.out"
+					printf 'FAIL\t%s\t%ss\t(shard %d of %d: %d tests)\n' "$pkg" "$took" $((i + 1)) "$shards" "$count"
+					failed=1
+				fi
+			fi
+			i=$((i + 1))
+		done
+	done
+	return "$failed"
+}
 
 # **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip
 # passes unseen. With the store this check is given, every test named in testdata/store-tests must have run against
 # it and passed, each said by name; without one, that is said. A pull request adds its own store tests to the list.
-
-
 listed=$(grep -v '^[[:space:]]*\(#\|$\)' testdata/store-tests || true)
 # Each line is exactly `<package> <TestName>`: a line of one field ran nothing and said nothing (novox/hq issue 515).
 malformed=$(printf '%s\n' "$listed" | awk 'NF != 2 && NF != 0' || true)
@@ -107,20 +195,55 @@ for entry in $(git show origin/main:testdata/store-tests 2>/dev/null | grep -v '
 done
 [ "$dropped" -eq 0 ] || exit 1
 
+# The listed store tests run beside the suite, one process per package (novox/hq issue 717): each package's output
+# and exit status are left in $work and judged once the suite is done.
+store_packages=$(printf '%s\n' "$listed" | awk 'NF {print $1}' | sort -u)
+if [ -n "$listed" ] && [ -n "${MESH_TEST_POSTGRES:-}" ]; then
+	for pkg in $store_packages; do
+		key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _)
+		names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
+		pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$"
+		(
+			status=0
+			CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$work/store.$key.out" 2>&1 || status=$?
+			echo "$status" >"$work/store.$key.status"
+		) &
+	done
+fi
+
+suite_failed=0
+if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then
+	echo "TRIED IN PART: only the store tests listed in testdata/store-tests"
+	wait
+else
+	module=$(go list -m)
+	skip=$(printf '%s\n' "$sharded" | awk -v m="$module" '{sub(/^\.\//, "", $1); print m "/" $1}')
+	rest=$(go list ./... | grep -vxF "$skip")
+	(
+		status=0
+		# shellcheck disable=SC2086 # the packages are one word each, split on purpose
+		suite $rest >"$work/rest.out" 2>&1 || status=$?
+		echo "$status" >"$work/rest.status"
+	) &
+	for entry in $(printf '%s\n' "$sharded" | awk '{print $1 "|" $2}'); do
+		shard_package "${entry%%|*}" "${entry#*|}" &
+	done
+	wait
+	cat "$work/rest.out"
+	[ "$(cat "$work/rest.status")" -eq 0 ] || suite_failed=1
+	say_shards || suite_failed=1
+fi
+
 if [ -z "$listed" ]; then
 	echo "store tests: none listed in testdata/store-tests"
 elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then
 	echo "store tests, each run against the store:"
 	missing=0
-	out=$(mktemp)
-	trap 'rm -f "$out"' EXIT
-	for pkg in $(printf '%s\n' "$listed" | awk '{print $1}' | sort -u); do
+	for pkg in $store_packages; do
+		key=$(printf '%s' "$pkg" | tr -c 'A-Za-z0-9' _)
 		names=$(printf '%s\n' "$listed" | awk -v p="$pkg" '$1 == p {print $2}')
-		pattern="^($(printf '%s\n' "$names" | paste -sd '|' -))\$"
-		status=0
-		CGO_ENABLED=0 go test -count=1 -v -run "$pattern" "$pkg" >"$out" 2>&1 || status=$?
 		# shellcheck disable=SC2086 # the names are one word each, split on purpose
-		judge_store_tests "$pkg" "$status" "$out" $names
+		judge_store_tests "$pkg" "$(cat "$work/store.$key.status")" "$work/store.$key.out" $names
 	done
 	if [ "$missing" -ne 0 ]; then
 		echo "$missing store test(s) listed in testdata/store-tests did not run and pass against the store"
@@ -129,3 +252,7 @@ elif [ -n "${MESH_TEST_POSTGRES:-}" ]; then
 else
 	echo "NOT STORE-CHECKED: no MESH_TEST_POSTGRES, so the store tests listed in testdata/store-tests skipped"
 fi
+if [ "$suite_failed" -ne 0 ]; then
+	echo "the suite failed: each package or shard that failed is said above with its output"
+	exit 1
+fi