Run the slow test packages in shards and stop compiling the standard library per judge test (hq issue 717)

The controller's check spent 3-5 minutes in cmd/mesh-controller, whose few hundred store tests run one
after another, while most packages end in seconds. merge-check.sh now compiles each slow package once and
deals its tests round-robin into processes that run beside each other and beside the rest of the suite;
every test still runs once, under the race detector, through its package's TestMain. The listed store
tests run beside the suite instead of after it.

internal/builder's judge tests ran a real go build with a fresh Go cache each, compiling the standard
library every time: half the suite's CPU. The fake runtime fills an empty cache from one warmed per
process. The lease tests run in parallel. The link test of slow work now sets its one-second window after
the serving loop asserted the controller's thirty-second one, so it proves the heartbeat (it passed
without it) in half the time.

internal/builder/release_branch_test.go

@@ -156,6 +156,9 @@ run)
   if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS" ]; then
     echo "HOME=$HOME GOCACHE=$GOCACHE GOMODCACHE=$GOMODCACHE GOENV=$GOENV GOFLAGS=$GOFLAGS DIR=$workdir" >> "$REPLAYS"
   fi
+  if [ "$1" = go ] && [ -n "$WARM_GO_CACHE" ] && [ -n "$GOCACHE" ] && [ -z "$(ls -A "$GOCACHE" 2>/dev/null)" ]; then
+    mkdir -p "$GOCACHE" && cp -R "$WARM_GO_CACHE/." "$GOCACHE/"
+  fi
   cd "$workdir" && exec "$@" ;;
 esac
 exit 1
@@ -165,6 +168,8 @@ exit 1
 	}
 	t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
 	t.Setenv("GOTOOLCHAIN", "local")
+	// An empty Go cache the check names is filled from a warm one before `go` runs (warm_go_cache_test.go).
+	t.Setenv("WARM_GO_CACHE", warmGoCache(t))
 	return built
 }
 
@@ -566,6 +571,8 @@ func TestTheBuildSeatNeverReadsMeshLabAtAReleaseBranch(t *testing.T) {
 // a temporary directory of the host; and no container a check runs for the pull request mounts the judge cache.
 func TestJudgesAreBuiltWarmFromAJudgeCacheNoCheckContainerMounts(t *testing.T) {
 	built := aRuntimeThatBuildsTheJudge(t)
+	// Not filled from the test's warm cache: what the first judge's build leaves in the judge cache is what is proved.
+	t.Setenv("WARM_GO_CACHE", "")
 	runs := filepath.Join(t.TempDir(), "runs")
 	t.Setenv("RUNS", runs)
 	tmp := t.TempDir()

internal/builder/warm_go_cache_test.go

@@ -0,0 +1,80 @@
+package builder
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+// **A judge's build in a test starts from a warm Go cache** (novox/hq issue 717). The runtime these tests put on
+// PATH (aRuntimeThatBuildsTheJudge) runs the check's `go build` and `go test` on this machine with the Go cache the
+// check names — a fresh directory in every test — so each of them compiled the standard library from nothing: some
+// ten CPU-seconds a build, under twenty tests, half of the CPU the whole suite spent. The runtime now fills an empty
+// Go cache it is given from one this process warmed once, before it runs `go`; what the check names, mounts and
+// keeps is unchanged, and so is what each test reads of it. A cache entry is keyed by everything the compile
+// depends on, flags included, so a build with a planted -toolexec hook still misses it and still runs its hook.
+//
+// The warm cache lives beside the system's temporary directory as it was when the package started, never under a
+// TMPDIR a test set and then asserts left empty, and is removed when the package's tests end.
+var (
+	warmCacheBase = os.TempDir()
+	warmCacheOnce sync.Once
+	warmCacheDir  string
+)
+
+func TestMain(m *testing.M) {
+	code := m.Run()
+	if warmCacheDir != "" {
+		_ = os.RemoveAll(warmCacheDir)
+	}
+	os.Exit(code)
+}
+
+// warmGoCache is a Go cache holding what a judge's build and the replays' `go test` compile of the standard
+// library, or "" when it cannot be warmed — a test then builds cold, as before, and is only slower.
+func warmGoCache(t *testing.T) string {
+	t.Helper()
+	warmCacheOnce.Do(func() {
+		dir, err := os.MkdirTemp(warmCacheBase, "judge-warm-go-cache-")
+		if err != nil {
+			return
+		}
+		src := filepath.Join(dir, "src")
+		files := map[string]string{
+			"go.mod":       "module example.org/warm\n\ngo 1.22\n",
+			"main.go":      "package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n)\n\nfunc main() { fmt.Fprintln(os.Stdout, len(os.Args)) }\n",
+			"main_test.go": "package main\n\nimport \"testing\"\n\nfunc TestWarm(t *testing.T) {}\n",
+		}
+		for name, body := range files {
+			if err := os.MkdirAll(src, 0o755); err != nil {
+				return
+			}
+			if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil {
+				return
+			}
+		}
+		cache := filepath.Join(dir, "go-cache")
+		env := append(os.Environ(), "GOCACHE="+cache, "GOENV=off", "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off",
+			"GOFLAGS=-buildvcs=false", "TMPDIR="+warmCacheBase)
+		// As the check builds a judge (CGO off) and runs the replays (CGO as the toolchain has it).
+		for _, run := range [][]string{
+			{"env", "CGO_ENABLED=0", "go", "build", "-o", filepath.Join(dir, "judge"), "."},
+			{"go", "test", "-count=1", "."},
+		} {
+			cmd := exec.Command(run[0], run[1:]...)
+			cmd.Dir, cmd.Env = src, env
+			if out, err := cmd.CombinedOutput(); err != nil {
+				t.Logf("the warm Go cache could not be built, so judges build cold: %v\n%s", err, out)
+				_ = os.RemoveAll(dir)
+				return
+			}
+		}
+		warmCacheDir = dir
+	})
+	if warmCacheDir == "" {
+		return ""
+	}
+	return filepath.Join(warmCacheDir, "go-cache")
+}

internal/lease/lease_test.go

@@ -22,6 +22,9 @@ import (
 // refused is a loss at once; and a bucket raised again from nothing never issues an epoch twice.
 //
 //	go test ./internal/lease/   (each test on a bus of its own: internal/testbus)
+//
+// Each test runs beside the others (novox/hq issue 717): they share nothing but the clock, and wait on it
+// most of their time, so one after another they took half a minute of a check.
 
 // The bounds in these tests are the design's divided by five, so the cases run in seconds.
 const (
@@ -95,6 +98,7 @@ func takeWithin(t *testing.T, l *Lease) <-chan uint64 {
 }
 
 func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil)
 
@@ -148,6 +152,7 @@ func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) {
 }
 
 func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil)
 	epochA, err := a.Take(t.Context())
@@ -193,6 +198,7 @@ func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) {
 }
 
 func TestARenewalRefusedIsALossAtOnce(t *testing.T) {
+	t.Parallel()
 	js, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	if _, err := a.Take(t.Context()); err != nil {
@@ -226,6 +232,7 @@ func TestARenewalRefusedIsALossAtOnce(t *testing.T) {
 }
 
 func TestKeepingRenewsAcrossManyAges(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	epoch, err := a.Take(t.Context())
@@ -249,6 +256,7 @@ func TestKeepingRenewsAcrossManyAges(t *testing.T) {
 }
 
 func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	// The mesh has issued epoch 500 before: its store says so. The bucket is new — a bus whose data
 	// was replaced — and its revisions start at one.
@@ -275,6 +283,7 @@ func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) {
 }
 
 func TestALeaseBucketWithoutAnAgeIsRefused(t *testing.T) {
+	t.Parallel()
 	url := testbus.URL(t)
 	conn, err := nats.Connect(url)
 	if err != nil {
@@ -339,6 +348,7 @@ func delays(l *Lease) <-chan error {
 }
 
 func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	told := delays(a)
@@ -390,6 +400,7 @@ func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) {
 }
 
 func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	bus := stallable(a, false)
@@ -426,6 +437,7 @@ func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) {
 }
 
 func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	bus := stallable(a, true)
@@ -460,6 +472,7 @@ func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) {
 }
 
 func TestAnotherHolderSeenAfterAStallIsALossAtOnce(t *testing.T) {
+	t.Parallel()
 	js, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	bus := stallable(a, false)

internal/link/builds_nats.go

@@ -332,7 +332,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 			// as the controller says it for its own long handlers, so the server neither hands
 			// the ask to a second machine nor counts the wait against its deliveries.
 			working := make(chan struct{})
-			go stillWorking(msg, working)
+			go stillWorking(msg, working, heartbeatWhileWorking)
 			do(ctx, build)
 			close(working)
 		}

internal/link/receive_nats.go

@@ -220,21 +220,22 @@ func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Con
 		// stops saying so, and the bus redelivers exactly as it should.
 		working := make(chan struct{})
 		defer close(working)
-		go stillWorking(msg, working)
+		go stillWorking(msg, working, heartbeatWhileWorking)
 	}
 	act(ctx, m)
 }
 
 // heartbeatWhileWorking is how often a handler still running tells the bus so — comfortably inside
-// the shortest acknowledgement window the mesh gives any of its consumers.
-const heartbeatWhileWorking = 10 * time.Second
+// the shortest acknowledgement window the mesh gives any of its consumers. A variable only so a test can
+// outlive a window of a second in a few (novox/hq issue 717); nothing else sets it.
+var heartbeatWhileWorking = 10 * time.Second
 
 // stillWorking keeps one message alive until the work on it returns.
 //
 // An error is not worth reporting: what the bus does when it is not told is redeliver, which is
 // exactly what happens if this fails, and the handler's own outcome is the thing worth logging.
-func stillWorking(msg *nats.Msg, done <-chan struct{}) {
-	tick := time.NewTicker(heartbeatWhileWorking)
+func stillWorking(msg *nats.Msg, done <-chan struct{}, every time.Duration) {
+	tick := time.NewTicker(every)
 	defer tick.Stop()
 	for {
 		select {

internal/link/receive_nats_test.go

@@ -405,16 +405,15 @@ func TestEverySubjectTheControllerFollowsDecodesToAKind(t *testing.T) {
 // The bus waits a fixed time to be told a message was taken and then redelivers, which is right for
 // a consumer that died and wrong for one that is busy. Acting on a merge builds modules — minutes
 // against a thirty-second window — and the same merge was handed over five times while the first
-// build was still running (2026-09-28). Here the window is two seconds and the work takes six.
+// build was still running (2026-09-28). Here the window is a second and the work takes three.
 func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
+	// Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq
+	// issue 717: the test once set its window before the serving loop asserted the controller's own of thirty
+	// seconds, so it waited nine seconds and passed with the heartbeat removed).
+	before := heartbeatWhileWorking
+	heartbeatWhileWorking = 250 * time.Millisecond
+	t.Cleanup(func() { heartbeatWhileWorking = before })
 	js := aBus(t)
-	// The controller's own consumer, with a window short enough to outlive in a test.
-	if err := js.EnsureConsumer(broker.Consumer{
-		Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 2,
-		Why: "a window short enough to outlive in a test",
-	}); err != nil {
-		t.Fatal(err)
-	}
 	var mu sync.Mutex
 	handled := 0
 	slow := make(chan struct{})
@@ -426,10 +425,32 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
 		first := handled == 1
 		mu.Unlock()
 		if first {
-			time.Sleep(6 * time.Second)
+			time.Sleep(3 * time.Second)
 			close(slow)
 		}
 	}}
+	// The controller's own consumer, given a window short enough to outlive in a test once the serving loop has
+	// asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds,
+	// which is how this test once waited out a window it never had.
+	for deadline := time.Now().Add(10 * time.Second); ; {
+		info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
+		if err == nil && info.PushBound {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("the serving loop never bound the controller's consumer: %v", err)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err := js.EnsureConsumer(broker.Consumer{
+		Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1,
+		Why: "a window short enough to outlive in a test",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second {
+		t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err)
+	}
 
 	body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
 	if err != nil {
@@ -444,12 +465,21 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
 		t.Fatal("the slow work never finished")
 	}
 	// A moment for a redelivery to arrive, if the bus were going to send one.
-	time.Sleep(3 * time.Second)
+	time.Sleep(1500 * time.Millisecond)
 	mu.Lock()
 	defer mu.Unlock()
 	if handled != 1 {
 		t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled)
 	}
+	// Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone
+	// would pass on a bus that redelivered the work every window; the bus's own count says it did not.
+	info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.NumRedelivered != 0 {
+		t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered)
+	}
 }
 
 // slowly is a listener that runs whatever it was given.