Merge pull request 'Run the slow test packages in shards and build judges from a warm Go cache (hq issue 717)' (#334) from fix/717-the-suite-runs-in-under-two-minutes into main

internal/builder/release_branch_test.go

@@ -156,6 +156,9 @@ run)
   if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS" ]; then
     echo "HOME=$HOME GOCACHE=$GOCACHE GOMODCACHE=$GOMODCACHE GOENV=$GOENV GOFLAGS=$GOFLAGS DIR=$workdir" >> "$REPLAYS"
   fi
+  if [ "$1" = go ] && [ -n "$WARM_GO_CACHE" ] && [ -n "$GOCACHE" ] && [ -z "$(ls -A "$GOCACHE" 2>/dev/null)" ]; then
+    mkdir -p "$GOCACHE" && cp -R "$WARM_GO_CACHE/." "$GOCACHE/"
+  fi
   cd "$workdir" && exec "$@" ;;
 esac
 exit 1
@@ -165,6 +168,8 @@ exit 1
 	}
 	t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
 	t.Setenv("GOTOOLCHAIN", "local")
+	// An empty Go cache the check names is filled from a warm one before `go` runs (warm_go_cache_test.go).
+	t.Setenv("WARM_GO_CACHE", warmGoCache(t))
 	return built
 }
 
@@ -566,6 +571,8 @@ func TestTheBuildSeatNeverReadsMeshLabAtAReleaseBranch(t *testing.T) {
 // a temporary directory of the host; and no container a check runs for the pull request mounts the judge cache.
 func TestJudgesAreBuiltWarmFromAJudgeCacheNoCheckContainerMounts(t *testing.T) {
 	built := aRuntimeThatBuildsTheJudge(t)
+	// Not filled from the test's warm cache: what the first judge's build leaves in the judge cache is what is proved.
+	t.Setenv("WARM_GO_CACHE", "")
 	runs := filepath.Join(t.TempDir(), "runs")
 	t.Setenv("RUNS", runs)
 	tmp := t.TempDir()

internal/builder/warm_go_cache_test.go

@@ -0,0 +1,80 @@
+package builder
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+// **A judge's build in a test starts from a warm Go cache** (novox/hq issue 717). The runtime these tests put on
+// PATH (aRuntimeThatBuildsTheJudge) runs the check's `go build` and `go test` on this machine with the Go cache the
+// check names — a fresh directory in every test — so each of them compiled the standard library from nothing: some
+// ten CPU-seconds a build, under twenty tests, half of the CPU the whole suite spent. The runtime now fills an empty
+// Go cache it is given from one this process warmed once, before it runs `go`; what the check names, mounts and
+// keeps is unchanged, and so is what each test reads of it. A cache entry is keyed by everything the compile
+// depends on, flags included, so a build with a planted -toolexec hook still misses it and still runs its hook.
+//
+// The warm cache lives beside the system's temporary directory as it was when the package started, never under a
+// TMPDIR a test set and then asserts left empty, and is removed when the package's tests end.
+var (
+	warmCacheBase = os.TempDir()
+	warmCacheOnce sync.Once
+	warmCacheDir  string
+)
+
+func TestMain(m *testing.M) {
+	code := m.Run()
+	if warmCacheDir != "" {
+		_ = os.RemoveAll(warmCacheDir)
+	}
+	os.Exit(code)
+}
+
+// warmGoCache is a Go cache holding what a judge's build and the replays' `go test` compile of the standard
+// library, or "" when it cannot be warmed — a test then builds cold, as before, and is only slower.
+func warmGoCache(t *testing.T) string {
+	t.Helper()
+	warmCacheOnce.Do(func() {
+		dir, err := os.MkdirTemp(warmCacheBase, "judge-warm-go-cache-")
+		if err != nil {
+			return
+		}
+		src := filepath.Join(dir, "src")
+		files := map[string]string{
+			"go.mod":       "module example.org/warm\n\ngo 1.22\n",
+			"main.go":      "package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n)\n\nfunc main() { fmt.Fprintln(os.Stdout, len(os.Args)) }\n",
+			"main_test.go": "package main\n\nimport \"testing\"\n\nfunc TestWarm(t *testing.T) {}\n",
+		}
+		for name, body := range files {
+			if err := os.MkdirAll(src, 0o755); err != nil {
+				return
+			}
+			if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil {
+				return
+			}
+		}
+		cache := filepath.Join(dir, "go-cache")
+		env := append(os.Environ(), "GOCACHE="+cache, "GOENV=off", "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off",
+			"GOFLAGS=-buildvcs=false", "TMPDIR="+warmCacheBase)
+		// As the check builds a judge (CGO off) and runs the replays (CGO as the toolchain has it).
+		for _, run := range [][]string{
+			{"env", "CGO_ENABLED=0", "go", "build", "-o", filepath.Join(dir, "judge"), "."},
+			{"go", "test", "-count=1", "."},
+		} {
+			cmd := exec.Command(run[0], run[1:]...)
+			cmd.Dir, cmd.Env = src, env
+			if out, err := cmd.CombinedOutput(); err != nil {
+				t.Logf("the warm Go cache could not be built, so judges build cold: %v\n%s", err, out)
+				_ = os.RemoveAll(dir)
+				return
+			}
+		}
+		warmCacheDir = dir
+	})
+	if warmCacheDir == "" {
+		return ""
+	}
+	return filepath.Join(warmCacheDir, "go-cache")
+}

internal/lease/lease_test.go

@@ -22,6 +22,9 @@ import (
 // refused is a loss at once; and a bucket raised again from nothing never issues an epoch twice.
 //
 //	go test ./internal/lease/   (each test on a bus of its own: internal/testbus)
+//
+// Each test runs beside the others (novox/hq issue 717): they share nothing but the clock, and wait on it
+// most of their time, so one after another they took half a minute of a check.
 
 // The bounds in these tests are the design's divided by five, so the cases run in seconds.
 const (
@@ -95,6 +98,7 @@ func takeWithin(t *testing.T, l *Lease) <-chan uint64 {
 }
 
 func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil)
 
@@ -148,6 +152,7 @@ func TestTheSecondControllerWaitsAndTakesAHigherEpochOnHandover(t *testing.T) {
 }
 
 func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a, b := aController(t, bucket, "a", nil), aController(t, bucket, "b", nil)
 	epochA, err := a.Take(t.Context())
@@ -193,6 +198,7 @@ func TestAHolderThatStopsRenewingStopsActingBeforeItIsTakenOver(t *testing.T) {
 }
 
 func TestARenewalRefusedIsALossAtOnce(t *testing.T) {
+	t.Parallel()
 	js, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	if _, err := a.Take(t.Context()); err != nil {
@@ -226,6 +232,7 @@ func TestARenewalRefusedIsALossAtOnce(t *testing.T) {
 }
 
 func TestKeepingRenewsAcrossManyAges(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	epoch, err := a.Take(t.Context())
@@ -249,6 +256,7 @@ func TestKeepingRenewsAcrossManyAges(t *testing.T) {
 }
 
 func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	// The mesh has issued epoch 500 before: its store says so. The bucket is new — a bus whose data
 	// was replaced — and its revisions start at one.
@@ -275,6 +283,7 @@ func TestAnEpochIsNeverIssuedTwiceWhenTheBucketStartsOver(t *testing.T) {
 }
 
 func TestALeaseBucketWithoutAnAgeIsRefused(t *testing.T) {
+	t.Parallel()
 	url := testbus.URL(t)
 	conn, err := nats.Connect(url)
 	if err != nil {
@@ -339,6 +348,7 @@ func delays(l *Lease) <-chan error {
 }
 
 func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	told := delays(a)
@@ -390,6 +400,7 @@ func TestAFailedRenewalKeepsTheLeaseUntilItsTTL(t *testing.T) {
 }
 
 func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	bus := stallable(a, false)
@@ -426,6 +437,7 @@ func TestARenewalMadeAfterAStallOpensTheGateAgain(t *testing.T) {
 }
 
 func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) {
+	t.Parallel()
 	_, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	bus := stallable(a, true)
@@ -460,6 +472,7 @@ func TestARenewalWrittenThoughItsAnswerWasLostIsStillTheHolders(t *testing.T) {
 }
 
 func TestAnotherHolderSeenAfterAStallIsALossAtOnce(t *testing.T) {
+	t.Parallel()
 	js, bucket := aBucket(t)
 	a := aController(t, bucket, "a", nil)
 	bus := stallable(a, false)

internal/link/builds_nats.go

@@ -332,7 +332,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 			// as the controller says it for its own long handlers, so the server neither hands
 			// the ask to a second machine nor counts the wait against its deliveries.
 			working := make(chan struct{})
-			go stillWorking(msg, working)
+			go stillWorking(msg, working, heartbeatWhileWorking)
 			do(ctx, build)
 			close(working)
 		}

internal/link/receive_nats.go

@@ -220,21 +220,22 @@ func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Con
 		// stops saying so, and the bus redelivers exactly as it should.
 		working := make(chan struct{})
 		defer close(working)
-		go stillWorking(msg, working)
+		go stillWorking(msg, working, heartbeatWhileWorking)
 	}
 	act(ctx, m)
 }
 
 // heartbeatWhileWorking is how often a handler still running tells the bus so — comfortably inside
-// the shortest acknowledgement window the mesh gives any of its consumers.
-const heartbeatWhileWorking = 10 * time.Second
+// the shortest acknowledgement window the mesh gives any of its consumers. A variable only so a test can
+// outlive a window of a second in a few (novox/hq issue 717); nothing else sets it.
+var heartbeatWhileWorking = 10 * time.Second
 
 // stillWorking keeps one message alive until the work on it returns.
 //
 // An error is not worth reporting: what the bus does when it is not told is redeliver, which is
 // exactly what happens if this fails, and the handler's own outcome is the thing worth logging.
-func stillWorking(msg *nats.Msg, done <-chan struct{}) {
-	tick := time.NewTicker(heartbeatWhileWorking)
+func stillWorking(msg *nats.Msg, done <-chan struct{}, every time.Duration) {
+	tick := time.NewTicker(every)
 	defer tick.Stop()
 	for {
 		select {

internal/link/receive_nats_test.go

@@ -405,16 +405,15 @@ func TestEverySubjectTheControllerFollowsDecodesToAKind(t *testing.T) {
 // The bus waits a fixed time to be told a message was taken and then redelivers, which is right for
 // a consumer that died and wrong for one that is busy. Acting on a merge builds modules — minutes
 // against a thirty-second window — and the same merge was handed over five times while the first
-// build was still running (2026-09-28). Here the window is two seconds and the work takes six.
+// build was still running (2026-09-28). Here the window is a second and the work takes three.
 func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
+	// Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq
+	// issue 717: the test once set its window before the serving loop asserted the controller's own of thirty
+	// seconds, so it waited nine seconds and passed with the heartbeat removed).
+	before := heartbeatWhileWorking
+	heartbeatWhileWorking = 250 * time.Millisecond
+	t.Cleanup(func() { heartbeatWhileWorking = before })
 	js := aBus(t)
-	// The controller's own consumer, with a window short enough to outlive in a test.
-	if err := js.EnsureConsumer(broker.Consumer{
-		Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 2,
-		Why: "a window short enough to outlive in a test",
-	}); err != nil {
-		t.Fatal(err)
-	}
 	var mu sync.Mutex
 	handled := 0
 	slow := make(chan struct{})
@@ -426,10 +425,32 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
 		first := handled == 1
 		mu.Unlock()
 		if first {
-			time.Sleep(6 * time.Second)
+			time.Sleep(3 * time.Second)
 			close(slow)
 		}
 	}}
+	// The controller's own consumer, given a window short enough to outlive in a test once the serving loop has
+	// asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds,
+	// which is how this test once waited out a window it never had.
+	for deadline := time.Now().Add(10 * time.Second); ; {
+		info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
+		if err == nil && info.PushBound {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("the serving loop never bound the controller's consumer: %v", err)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err := js.EnsureConsumer(broker.Consumer{
+		Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1,
+		Why: "a window short enough to outlive in a test",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second {
+		t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err)
+	}
 
 	body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
 	if err != nil {
@@ -444,12 +465,21 @@ func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
 		t.Fatal("the slow work never finished")
 	}
 	// A moment for a redelivery to arrive, if the bus were going to send one.
-	time.Sleep(3 * time.Second)
+	time.Sleep(1500 * time.Millisecond)
 	mu.Lock()
 	defer mu.Unlock()
 	if handled != 1 {
 		t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled)
 	}
+	// Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone
+	// would pass on a bus that redelivered the work every window; the bus's own count says it did not.
+	info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.NumRedelivered != 0 {
+		t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered)
+	}
 }
 
 // slowly is a listener that runs whatever it was given.