Run the slow test packages in shards and stop compiling the standard library per judge test (hq issue 717)

The controller's check spent 3-5 minutes in cmd/mesh-controller, whose few hundred store tests run one
after another, while most packages end in seconds. merge-check.sh now compiles each slow package once and
deals its tests round-robin into processes that run beside each other and beside the rest of the suite;
every test still runs once, under the race detector, through its package's TestMain. The listed store
tests run beside the suite instead of after it.

internal/builder's judge tests ran a real go build with a fresh Go cache each, compiling the standard
library every time: half the suite's CPU. The fake runtime fills an empty cache from one warmed per
process. The lease tests run in parallel. The link test of slow work now sets its one-second window after
the serving loop asserted the controller's thirty-second one, so it proves the heartbeat (it passed
without it) in half the time.

internal/builder/release_branch_test.go

@@ -156,6 +156,9 @@ run)
   if [ "$1" = go ] && [ "$2" = test ] && [ -n "$REPLAYS" ]; then
     echo "HOME=$HOME GOCACHE=$GOCACHE GOMODCACHE=$GOMODCACHE GOENV=$GOENV GOFLAGS=$GOFLAGS DIR=$workdir" >> "$REPLAYS"
   fi
+  if [ "$1" = go ] && [ -n "$WARM_GO_CACHE" ] && [ -n "$GOCACHE" ] && [ -z "$(ls -A "$GOCACHE" 2>/dev/null)" ]; then
+    mkdir -p "$GOCACHE" && cp -R "$WARM_GO_CACHE/." "$GOCACHE/"
+  fi
   cd "$workdir" && exec "$@" ;;
 esac
 exit 1
@@ -165,6 +168,8 @@ exit 1
 	}
 	t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
 	t.Setenv("GOTOOLCHAIN", "local")
+	// An empty Go cache the check names is filled from a warm one before `go` runs (warm_go_cache_test.go).
+	t.Setenv("WARM_GO_CACHE", warmGoCache(t))
 	return built
 }
 
@@ -566,6 +571,8 @@ func TestTheBuildSeatNeverReadsMeshLabAtAReleaseBranch(t *testing.T) {
 // a temporary directory of the host; and no container a check runs for the pull request mounts the judge cache.
 func TestJudgesAreBuiltWarmFromAJudgeCacheNoCheckContainerMounts(t *testing.T) {
 	built := aRuntimeThatBuildsTheJudge(t)
+	// Not filled from the test's warm cache: what the first judge's build leaves in the judge cache is what is proved.
+	t.Setenv("WARM_GO_CACHE", "")
 	runs := filepath.Join(t.TempDir(), "runs")
 	t.Setenv("RUNS", runs)
 	tmp := t.TempDir()

internal/builder/warm_go_cache_test.go

@@ -0,0 +1,80 @@
+package builder
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+// **A judge's build in a test starts from a warm Go cache** (novox/hq issue 717). The runtime these tests put on
+// PATH (aRuntimeThatBuildsTheJudge) runs the check's `go build` and `go test` on this machine with the Go cache the
+// check names — a fresh directory in every test — so each of them compiled the standard library from nothing: some
+// ten CPU-seconds a build, under twenty tests, half of the CPU the whole suite spent. The runtime now fills an empty
+// Go cache it is given from one this process warmed once, before it runs `go`; what the check names, mounts and
+// keeps is unchanged, and so is what each test reads of it. A cache entry is keyed by everything the compile
+// depends on, flags included, so a build with a planted -toolexec hook still misses it and still runs its hook.
+//
+// The warm cache lives beside the system's temporary directory as it was when the package started, never under a
+// TMPDIR a test set and then asserts left empty, and is removed when the package's tests end.
+var (
+	warmCacheBase = os.TempDir()
+	warmCacheOnce sync.Once
+	warmCacheDir  string
+)
+
+func TestMain(m *testing.M) {
+	code := m.Run()
+	if warmCacheDir != "" {
+		_ = os.RemoveAll(warmCacheDir)
+	}
+	os.Exit(code)
+}
+
+// warmGoCache is a Go cache holding what a judge's build and the replays' `go test` compile of the standard
+// library, or "" when it cannot be warmed — a test then builds cold, as before, and is only slower.
+func warmGoCache(t *testing.T) string {
+	t.Helper()
+	warmCacheOnce.Do(func() {
+		dir, err := os.MkdirTemp(warmCacheBase, "judge-warm-go-cache-")
+		if err != nil {
+			return
+		}
+		src := filepath.Join(dir, "src")
+		files := map[string]string{
+			"go.mod":       "module example.org/warm\n\ngo 1.22\n",
+			"main.go":      "package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n)\n\nfunc main() { fmt.Fprintln(os.Stdout, len(os.Args)) }\n",
+			"main_test.go": "package main\n\nimport \"testing\"\n\nfunc TestWarm(t *testing.T) {}\n",
+		}
+		for name, body := range files {
+			if err := os.MkdirAll(src, 0o755); err != nil {
+				return
+			}
+			if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil {
+				return
+			}
+		}
+		cache := filepath.Join(dir, "go-cache")
+		env := append(os.Environ(), "GOCACHE="+cache, "GOENV=off", "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off",
+			"GOFLAGS=-buildvcs=false", "TMPDIR="+warmCacheBase)
+		// As the check builds a judge (CGO off) and runs the replays (CGO as the toolchain has it).
+		for _, run := range [][]string{
+			{"env", "CGO_ENABLED=0", "go", "build", "-o", filepath.Join(dir, "judge"), "."},
+			{"go", "test", "-count=1", "."},
+		} {
+			cmd := exec.Command(run[0], run[1:]...)
+			cmd.Dir, cmd.Env = src, env
+			if out, err := cmd.CombinedOutput(); err != nil {
+				t.Logf("the warm Go cache could not be built, so judges build cold: %v\n%s", err, out)
+				_ = os.RemoveAll(dir)
+				return
+			}
+		}
+		warmCacheDir = dir
+	})
+	if warmCacheDir == "" {
+		return ""
+	}
+	return filepath.Join(warmCacheDir, "go-cache")
+}