Build a merge's modules of a tier at once, and beside the check's tests where the judge cache allows (hq issue 716)

A check built its images one after another, after merge-check.sh. With a judge cache the
builds run during the tests, out of their reach; a failed check lets go of them.

internal/builder/replay716_test.go

@@ -0,0 +1,158 @@
+package builder
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// novox/hq issue 716: a check built the images its merge moves one after another, and only after its merge-check.sh
+// had passed — 3 to 6 minutes on a check of about 5. The modules of a tier stand on none of each other and are built
+// at the same time; with a judge cache, beside the tests; a check that does not pass lets go of them.
+
+// atOnce is a fake engine whose compiles wait until every one of a tier's has started: builds one after another never
+// get there, and the first one's compile says so.
+type atOnce struct {
+	mu      sync.Mutex
+	r       *recorded
+	want    int
+	started int
+	all     chan struct{}
+	shared  string
+}
+
+func (a *atOnce) run(ctx context.Context, dir, name string, args ...string) (string, error) {
+	if name == "docker" && len(args) > 0 && args[0] == "run" && strings.Contains(strings.Join(args, " "), "go build") {
+		a.mu.Lock()
+		a.started++
+		if a.started == a.want {
+			close(a.all)
+		}
+		a.mu.Unlock()
+		select {
+		case <-a.all:
+		case <-time.After(5 * time.Second):
+			return "", errors.New("built alone: no other module of its tier was being built")
+		}
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return onTrunk{recorded: a.r}.run(ctx, dir, name, args...)
+}
+
+func (a *atOnce) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.r.PublishImage(ctx, localTag, repository)
+}
+
+func (a *atOnce) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.r.PublishArchive(ctx, repository, body, digest)
+}
+
+// aLockedScratch is the scratch namespace in memory, said to by builds at once.
+type aLockedScratch struct {
+	mu sync.Mutex
+	*aScratch
+	discarded []CheckBuild
+}
+
+func (s *aLockedScratch) Keep(ctx context.Context, p PullRef, c string, at time.Time, b []CheckBuild) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.aScratch.Keep(ctx, p, c, at, b)
+}
+
+func (s *aLockedScratch) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.aScratch.Find(ctx, module, key)
+}
+
+func (s *aLockedScratch) Discard(_ context.Context, b []CheckBuild) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.discarded = append(s.discarded, b...)
+	return nil
+}
+
+// threeAtOnce are three Go modules in one tier, each from its own repository.
+func threeAtOnce(t *testing.T) (*atOnce, CheckSpec, map[string]string) {
+	t.Helper()
+	r := &recorded{tree: "tree-of-the-merge", contents: aGoModule("mesh-controller"),
+		secondary: map[string]map[string]string{}}
+	spec := CheckSpec{ID: "build-716", Owner: "novox", Repo: "mesh-controller", Number: 716,
+		Builds: []CheckBuildSpec{{Module: "mesh-controller", Merged: true, Tier: 0}}}
+	for _, module := range []string{"build-agent", "route-proxy"} {
+		repository := "https://forge.invalid/" + module + ".git"
+		r.secondary[repository] = aGoModule(module)
+		spec.Builds = append(spec.Builds, CheckBuildSpec{Module: module, Repository: repository, Ref: "main", Tier: 0})
+	}
+	held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
+	return &atOnce{r: r, want: 3, all: make(chan struct{})}, spec, held
+}
+
+// The modules of one tier are built at the same time, each in a working copy of its own.
+func TestReplay716TheModulesOfATierAreBuiltAtTheSameTime(t *testing.T) {
+	withVersion(t, "builder-1")
+	engine, spec, held := threeAtOnce(t)
+	scratch := &aLockedScratch{aScratch: newScratch()}
+	built := buildForTheMerge(t.Context(), engine.run, spec, CheckBuilding{Publish: engine, Scratch: scratch, Held: held},
+		t.TempDir(), filepath.Join(t.TempDir(), "merged"), "2d22279b1d07",
+		CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053"}, GitCredential{}, nil, logging(nil))
+	if builtCount(built) != 3 {
+		t.Fatalf("the check built %d of 3 module(s) of one tier: %+v", builtCount(built), built)
+	}
+	if len(scratch.builds) != 3 {
+		t.Fatalf("the check kept %d build(s), not 3", len(scratch.builds))
+	}
+}
+
+// With a judge cache, the builds run while the tests run, where no container of the check reaches; a check that passed
+// keeps them, and one that failed lets go of every one and names none.
+func TestReplay716TheBuildsRunBesideTheTestsAndAFailedCheckLetsGoOfThem(t *testing.T) {
+	withVersion(t, "builder-1")
+	workspace := t.TempDir()
+	if _, err := buildsBesideTheTests(filepath.Join(workspace, "judge-cache"), workspace, "build-716"); err == nil {
+		t.Fatal("a judge cache inside the workspace was built in, where a pull request's script reaches")
+	}
+	for _, passed := range []bool{true, false} {
+		cache := t.TempDir()
+		place, err := buildsBesideTheTests(cache, workspace, "build-716")
+		if err != nil || !withinDir(cache, place) || withinDir(workspace, place) {
+			t.Fatalf("the builds' place %q (%v) is not the judge cache's, outside the workspace", place, err)
+		}
+		engine, spec, held := threeAtOnce(t)
+		scratch := &aLockedScratch{aScratch: newScratch()}
+		spec.BuildWith = &CheckBuilding{Publish: engine, Scratch: scratch, Held: held}
+		judged := CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053", Merge: "2d22279b1d07"}
+		beside := startBesideTheTests(t.Context(), engine.run, spec, place, filepath.Join(t.TempDir(), "merged"),
+			judged, GitCredential{}, nil, logging(nil))
+		// The tests: the builds are under way before the check has a verdict.
+		select {
+		case <-engine.all:
+		case <-time.After(5 * time.Second):
+			t.Fatal("nothing was built while the tests ran")
+		}
+		verdict := CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "pass"}}
+		if !passed {
+			verdict.Repo.Verdict = "fail"
+		}
+		built := beside.settle(t.Context(), verdict)
+		beside.abandon()
+		switch {
+		case passed && (builtCount(built) != 3 || len(scratch.builds) != 3 || len(scratch.discarded) != 0):
+			t.Fatalf("a check that passed: built %+v, kept %d, let go of %d", built, len(scratch.builds),
+				len(scratch.discarded))
+		case !passed && (built != nil || len(scratch.builds) != 0 || len(scratch.discarded) != 3):
+			t.Fatalf("a check that failed: named %+v, kept %d, let go of %d", built, len(scratch.builds),
+				len(scratch.discarded))
+		}
+	}
+}