Build a merge's modules of a tier at once, and beside the check's tests where the judge cache allows (hq issue 716)

A check built its images one after another, after merge-check.sh. With a judge cache the
builds run during the tests, out of their reach; a failed check lets go of them.

cmd/mesh-builder/main.go

@@ -26,6 +26,7 @@ import (
 	"os/signal"
 	"sort"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -272,7 +273,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 	// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
 	// watching, reads the same lines this container's log holds, live, and after the fact from the
 	// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
+	// One line at a time: a check's builds for its merge say their lines while its tests say theirs (novox/hq
+	// issue 716).
+	var saying sync.Mutex
 	say := func(step, message string) {
+		saying.Lock()
+		defer saying.Unlock()
 		fmt.Fprintf(os.Stderr, "  [%s] %s\n", step, message)
 		work.Say(step, message)
 		if mine != nil && step != "run" && step != "output" {

internal/builder/check.go

@@ -15,6 +15,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/novox/mesh-controller/internal/artifacts"
@@ -300,7 +301,17 @@ func ScriptToolchain(script []byte) string {
 // Check runs one merge check. An error is that it could not run; the verdict is then "error".
 func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential,
 	log Log) (CheckVerdict, error) {
-	// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462).
+	// Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462). A line
+	// at a time: its builds for the merge say theirs while its tests run (novox/hq issue 716).
+	if log != nil {
+		var saying sync.Mutex
+		inner := log
+		log = func(step, message string) {
+			saying.Lock()
+			defer saying.Unlock()
+			inner(step, message)
+		}
+	}
 	redact := redactorFor(forge)
 	say := logging(func(step, message string) {
 		if log != nil {
@@ -532,6 +543,25 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	// Every clone is made: the credential is gone before anything of the check runs (novox/hq issue 462).
 	forget()
 
+	// **What the merge moves is built beside the tests** (novox/hq issue 716) when the build agent has a judge cache to
+	// build it in, where no container of the check reaches; without one, after the check has judged.
+	logged := func(step, message string) {
+		if log != nil {
+			log(step, redact.redact(message))
+		}
+	}
+	var beside *besideTheTests
+	if buildingTree != "" {
+		place, err := buildsBesideTheTests(spec.JudgeCache, workspace, spec.ID)
+		if err != nil {
+			return CheckVerdict{}, err
+		}
+		if place != "" {
+			beside = startBesideTheTests(ctx, run, spec, place, buildingTree, judged, forge, logged, say)
+			defer beside.abandon()
+		}
+	}
+
 	// The facts, and the versions they say the mesh runs.
 	if registry == "" {
 		return CheckVerdict{}, errors.New("no artifact store to read the facts snapshot from")
@@ -752,12 +782,11 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 
 	// **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it
 	// judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again.
-	if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
-		v.Built = forTheMerge(ctx, run, spec, v, workspace, buildingTree, judged, forge, func(step, message string) {
-			if log != nil {
-				log(step, redact.redact(message))
-			}
-		}, say)
+	switch {
+	case beside != nil:
+		v.Built = beside.settle(ctx, v)
+	case len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0:
+		v.Built = forTheMerge(ctx, run, spec, v, workspace, buildingTree, judged, forge, logged, say)
 	}
 	return named(v), nil
 }

internal/builder/check_builds.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
@@ -68,108 +69,266 @@ func skipsTheBuilds(v CheckVerdict) string {
 	return ""
 }
 
-// buildForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace, and keeps what it
-// built under its keys. A module standing after one not built is not built either: what it stands on in the walk
-// is not what it would stand on here. tree is the check's clone and merge the merge commit it made there; workspace
-// is where each build's working copy is made, a directory the host's container engine sees at the same path — the
-// builder's own workspace, never a temporary directory of its container (novox/hq issue 713).
+// buildForTheMerge builds each module the plan moves into the scratch namespace (makeForTheMerge), and keeps what it
+// built under its keys (keepForTheMerge): the order of a check that builds after it has judged.
 func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
 	judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
+	out, kept := makeForTheMerge(ctx, run, spec, with, workspace, tree, merge, judged, forge, log, say)
+	return keepForTheMerge(ctx, spec, with, out, kept, say)
+}
+
+// makeForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace — **the modules of a
+// tier at the same time** (novox/hq issue 716): none of them stands on another, and a tier waits only for the ones
+// before it, whose builds it is handed. A module standing after one not built is not built either: what it stands on
+// in the walk is not what it would stand on here. tree is the check's copy of the merge and merge the merge commit it
+// made there; workspace is where each build's working copy is made, one directory per module, which the host's
+// container engine sees at the same path (novox/hq issue 713). Answers what each module came to, in the plan's order,
+// and the builds to keep — nothing is kept here.
+func makeForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
+	judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) ([]CheckBuilt, []CheckBuild) {
 	if len(spec.Builds) == 0 || with.Publish == nil || with.Scratch == nil {
-		return nil
+		return nil, nil
+	}
+	// What an earlier check left, ended before it could remove it, is not built on.
+	place := filepath.Join(workspace, "for-the-merge")
+	_ = os.RemoveAll(place)
+	defer os.RemoveAll(place)
+	// One log, said to by builds running at once: a line at a time.
+	var saying sync.Mutex
+	if log != nil {
+		inner := log
+		log = func(step, message string) {
+			saying.Lock()
+			defer saying.Unlock()
+			inner(step, message)
+		}
+	}
+	inner := say
+	say = func(step, format string, args ...any) {
+		saying.Lock()
+		defer saying.Unlock()
+		inner(step, format, args...)
 	}
 	ref := merge
 	if ref == "" {
 		ref = spec.Ref
 	}
-	// What an earlier check left, ended before it could remove it, is not built on.
-	_ = os.RemoveAll(filepath.Join(workspace, "for-the-merge"))
 	held := map[string]string{}
 	for k, v := range with.Held {
 		held[k] = v
 	}
-	var out []CheckBuilt
+	out := make([]CheckBuilt, len(spec.Builds))
+	results := make([]Result, len(spec.Builds))
 	var kept []CheckBuild
 	stopAt, stopWhy := -1, ""
-	for _, b := range spec.Builds {
-		began := time.Now()
-		if stopAt >= 0 && b.Tier > stopAt {
-			out = append(out, CheckBuilt{Module: b.Module, NotBuilt: stopWhy})
-			continue
+	for start := 0; start < len(spec.Builds); {
+		tier := spec.Builds[start].Tier
+		end := start
+		for end < len(spec.Builds) && spec.Builds[end].Tier == tier {
+			end++
 		}
-		if b.Skip != "" {
-			say("build", "%s not built for the merge: %s", b.Module, b.Skip)
-			out = append(out, CheckBuilt{Module: b.Module, NotBuilt: b.Skip})
-			if stopAt < 0 || b.Tier < stopAt {
-				stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
+		// What this tier's builds are handed: the builds of the tiers before it, read alone while they run.
+		handed := map[string]string{}
+		for k, v := range held {
+			handed[k] = v
+		}
+		var building sync.WaitGroup
+		for i := start; i < end; i++ {
+			b := spec.Builds[i]
+			if stopAt >= 0 && b.Tier > stopAt {
+				out[i] = CheckBuilt{Module: b.Module, NotBuilt: stopWhy}
+				continue
 			}
-			continue
-		}
-		repository, at := b.Repository, b.Ref
-		if b.Merged {
-			repository, at = tree, ref
-		}
-		say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
-		var mirrored []string
-		result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at,
-			filepath.Join(workspace, "for-the-merge"), held, with.Npmrc, forge, log, &mirrored,
-			&Reuse{keyFirst: true, Scratch: with.Scratch, Named: with.Named,
-				pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, head: judged.Head}, with.Seats)
-		built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
-		switch {
-		case errors.Is(err, errNoKey):
-			built.NotBuilt = err.Error()
-		case err != nil:
-			built.NotBuilt = "its build failed: " + err.Error()
-		case result.ReuseKey == "":
-			built.NotBuilt = "it has no reuse key: " + result.NoKey
-		}
-		if built.NotBuilt != "" {
-			say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
-			out = append(out, built)
-			if stopAt < 0 || b.Tier < stopAt {
-				stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
+			if b.Skip != "" {
+				say("build", "%s not built for the merge: %s", b.Module, b.Skip)
+				out[i] = CheckBuilt{Module: b.Module, NotBuilt: b.Skip}
+				continue
 			}
-			continue
-		}
-		built.Made = result.Built
-		for _, m := range result.Built {
-			held[b.Module+"/"+m.Name] = m.Reference
-		}
-		if result.Reused != nil {
-			// **Kept, not built** (novox/hq issue 703): the earlier check's build, named as that check's, so the
-			// walk takes it as the verdict names it; this check's document names it too, so it stays while this
-			// check does (ADR 0350 §5).
-			built.Kept = result.Reused.From.Check
-			say("build", "%s kept for the merge under %s: %s", b.Module, twelve(result.ReuseKey[3:]),
-				result.Reused.From.Said())
-			out = append(out, built)
-			kept = append(kept, result.Reused.From)
-			continue
-		}
-		say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]), built.Took.Round(time.Second))
-		out = append(out, built)
-		if len(result.Built) > 0 {
-			kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
-				Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
-				Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
-		}
-	}
-	_ = os.RemoveAll(filepath.Join(workspace, "for-the-merge"))
-	if len(kept) > 0 {
-		if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
-			time.Now().UTC(), kept); err != nil {
-			// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
-			say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
-			if err := with.Scratch.Discard(ctx, kept); err != nil {
-				say("build", "and what they made could not all be let go of: %v", err)
+			repository, at := b.Repository, b.Ref
+			if b.Merged {
+				repository, at = tree, ref
 			}
-			for i := range out {
-				if out[i].NotBuilt == "" {
-					out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
+			name := b.Module
+			if !safeName.MatchString(name) {
+				name = fmt.Sprintf("module-%d", i)
+			}
+			building.Add(1)
+			go func(i int, b CheckBuildSpec, repository, at, dir string) {
+				defer building.Done()
+				began := time.Now()
+				say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
+				var mirrored []string
+				result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at, dir, handed,
+					with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true, Scratch: with.Scratch, Named: with.Named,
+						pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, head: judged.Head},
+					with.Seats)
+				built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
+				switch {
+				case errors.Is(err, errNoKey):
+					built.NotBuilt = err.Error()
+				case err != nil:
+					built.NotBuilt = "its build failed: " + err.Error()
+				case result.ReuseKey == "":
+					built.NotBuilt = "it has no reuse key: " + result.NoKey
 				}
+				out[i], results[i] = built, result
+			}(i, b, repository, at, filepath.Join(place, name))
+		}
+		building.Wait()
+		for i := start; i < end; i++ {
+			b, built, result := spec.Builds[i], &out[i], results[i]
+			if built.NotBuilt != "" {
+				if built.Key != "" || built.Took > 0 {
+					say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
+				}
+				if stopAt < 0 || b.Tier < stopAt {
+					stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
+				}
+				continue
+			}
+			built.Made = result.Built
+			for _, m := range result.Built {
+				held[b.Module+"/"+m.Name] = m.Reference
+			}
+			if result.Reused != nil {
+				// **Kept, not built** (novox/hq issue 703): the earlier check's build, named as that check's, so the
+				// walk takes it as the verdict names it; this check's document names it too, so it stays while this
+				// check does (ADR 0350 §5).
+				built.Kept = result.Reused.From.Check
+				say("build", "%s kept for the merge under %s: %s", b.Module, twelve(result.ReuseKey[3:]),
+					result.Reused.From.Said())
+				kept = append(kept, result.Reused.From)
+				continue
+			}
+			say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]),
+				built.Took.Round(time.Second))
+			if len(result.Built) > 0 {
+				kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
+					Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
+					Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
+			}
+		}
+		start = end
+	}
+	return out, kept
+}
+
+// keepForTheMerge keeps a check's builds under their keys, its own document naming them, so the walk of its merge
+// finds them; what cannot be kept is let go of, said, and each module says it — never the check's verdict.
+func keepForTheMerge(ctx context.Context, spec CheckSpec, with CheckBuilding, out []CheckBuilt, kept []CheckBuild,
+	say func(step, format string, args ...any)) []CheckBuilt {
+	if len(kept) == 0 {
+		return out
+	}
+	if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
+		time.Now().UTC(), kept); err != nil {
+		// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
+		say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
+		if err := with.Scratch.Discard(ctx, kept); err != nil {
+			say("build", "and what they made could not all be let go of: %v", err)
+		}
+		for i := range out {
+			if out[i].NotBuilt == "" {
+				out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
 			}
 		}
 	}
 	return out
 }
+
+// discardForTheMerge lets go of builds a check made beside its tests when the check did not pass (novox/hq issue 716):
+// a change that fails is not merged, and nothing it built is kept. A build another check names stays (Discard).
+func discardForTheMerge(ctx context.Context, with CheckBuilding, kept []CheckBuild, why string,
+	say func(step, format string, args ...any)) {
+	say("check", "%s: the %d build(s) made beside it are let go of", why, len(kept))
+	if len(kept) == 0 {
+		return
+	}
+	if err := with.Scratch.Discard(ctx, kept); err != nil {
+		say("build", "what they made could not all be let go of, and goes with the week's sweep: %v", err)
+	}
+}
+
+// besideTheTests are a check's builds for its merge running while its gate and its own check run (novox/hq issue
+// 716): started once every clone is made, waited for when the check has judged, kept when it passed and let go of
+// when it did not.
+type besideTheTests struct {
+	spec    CheckSpec
+	with    CheckBuilding
+	cancel  context.CancelFunc
+	done    chan struct{}
+	place   string
+	out     []CheckBuilt
+	kept    []CheckBuild
+	began   time.Time
+	settled bool
+	say     func(step, format string, args ...any)
+}
+
+// buildsBesideTheTests is where a check's builds run beside its tests: a directory made for this check under the judge
+// cache's homes/ (ADR 0358 §6), on the host path the build agent declares — so the host's container engine mounts what
+// the agent sees (issue 713) — and never mounted by a check's container, so nothing a pull request's script runs can
+// change a build while it is made (issue 703). Empty, and the builds wait for the check, without a judge cache.
+func buildsBesideTheTests(cache, workspace, id string) (string, error) {
+	if cache == "" {
+		return "", nil
+	}
+	if withinDir(workspace, cache) {
+		return "", fmt.Errorf("the judge cache %s is inside the workspace %s, which a check's containers mount: "+
+			"nothing is built for the merge where a pull request could reach it", cache, workspace)
+	}
+	if !safeName.MatchString(id) {
+		return "", fmt.Errorf("the check's id %q cannot name its place in the judge cache", id)
+	}
+	place := filepath.Join(cache, "homes", id+"-builds")
+	if err := os.RemoveAll(place); err != nil {
+		return "", err
+	}
+	if err := os.MkdirAll(place, 0o700); err != nil {
+		return "", err
+	}
+	return place, nil
+}
+
+// startBesideTheTests starts the builds for the merge in place, from merged, the check's copy of the merge.
+func startBesideTheTests(ctx context.Context, run Runner, spec CheckSpec, place, merged string, judged CheckVerdict,
+	forge GitCredential, log Log, say func(step, format string, args ...any)) *besideTheTests {
+	building, cancel := context.WithCancel(ctx)
+	b := &besideTheTests{spec: spec, with: *spec.BuildWith, cancel: cancel, done: make(chan struct{}), place: place,
+		began: time.Now(), say: say}
+	say("check", "building what the merge moves (%d module(s)) beside its tests", len(spec.Builds))
+	go func() {
+		defer close(b.done)
+		b.out, b.kept = makeForTheMerge(building, run, spec, b.with, place, merged, judged.Merge, judged, forge, log, say)
+	}()
+	return b
+}
+
+// settle waits for the builds and keeps them when the check passed; a check that did not pass lets go of them, and
+// its verdict names none.
+func (b *besideTheTests) settle(ctx context.Context, verdict CheckVerdict) []CheckBuilt {
+	<-b.done
+	b.settled = true
+	defer os.RemoveAll(b.place)
+	if why := skipsTheBuilds(verdict); why != "" {
+		discardForTheMerge(ctx, b.with, b.kept, why, b.say)
+		return nil
+	}
+	built := keepForTheMerge(ctx, b.spec, b.with, b.out, b.kept, b.say)
+	b.say("check", "built for the merge: %d of %d module(s), beside the tests (%s)", builtCount(built), len(built),
+		time.Since(b.began).Round(time.Second))
+	return built
+}
+
+// abandon ends builds a check that ended without a verdict leaves, and lets go of what they made.
+func (b *besideTheTests) abandon() {
+	if b.settled {
+		return
+	}
+	b.cancel()
+	<-b.done
+	defer os.RemoveAll(b.place)
+	letGo, done := context.WithTimeout(context.Background(), time.Minute)
+	defer done()
+	discardForTheMerge(letGo, b.with, b.kept, "the check ended without a verdict", b.say)
+}

internal/builder/replay716_test.go

@@ -0,0 +1,158 @@
+package builder
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// novox/hq issue 716: a check built the images its merge moves one after another, and only after its merge-check.sh
+// had passed — 3 to 6 minutes on a check of about 5. The modules of a tier stand on none of each other and are built
+// at the same time; with a judge cache, beside the tests; a check that does not pass lets go of them.
+
+// atOnce is a fake engine whose compiles wait until every one of a tier's has started: builds one after another never
+// get there, and the first one's compile says so.
+type atOnce struct {
+	mu      sync.Mutex
+	r       *recorded
+	want    int
+	started int
+	all     chan struct{}
+	shared  string
+}
+
+func (a *atOnce) run(ctx context.Context, dir, name string, args ...string) (string, error) {
+	if name == "docker" && len(args) > 0 && args[0] == "run" && strings.Contains(strings.Join(args, " "), "go build") {
+		a.mu.Lock()
+		a.started++
+		if a.started == a.want {
+			close(a.all)
+		}
+		a.mu.Unlock()
+		select {
+		case <-a.all:
+		case <-time.After(5 * time.Second):
+			return "", errors.New("built alone: no other module of its tier was being built")
+		}
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return onTrunk{recorded: a.r}.run(ctx, dir, name, args...)
+}
+
+func (a *atOnce) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.r.PublishImage(ctx, localTag, repository)
+}
+
+func (a *atOnce) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.r.PublishArchive(ctx, repository, body, digest)
+}
+
+// aLockedScratch is the scratch namespace in memory, said to by builds at once.
+type aLockedScratch struct {
+	mu sync.Mutex
+	*aScratch
+	discarded []CheckBuild
+}
+
+func (s *aLockedScratch) Keep(ctx context.Context, p PullRef, c string, at time.Time, b []CheckBuild) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.aScratch.Keep(ctx, p, c, at, b)
+}
+
+func (s *aLockedScratch) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.aScratch.Find(ctx, module, key)
+}
+
+func (s *aLockedScratch) Discard(_ context.Context, b []CheckBuild) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.discarded = append(s.discarded, b...)
+	return nil
+}
+
+// threeAtOnce are three Go modules in one tier, each from its own repository.
+func threeAtOnce(t *testing.T) (*atOnce, CheckSpec, map[string]string) {
+	t.Helper()
+	r := &recorded{tree: "tree-of-the-merge", contents: aGoModule("mesh-controller"),
+		secondary: map[string]map[string]string{}}
+	spec := CheckSpec{ID: "build-716", Owner: "novox", Repo: "mesh-controller", Number: 716,
+		Builds: []CheckBuildSpec{{Module: "mesh-controller", Merged: true, Tier: 0}}}
+	for _, module := range []string{"build-agent", "route-proxy"} {
+		repository := "https://forge.invalid/" + module + ".git"
+		r.secondary[repository] = aGoModule(module)
+		spec.Builds = append(spec.Builds, CheckBuildSpec{Module: module, Repository: repository, Ref: "main", Tier: 0})
+	}
+	held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)}
+	return &atOnce{r: r, want: 3, all: make(chan struct{})}, spec, held
+}
+
+// The modules of one tier are built at the same time, each in a working copy of its own.
+func TestReplay716TheModulesOfATierAreBuiltAtTheSameTime(t *testing.T) {
+	withVersion(t, "builder-1")
+	engine, spec, held := threeAtOnce(t)
+	scratch := &aLockedScratch{aScratch: newScratch()}
+	built := buildForTheMerge(t.Context(), engine.run, spec, CheckBuilding{Publish: engine, Scratch: scratch, Held: held},
+		t.TempDir(), filepath.Join(t.TempDir(), "merged"), "2d22279b1d07",
+		CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053"}, GitCredential{}, nil, logging(nil))
+	if builtCount(built) != 3 {
+		t.Fatalf("the check built %d of 3 module(s) of one tier: %+v", builtCount(built), built)
+	}
+	if len(scratch.builds) != 3 {
+		t.Fatalf("the check kept %d build(s), not 3", len(scratch.builds))
+	}
+}
+
+// With a judge cache, the builds run while the tests run, where no container of the check reaches; a check that passed
+// keeps them, and one that failed lets go of every one and names none.
+func TestReplay716TheBuildsRunBesideTheTestsAndAFailedCheckLetsGoOfThem(t *testing.T) {
+	withVersion(t, "builder-1")
+	workspace := t.TempDir()
+	if _, err := buildsBesideTheTests(filepath.Join(workspace, "judge-cache"), workspace, "build-716"); err == nil {
+		t.Fatal("a judge cache inside the workspace was built in, where a pull request's script reaches")
+	}
+	for _, passed := range []bool{true, false} {
+		cache := t.TempDir()
+		place, err := buildsBesideTheTests(cache, workspace, "build-716")
+		if err != nil || !withinDir(cache, place) || withinDir(workspace, place) {
+			t.Fatalf("the builds' place %q (%v) is not the judge cache's, outside the workspace", place, err)
+		}
+		engine, spec, held := threeAtOnce(t)
+		scratch := &aLockedScratch{aScratch: newScratch()}
+		spec.BuildWith = &CheckBuilding{Publish: engine, Scratch: scratch, Held: held}
+		judged := CheckVerdict{Head: "98e856874913", Trunk: "c975dbf7b053", Merge: "2d22279b1d07"}
+		beside := startBesideTheTests(t.Context(), engine.run, spec, place, filepath.Join(t.TempDir(), "merged"),
+			judged, GitCredential{}, nil, logging(nil))
+		// The tests: the builds are under way before the check has a verdict.
+		select {
+		case <-engine.all:
+		case <-time.After(5 * time.Second):
+			t.Fatal("nothing was built while the tests ran")
+		}
+		verdict := CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "pass"}}
+		if !passed {
+			verdict.Repo.Verdict = "fail"
+		}
+		built := beside.settle(t.Context(), verdict)
+		beside.abandon()
+		switch {
+		case passed && (builtCount(built) != 3 || len(scratch.builds) != 3 || len(scratch.discarded) != 0):
+			t.Fatalf("a check that passed: built %+v, kept %d, let go of %d", built, len(scratch.builds),
+				len(scratch.discarded))
+		case !passed && (built != nil || len(scratch.builds) != 0 || len(scratch.discarded) != 3):
+			t.Fatalf("a check that failed: named %+v, kept %d, let go of %d", built, len(scratch.builds),
+				len(scratch.discarded))
+		}
+	}
+}