Merge remote-tracking branch 'origin/main' into feat/try-speed

A check ask stopped its level at the first test step that did not pass,
so the packages after the changed ones never ran and a check found one
fault per round. Every step now runs; the level still fails, and no
later level runs (ADR 0484 clause 13). The verdict gains Failures, every
failed test with its first line, at most MaxFailures and a count.

go/buildstep/askrun.go

@@ -8,6 +8,7 @@ import (
 	"slices"
 	"sort"
 	"strings"
+	"sync"
 
 	"git.novox.be/novox/mesh-sdk/go/delivery"
 )
@@ -170,16 +171,36 @@ func (s AskSteps) check(ctx context.Context, job delivery.CheckJob) (any, error)
 	if job.Runner != "" && (job.RunnerWhen == "" || changesUnder(f.Changed, job.RunnerWhen)) {
 		runner = job.Runner
 	}
-	for _, level := range job.Levels {
+	// The levels run side by side (issue 1921, try speed S10; the operator's yes to D1, ADR 0541): the integration level
-		la := delivery.LevelAnswer{Level: level}
+	// does not wait for the unit level. They are answered in their order, so the first level that did not pass is still
-		passed, err := s.level(ctx, job, level, f, runner, &la)
+	// the first failure the asker reads (ADR 0484 clause 13); the levels after it are kept in the answer for the log.
-		answer.Levels = append(answer.Levels, la)
+	type ran struct {
-		if err != nil {
+		la     delivery.LevelAnswer
-			return nil, err
+		passed bool
-		}
+		err    error
-		if !passed {
+	}
-			return answer, nil // the first level that did not pass ends the check (ADR 0484 clause 13)
+	runs := make([]ran, len(job.Levels))
+	var wg sync.WaitGroup
+	for i, level := range job.Levels {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			runs[i].la = delivery.LevelAnswer{Level: level}
+			runs[i].passed, runs[i].err = s.level(ctx, job, level, f, runner, &runs[i].la)
+		}()
+	}
+	wg.Wait()
+	failed := false
+	for _, r := range runs {
+		if r.err != nil {
+			if failed {
+				continue // a level after the first failure that could not run: the failure is the answer
+			}
+			answer.Levels = append(answer.Levels, r.la)
+			return nil, r.err
 		}
+		answer.Levels = append(answer.Levels, r.la)
+		failed = failed || !r.passed
 	}
 	return answer, nil
 }
@@ -208,6 +229,9 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
 		_ = json.Unmarshal(r, &t)
 		return t.Passed, nil
 	}
+	// Every test step of the level runs, also after one that did not pass, so one check names every failing test of the
+	// level and not only the first (issue 1921); the level as a whole passes only when each of its steps did.
+	all := true
 	if s.Digest == nil {
 		// A module with no digest (Node) tests each module directory of the tree, those whose tests changed first.
 		for _, priority := range []bool{true, false} {
@@ -219,14 +243,12 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
 				if err != nil {
 					return false, fmt.Errorf("the %s level of %s: %w", level, dir, err)
 				}
-				if !passed {
+				all = all && passed
-					return false, nil
-				}
 			}
 		}
-		return true, nil
+		return all, nil
 	}
-	todo, all, keepRunner := s.plan(ctx, job, level, f, runner, withRunner, la)
+	todo, whole, keepRunner := s.plan(ctx, job, level, f, runner, withRunner, la)
 	if !keepRunner {
 		runner = "" // an installed module that ignores the runner judges the level as it judges any other (ADR 0535 §2)
 	}
@@ -235,7 +257,7 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
 		if job.Run != "" {
 			args["run"] = job.Run
 		}
-		if all {
+		if whole {
 			if priority {
 				continue // nothing to put first: the level runs whole after
 			}
@@ -255,11 +277,9 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string
 		if err != nil {
 			return false, fmt.Errorf("the %s level: %w", level, err)
 		}
-		if !passed {
+		all = all && passed // the packages left still run (issue 1921)
-			return false, nil
-		}
 	}
-	return true, nil
+	return all, nil
 }
 
 // release removes an ask's workspace, saying a failure: the module's sweep removes it later.

go/buildstep/askrun_levels_test.go

@@ -0,0 +1,60 @@
+package buildstep
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// 1921/S10 (the operator's yes to D1, ADR 0541): a check's levels run side by side: the integration level has started
+// before the unit level ends. The answer holds the levels in their order, so with the unit level failing it still names
+// the unit level first; the integration level's answer is kept after it for the log.
+func TestACheckRunsItsLevelsSideBySideAndAnswersThemInOrder(t *testing.T) {
+	integrationStarted := make(chan struct{})
+	var once sync.Once
+	var mu sync.Mutex
+	unitWaited := false
+	steps := AskSteps{
+		Fetch: func(context.Context, map[string]any) (any, error) {
+			return map[string]any{"workspace": "/w", "modules": []string{"."}}, nil
+		},
+		Test: func(ctx context.Context, args map[string]any) (any, error) {
+			switch args["level"] {
+			case delivery.LevelUnit:
+				select {
+				case <-integrationStarted:
+					mu.Lock()
+					unitWaited = true
+					mu.Unlock()
+				case <-time.After(2 * time.Second):
+				}
+				return map[string]any{"passed": false, "level": "unit"}, nil
+			default:
+				once.Do(func() { close(integrationStarted) })
+				return map[string]any{"passed": true, "level": "integration"}, nil
+			}
+		},
+	}
+	out, err := RunAsk(context.Background(), steps, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration},
+		Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if !unitWaited {
+		t.Error("the integration level did not start while the unit level ran")
+	}
+	a := out.(delivery.CheckAnswer)
+	if len(a.Levels) != 2 || a.Levels[0].Level != delivery.LevelUnit || a.Levels[1].Level != delivery.LevelIntegration {
+		t.Fatalf("levels %+v, want unit then integration", a.Levels)
+	}
+	var first struct{ Passed bool }
+	if err := json.Unmarshal(a.Levels[0].Tests[0], &first); err != nil || first.Passed {
+		t.Errorf("the first level's result %s, want the unit level's failure", a.Levels[0].Tests[0])
+	}
+}

go/buildstep/askrun_test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -16,6 +17,7 @@ const runCommit = "0123456789abcdef0123456789abcdef01234567"
 
 // recorder is a build module's step tools that record each call.
 type recorder struct {
+	mu      sync.Mutex // a check's levels run side by side (issue 1921, S10)
 	calls   []string
 	fetch   map[string]any
 	tests   map[string]bool // package or module directory -> passed
@@ -26,7 +28,9 @@ func (r *recorder) steps() AskSteps {
 	call := func(name string, answer func(args map[string]any) any) StepFunc {
 		return func(ctx context.Context, args map[string]any) (any, error) {
 			b, _ := json.Marshal(args)
+			r.mu.Lock()
 			r.calls = append(r.calls, name+" "+string(b))
+			r.mu.Unlock()
 			return answer(args), nil
 		}
 	}
@@ -109,9 +113,9 @@ func TestRunAskRefusesAnotherToolchain(t *testing.T) {
 	}
 }
 
-// 1887/A3, A5a, issue 1887 design limit 1: a check ask runs the levels of one check after one fetch: it digests the unit
+// 1887/A3, A5a: a check ask runs the levels of one check after one fetch: it digests the unit level, takes the kept
-// level, takes the kept verdicts, tests the packages whose tests changed first, and stops at the first failure, running
+// verdicts, tests the packages whose tests changed first, then the rest also after a failure (issue 1921); the levels run
-// no later level.
+// side by side and are answered in order, the first that did not pass first (ADR 0484 clause 13; issue 1921 S10).
 func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) {
 	r := &recorder{
 		fetch:   map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}},
@@ -123,19 +127,26 @@ func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	got := strings.Join(r.calls, "\n")
+	var unit []string
+	for _, c := range r.calls {
+		if !strings.Contains(c, `"level":"integration"`) {
+			unit = append(unit, c)
+		}
+	}
+	got := strings.Join(unit, "\n")
 	want := `fetch {"base":"` + runCommit + `","commit":"` + runCommit + `","repository":"novox/x"}
 digest {"level":"unit","workspace":"/w"}
-test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}`
+test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}
+test {"level":"unit","packages":["x/pkg/a"],"workspace":"/w"}`
 	if got != want {
 		t.Fatalf("calls:\n%s\nwant:\n%s", got, want)
 	}
 	a := out.(delivery.CheckAnswer)
-	if len(a.Levels) != 1 {
+	if len(a.Levels) != 2 || a.Levels[1].Level != delivery.LevelIntegration {
-		t.Fatalf("levels %+v, want the unit level alone", a.Levels)
+		t.Fatalf("levels %+v, want the unit level first, the integration level after it", a.Levels)
 	}
 	u := a.Levels[0]
-	if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 1 || u.Digest == nil {
+	if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 2 || u.Digest == nil {
 		t.Fatalf("the unit level %+v", u)
 	}
 }
@@ -203,3 +214,42 @@ func TestRunAskChecksNodeModulesAndConflicts(t *testing.T) {
 		t.Fatalf("a conflict: %v %v", err, r.calls)
 	}
 }
+
+// Issue 1921: a failure in the packages whose tests changed does not stop the level: the other packages still run, so two
+// failing packages, one changed and one not, are two test results of the one level, and the level does not pass. A Node
+// check runs every module directory after one fails too.
+func TestRunAskRunsEveryPackageAfterAFailure(t *testing.T) {
+	r := &recorder{
+		fetch:   map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}},
+		digests: map[string]string{"x/pkg/a": "d-a", "x/pkg/b": "d-b"},
+		tests:   map[string]bool{"x/pkg/a": false, "x/pkg/b": false},
+	}
+	out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelIntegration, delivery.LevelUnit},
+		Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	a := out.(delivery.CheckAnswer)
+	if len(a.Levels) != 2 || len(a.Levels[0].Tests) != 2 || a.Levels[0].Level != delivery.LevelIntegration {
+		t.Fatalf("levels %+v, want the first level with both packages' results, in the order asked", a.Levels)
+	}
+	for i, raw := range a.Levels[0].Tests {
+		var res struct{ Passed bool }
+		if err := json.Unmarshal(raw, &res); err != nil || res.Passed {
+			t.Errorf("result %d %s, want a failure", i, raw)
+		}
+	}
+
+	r = &recorder{fetch: map[string]any{"workspace": "/w", "modules": []string{"a", "b"}, "changed": []string{"b/test/x.test.ts"}},
+		tests: map[string]bool{"a": true, "b": false}}
+	s := r.steps()
+	s.Digest = nil
+	out, err = RunAsk(context.Background(), s, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration},
+		Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a := out.(delivery.CheckAnswer); len(r.calls) != 5 || len(a.Levels) != 2 || len(a.Levels[0].Tests) != 2 || a.Levels[0].Level != delivery.LevelUnit {
+		t.Fatalf("calls %v, levels %+v: want both directories of each level, the unit level first", r.calls, a.Levels)
+	}
+}

go/buildstep/isolate.go

@@ -3,10 +3,13 @@ package buildstep
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
+	"sync"
 
 	"git.novox.be/novox/mesh-sdk/go/forgekey"
 )
@@ -29,16 +32,25 @@ const Bubblewrap = "bwrap"
 //   - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step
 //     downloaded) read-only;
 //   - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone
-//     (WriteCaches), so a candidate's code never changes what a later build reads.
+//     (WriteCaches), so a candidate's code never changes what a later build reads;
+//   - with a CacheLayer, what one command writes into the caches kept for the later commands of the same ask (issue 1921,
+//     the try speed-up): the vet, the unit tests and the integration compile of one ask compile each package once.
 type Isolation struct {
 	Workspace   string
 	Writable    []string
 	ReadOnly    []string
 	Caches      []string
 	WriteCaches bool
-	Env         []string
+	// CacheLayer is a directory of the ask's own, in its workspace and gone with it: each command writes the caches into
+	// a layer of its own beneath it (overlayfs refuses an upper layer two mounts share, and an ask's commands run side by
+	// side), which Isolated moves into the ask's kept layer when the command ends; every later command reads the kept
+	// layer above the shared cache. The shared cache stays unchanged. Empty: what a command writes is thrown away.
+	CacheLayer string
+	Env        []string
 	// Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback.
 	Hosts string
+	// run is the directory of this command's own writable layers under CacheLayer; Isolated makes one per command.
+	run string
 }
 
 // etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the
@@ -91,7 +103,7 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin
 	for _, group := range []struct {
 		what string
 		dirs []string
-	}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} {
+	}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}, {"cache layer", layerDirs(iso.CacheLayer)}} {
 		for _, d := range group.dirs {
 			if err := checkDir(group.what, d); err != nil {
 				return nil, err
@@ -113,7 +125,7 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin
 	for _, d := range iso.ReadOnly {
 		argv = append(argv, "--ro-bind", d, d)
 	}
-	argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...)
+	argv = append(argv, iso.cacheArgs()...)
 	argv = append(argv, "--bind", iso.Workspace, iso.Workspace)
 	for _, d := range iso.Writable {
 		argv = append(argv, "--bind", d, d)
@@ -134,19 +146,107 @@ func (iso Isolation) command(env []string, name string, args ...string) ([]strin
 
 // cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other
 // read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and
-// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace).
+// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace). With a
-func cacheArgs(caches []string, write bool) []string {
+// CacheLayer the ask's kept layer lies above the shared cache and the command's writes go to its own layer on disk
+// (--overlay), which Isolated keeps for the ask when the command ends.
+func (iso Isolation) cacheArgs() []string {
 	var argv []string
-	for _, c := range caches {
+	for i, c := range iso.Caches {
-		if write {
+		switch {
+		case iso.WriteCaches:
 			argv = append(argv, "--bind", c, c)
-			continue
+		case iso.CacheLayer != "":
+			upper, work := iso.runLayer(i)
+			argv = append(argv, "--overlay-src", c, "--overlay-src", keptLayer(iso.CacheLayer, i), "--overlay", upper, work, c)
+		default:
+			argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
 		}
-		argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
 	}
 	return argv
 }
 
+func layerDirs(layer string) []string {
+	if layer == "" {
+		return nil
+	}
+	return []string{layer}
+}
+
+// keptLayer is where the ask keeps what its commands wrote into the cache i.
+func keptLayer(layer string, i int) string { return filepath.Join(layer, "kept", strconv.Itoa(i)) }
+
+// runLayer is the upper and work directory of this command's writes into the cache i. Command, which makes no
+// directory, names the layer "run"; Isolated gives each command one of its own.
+func (iso Isolation) runLayer(i int) (upper, work string) {
+	run := iso.run
+	if run == "" {
+		run = filepath.Join(iso.CacheLayer, "run")
+	}
+	return filepath.Join(run, strconv.Itoa(i), "upper"), filepath.Join(run, strconv.Itoa(i), "work")
+}
+
+// layerLocks hold one mutex per cache layer, so two commands ending at once keep their writes one after the other.
+var layerLocks sync.Map
+
+// prepareLayer makes the kept layers and a run directory of the command's own under the cache layer.
+func (iso *Isolation) prepareLayer() error {
+	if iso.CacheLayer == "" || iso.WriteCaches || len(iso.Caches) == 0 {
+		return nil
+	}
+	if err := os.MkdirAll(filepath.Join(iso.CacheLayer, "runs"), 0o700); err != nil {
+		return err
+	}
+	run, err := os.MkdirTemp(filepath.Join(iso.CacheLayer, "runs"), "run-")
+	if err != nil {
+		return err
+	}
+	iso.run = run
+	for i := range iso.Caches {
+		upper, work := iso.runLayer(i)
+		for _, d := range []string{keptLayer(iso.CacheLayer, i), upper, work} {
+			if err := os.MkdirAll(d, 0o700); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// keepLayer moves what the command wrote into the caches into the ask's kept layers, and removes its run directory.
+// A file the command wrote replaces the kept one (a cache's newest write wins); a whiteout (a file the command
+// deleted) is not kept, since a cache entry deleted by one command is no fault of a later one. A write that cannot be
+// kept only means a later command builds it again.
+func (iso Isolation) keepLayer() {
+	if iso.run == "" {
+		return
+	}
+	defer os.RemoveAll(iso.run)
+	mu, _ := layerLocks.LoadOrStore(iso.CacheLayer, &sync.Mutex{})
+	mu.(*sync.Mutex).Lock()
+	defer mu.(*sync.Mutex).Unlock()
+	for i := range iso.Caches {
+		upper, _ := iso.runLayer(i)
+		kept := keptLayer(iso.CacheLayer, i)
+		filepath.WalkDir(upper, func(p string, d fs.DirEntry, err error) error {
+			if err != nil || p == upper {
+				return nil
+			}
+			rel, _ := filepath.Rel(upper, p)
+			to := filepath.Join(kept, rel)
+			switch {
+			case d.IsDir():
+				if fi, err := os.Lstat(to); err == nil && !fi.IsDir() {
+					os.Remove(to)
+				}
+				os.MkdirAll(to, 0o700)
+			case d.Type().IsRegular():
+				os.Rename(p, to)
+			}
+			return nil
+		})
+	}
+}
+
 // loopbackHosts is the hosts file of a step: the loopback and nothing else.
 const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n"
 
@@ -166,6 +266,10 @@ func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner {
 		}
 		iso := iso
 		iso.Hosts = hosts
+		if err := iso.prepareLayer(); err != nil {
+			return forgekey.Ran{Status: 1, Err: "the step's cache layer: " + err.Error()}
+		}
+		defer iso.keepLayer()
 		argv, err := iso.command(env, name, args...)
 		if err != nil {
 			return forgekey.Ran{Status: 1, Err: err.Error()}

go/buildstep/isolate_integration_test.go

@@ -64,3 +64,39 @@ func TestARunStepReachesNothingAndLeavesTheCachesUnchanged(t *testing.T) {
 		t.Fatalf("the step's write to its workspace did not stay (status %d, %s%s): %q %v", r.Status, r.Stderr, r.Err, b, err)
 	}
 }
+
+// 1921/S2: in one ask (one cache layer), a second go build of the same package does not compile it again, and the
+// shared cache is unchanged afterwards.
+func TestASecondBuildOfTheAskFindsTheFirstsCompiledAndTheSharedCacheIsUnchanged(t *testing.T) {
+	if _, err := exec.LookPath(Bubblewrap); err != nil {
+		t.Fatalf("bubblewrap is not installed here (issue 1309): %v", err)
+	}
+	goBin, err := exec.LookPath("go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	root := t.TempDir()
+	shared := filepath.Join(root, "gobuild")
+	if err := os.MkdirAll(shared, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	workspace := filepath.Join(root, "ws")
+	tree := filepath.Join(workspace, "tree")
+	writeFile(t, tree, "go.mod", "module probe\n\ngo 1.22\n")
+	writeFile(t, tree, "probe.go", "package probe\n\nfunc Probe() int { return 42 }\n")
+	run := Isolated(forgekey.ExecRunnerFor(0), Isolation{Workspace: workspace, Caches: []string{shared},
+		CacheLayer: filepath.Join(workspace, "cachelayer"), ReadOnly: []string{filepath.Dir(filepath.Dir(goBin))}})
+	env := []string{"PATH=" + filepath.Dir(goBin) + ":/usr/bin:/bin", "GOCACHE=" + shared, "GOTOOLCHAIN=local", "GOFLAGS=-mod=mod"}
+	for i, want := range []bool{true, false} {
+		r := run(context.Background(), env, "go", "-C", tree, "build", "-x", "./...")
+		if r.Status != 0 || r.Err != "" {
+			t.Fatalf("build %d: status %d %s\n%s", i+1, r.Status, r.Err, r.Stderr)
+		}
+		if compiled := strings.Contains(r.Stderr, "/compile "); compiled != want {
+			t.Errorf("build %d compiled the package: %v, want %v\n%s", i+1, compiled, want, r.Stderr)
+		}
+	}
+	if entries, _ := os.ReadDir(shared); len(entries) != 0 {
+		t.Errorf("the shared cache was written: %d entries", len(entries))
+	}
+}

go/buildstep/isolate_layer_test.go

@@ -0,0 +1,110 @@
+package buildstep
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+)
+
+// overlayOf answers the upper directory a command line gives the cache c, "" when it gives none.
+func overlayOf(argv []string, c string) string {
+	for i := 0; i+3 < len(argv); i++ {
+		if argv[i] == "--overlay" && argv[i+3] == c {
+			return argv[i+1]
+		}
+	}
+	return ""
+}
+
+// 1921/S2: with a cache layer, a command reads the ask's kept layer above the shared cache and writes into a layer on
+// disk of its own, never into the shared cache and never into a throwaway tmpfs.
+func TestACommandWithACacheLayerReadsTheKeptLayerAndWritesItsOwn(t *testing.T) {
+	fixedSystem(t)
+	iso := Isolation{Workspace: "/w", Caches: []string{"/cache/gomod", "/cache/gobuild"}, CacheLayer: "/w/cachelayer"}
+	argv, err := iso.Command("go", "vet")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i, c := range iso.Caches {
+		if !holds(argv, "--overlay-src", c, "--overlay-src", keptLayer(iso.CacheLayer, i), "--overlay") {
+			t.Errorf("the cache %s is not read beneath the ask's kept layer: %v", c, argv)
+		}
+		if up := overlayOf(argv, c); up == "" || filepath.Dir(filepath.Dir(filepath.Dir(up))) != "/w/cachelayer" {
+			t.Errorf("the cache %s is written into %q, not a layer of the ask's own: %v", c, up, argv)
+		}
+		if holds(argv, "--tmp-overlay", c) || holds(argv, "--bind", c, c) {
+			t.Errorf("the cache %s is thrown away or written: %v", c, argv)
+		}
+	}
+	if _, err := (Isolation{Workspace: "/w", CacheLayer: "cachelayer"}).Command("go"); err == nil {
+		t.Error("a relative cache layer was taken")
+	}
+}
+
+// 1921/S2: what one command wrote into the caches is kept for the ask's later commands, the shared cache stays
+// unchanged, two commands at once write into layers of their own (overlayfs refuses an upper layer two mounts share),
+// and a command's own layer is gone when it ends.
+func TestWhatOneCommandCompiledIsReadByTheNextOfTheSameAsk(t *testing.T) {
+	fixedSystem(t)
+	root := t.TempDir()
+	shared := filepath.Join(root, "gobuild")
+	if err := os.MkdirAll(shared, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	layer := filepath.Join(root, "ws", "cachelayer")
+	var mu sync.Mutex
+	uppers := map[string]bool{}
+	gate := make(chan struct{})
+	var started sync.WaitGroup
+	started.Add(2)
+	inner := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		up := overlayOf(args, shared)
+		mu.Lock()
+		uppers[up] = true
+		mu.Unlock()
+		if holds(args, "--", "go", "build") {
+			started.Done()
+			<-gate // both commands are running before either ends
+			os.MkdirAll(filepath.Join(up, "ab"), 0o700)
+			os.WriteFile(filepath.Join(up, "ab", args[len(args)-1]+"-d"), []byte("compiled"), 0o600)
+		}
+		return forgekey.Ran{}
+	}
+	run := Isolated(inner, Isolation{Workspace: filepath.Join(root, "ws"), Caches: []string{shared}, CacheLayer: layer})
+	var wg sync.WaitGroup
+	for _, pkg := range []string{"one", "two"} {
+		wg.Add(1)
+		go func() { defer wg.Done(); run(context.Background(), nil, "go", "build", pkg) }()
+	}
+	started.Wait()
+	close(gate)
+	wg.Wait()
+	if len(uppers) != 2 {
+		t.Fatalf("two commands at once wrote into %d layers, want 2 of their own: %v", len(uppers), uppers)
+	}
+	for _, pkg := range []string{"one", "two"} {
+		if b, err := os.ReadFile(filepath.Join(keptLayer(layer, 0), "ab", pkg+"-d")); err != nil || string(b) != "compiled" {
+			t.Errorf("what the build of %s compiled is not kept for the ask: %q %v", pkg, b, err)
+		}
+	}
+	for up := range uppers {
+		if _, err := os.Stat(up); !os.IsNotExist(err) {
+			t.Errorf("the command's own layer %s outlived it: %v", up, err)
+		}
+	}
+	var next []string
+	Isolated(func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		next = args
+		return forgekey.Ran{}
+	}, Isolation{Workspace: filepath.Join(root, "ws"), Caches: []string{shared}, CacheLayer: layer})(context.Background(), nil, "go", "test")
+	if !holds(next, "--overlay-src", shared, "--overlay-src", keptLayer(layer, 0)) {
+		t.Errorf("the next command does not read the kept layer: %v", next)
+	}
+	if entries, _ := os.ReadDir(shared); len(entries) != 0 {
+		t.Errorf("the shared cache was written: %v", entries)
+	}
+}

go/buildstep/merge.go

@@ -24,7 +24,8 @@ type MergeResult struct {
 // signature, no hook, nothing of the machine's git configuration that changes a merge. Nothing is pushed.
 //
 // A merge needs both histories, so it makes the fetch whole (the shallow clone FetchCommit made is deepened) and
-// fetches base by its sha. A head that does not merge cleanly is an answer, not an error: the files in conflict, the
+// fetches base by its sha: from the mirror WithMirror names, when it holds base (UpdateMirror to base first), else
+// from remote. A head that does not merge cleanly is an answer, not an error: the files in conflict, the
 // merge undone and the workspace left at base. A head already in base answers a Merge equal to Base and no changed
 // paths. An error is that the merge could not be tried: a base or commit that is not a full sha, no base, a workspace
 // not at commit, or a fetch or git command that failed. remote and the options are those of FetchCommit; the runner's
@@ -67,12 +68,20 @@ func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, comm
 	if err != nil {
 		return res, err
 	}
-	fetch := []string{"fetch", "--no-tags", "origin", base}
+	fetchFrom := func(source string) error {
-	if strings.TrimSpace(shallow) == "true" {
+		fetch := []string{"fetch", "--no-tags", source, base}
-		fetch = []string{"fetch", "--no-tags", "--unshallow", "origin", base}
+		if strings.TrimSpace(shallow) == "true" {
+			fetch = []string{"fetch", "--no-tags", "--unshallow", source, base}
+		}
+		_, err := git(fetch...)
+		return err
 	}
-	if _, err := git(fetch...); err != nil {
+	// With a mirror brought to the trunk commit, the history is deepened from it, a local fetch (issue 1921, try speed
-		return res, fmt.Errorf("the trunk commit %s: %w", base, err)
+	// S6); a mirror that cannot answer leaves it to the forge, as before.
+	if o.mirror == "" || fetchFrom(o.mirror) != nil {
+		if err := fetchFrom("origin"); err != nil {
+			return res, fmt.Errorf("the trunk commit %s: %w", base, err)
+		}
 	}
 	if _, err := git("checkout", "-q", "--detach", base); err != nil {
 		return res, fmt.Errorf("the trunk commit %s cannot be checked out: %w", base, err)

go/buildstep/merge_integration_test.go

@@ -170,3 +170,33 @@ func TestMergeIntoAHeadAlreadyInTheTrunkChangesNothing(t *testing.T) {
 		t.Errorf("result = %+v, %v", res, err)
 	}
 }
+
+// 1921/S6: with a mirror brought to the head and to the trunk commit, the merge asks the forge for nothing: it succeeds
+// with the forge gone.
+func TestMergeIntoWithAMirrorAtTheTrunkNeedsNoForge(t *testing.T) {
+	r := newMergeRig(t)
+	head := r.head(map[string]string{"b.txt": "head\n"})
+	base := r.advance(map[string]string{"a.txt": "one\ntwo\nthree\nfour\n"})
+	mirror := filepath.Join(t.TempDir(), "mirror.git")
+	if err := os.MkdirAll(mirror, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []string{head, base} {
+		if err := UpdateMirror(context.Background(), forgekey.ExecRunner, mirror, r.remote, c, "", AllowFileRemote()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := FetchCommit(context.Background(), forgekey.ExecRunner, r.ws, r.remote, head, "", AllowFileRemote(), WithMirror(mirror)); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.RemoveAll(r.bare); err != nil { // the forge is gone
+		t.Fatal(err)
+	}
+	res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote(), WithMirror(mirror))
+	if err != nil {
+		t.Fatalf("the merge asked the forge: %v", err)
+	}
+	if len(res.Merge) != 40 || !reflect.DeepEqual(res.Changed, []string{"b.txt"}) {
+		t.Errorf("result = %+v", res)
+	}
+}

go/buildstep/merge_test.go

@@ -72,3 +72,39 @@ func TestMergeIntoRunsTheCheckMergesCommands(t *testing.T) {
 		}
 	}
 }
+
+// 1921/S6: with a mirror that holds the trunk commit, the merge deepens the workspace from the mirror, a local fetch,
+// and asks the forge nothing; a mirror that cannot answer falls back to the forge.
+func TestMergeIntoWithAMirrorFetchesTheTrunkFromTheMirror(t *testing.T) {
+	base := strings.Repeat("b", 40)
+	mirror := t.TempDir()
+	answer := func(failMirror bool) *fakeGit {
+		return &fakeGit{rule: func(l string) forgekey.Ran {
+			switch {
+			case strings.HasSuffix(l, "rev-parse HEAD"):
+				return forgekey.Ran{Stdout: sha + "\n"}
+			case strings.Contains(l, "--is-shallow-repository"):
+				return forgekey.Ran{Stdout: "true\n"}
+			case failMirror && strings.Contains(l, " fetch ") && strings.Contains(l, mirror):
+				return forgekey.Ran{Status: 128, Stderr: "fatal: not our ref"}
+			}
+			return forgekey.Ran{}
+		}}
+	}
+	f := answer(false)
+	dir := t.TempDir()
+	if _, err := MergeInto(context.Background(), f.run, dir, "ssh://mesh-forge/o/r.git", base, sha, WithMirror(mirror)); err != nil {
+		t.Fatal(err)
+	}
+	all := strings.Join(f.calls, "\n")
+	if !strings.Contains(all, "fetch --no-tags --unshallow "+mirror+" "+base) || strings.Contains(all, "--unshallow origin") {
+		t.Errorf("the trunk was not fetched from the mirror alone:\n%s", all)
+	}
+	g := answer(true)
+	if _, err := MergeInto(context.Background(), g.run, t.TempDir(), "ssh://mesh-forge/o/r.git", base, sha, WithMirror(mirror)); err != nil {
+		t.Fatal(err)
+	}
+	if all := strings.Join(g.calls, "\n"); !strings.Contains(all, "fetch --no-tags --unshallow origin "+base) {
+		t.Errorf("a mirror that could not answer did not fall back to the forge:\n%s", all)
+	}
+}

go/delivery/candidate_merged_test.go

@@ -0,0 +1,27 @@
+package delivery
+
+import (
+	"strings"
+	"testing"
+)
+
+// 1791/1921 S9: a member of a group that merged stays in the candidate, marked merged, with its head, so the heads hash
+// the Approve covers holds until every member merged; a candidate whose every member merged is refused.
+func TestACandidateKeepsAMergedMemberAndRefusesOneWhoseEveryMemberMerged(t *testing.T) {
+	a := Head{Repository: "novox/a", Number: 1, Head: strings.Repeat("1", 40)}
+	b := Head{Repository: "novox/b", Number: 2, Head: strings.Repeat("2", 40)}
+	hash, _ := HeadsHash([]Head{a, b})
+	c := Candidate{Group: "feat/x", HeadsHash: hash, Pulls: []CandidatePull{{Head: a, Modules: []string{}, Merged: true}, {Head: b, Modules: []string{}}}}
+	raw, err := EncodeCandidate(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	back, err := DecodeCandidate(raw)
+	if err != nil || !back.Pulls[0].Merged || back.Pulls[1].Merged || back.HeadsHash != hash {
+		t.Fatalf("read back %+v, %v", back, err)
+	}
+	c.Pulls[1].Merged = true
+	if _, err := EncodeCandidate(c); err == nil {
+		t.Error("a candidate whose every member merged was written")
+	}
+}

go/delivery/checker.go

@@ -139,16 +139,25 @@ type FailedFirst struct {
 	Words   string `json:"words,omitempty"`
 }
 
+// MaxFailures is the most entries a verdict's Failures holds (issue 1921): fifty tests named one by one, and an entry
+// after them saying how many more failed.
+const MaxFailures = 50
+
 // Verdict is the value of a verdict key: the group and its heads hash, the trunk head of each repository it was judged
 // against (repository to full sha), the State (running, pass or fail), what failed first, the tests run and kept per
 // level, each part's time, the machines composed and the mesh-sdk version the composition ran with (ADR 0446 clause 5).
 type Verdict struct {
-	V        int                    `json:"v"`
+	V      int               `json:"v"`
-	Group    string                 `json:"group"`
+	Group  string            `json:"group"`
-	Heads    string                 `json:"heads"`
+	Heads  string            `json:"heads"`
-	Trunk    map[string]string      `json:"trunk,omitempty"`
+	Trunk  map[string]string `json:"trunk,omitempty"`
-	State    string                 `json:"state"`
+	State  string            `json:"state"`
-	Failed   FailedFirst            `json:"failed,omitzero"`
+	Failed FailedFirst       `json:"failed,omitzero"`
+	// Failures names every test that failed, in the order judged, each with the first line of its words (issue 1921):
+	// Failed keeps the first one's words whole, and a check that ran every test says all it found here. At most
+	// MaxFailures entries, the last of a longer list saying how many more failed. Absent in a value written before it,
+	// and from a verdict whose failure is no test.
+	Failures []FailedFirst          `json:"failures,omitempty"`
 	Levels   map[string]LevelResult `json:"levels,omitempty"`
 	Took     Took                   `json:"took,omitzero"`
 	Machines []string               `json:"machines,omitempty"`
@@ -222,6 +231,12 @@ func (v Verdict) check() error {
 		}
 	}
 	named := v.Failed.Test != "" || v.Failed.Machine != ""
+	if len(v.Failures) > 0 && v.State != VerdictFail {
+		return fmt.Errorf("delivery: a %s verdict names no failures", v.State)
+	}
+	if len(v.Failures) > MaxFailures+1 {
+		return fmt.Errorf("delivery: the verdict names %d failures, more than %d and the count of the rest", len(v.Failures), MaxFailures)
+	}
 	switch v.State {
 	case VerdictRunning:
 		if v.Failed != (FailedFirst{}) {

go/delivery/checker_test.go

@@ -3,6 +3,7 @@ package delivery
 import (
 	"encoding/json"
 	"errors"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -600,3 +601,38 @@ func TestPreviewOfAHeadInConflict(t *testing.T) {
 		t.Errorf("the old reader: %+v %v", old, err)
 	}
 }
+
+// Issue 1921: a failed verdict names every failed test in Failures and reads back the same; an older reader's value
+// without it reads too; Failures on a verdict that is no fail, and more than MaxFailures and the count of the rest, are
+// refused.
+func TestVerdictNamesEveryFailure(t *testing.T) {
+	v := passVerdict()
+	v.State = VerdictFail
+	v.Failed = FailedFirst{Test: "integration: x/a.TestA", Words: "a_test.go:9: want 1, got 2\nmore"}
+	v.Failures = []FailedFirst{{Test: "integration: x/a.TestA", Words: "a_test.go:9: want 1, got 2"}, {Test: "integration: x/b.TestB", Words: "b_test.go:3: boom"}}
+	data, err := EncodeVerdict(v)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DecodeVerdict(data)
+	v.V = Version
+	if err != nil || !SameVerdict(got, v) || !strings.Contains(string(data), `"failures":[{"test":"integration: x/a.TestA"`) {
+		t.Fatalf("%s: %+v %v", data, got, err)
+	}
+	old := strings.Replace(string(data), string(data[strings.Index(string(data), `,"failures":`):strings.Index(string(data), `,"levels":`)]), "", 1)
+	if got, err := DecodeVerdict([]byte(old)); err != nil || len(got.Failures) != 0 || got.Failed != v.Failed {
+		t.Fatalf("a value without failures: %+v %v (%s)", got, err, old)
+	}
+	p := passVerdict()
+	p.Failures = v.Failures
+	if _, err := EncodeVerdict(p); err == nil {
+		t.Error("a pass with failures was written")
+	}
+	v.Failures = make([]FailedFirst, MaxFailures+2)
+	for i := range v.Failures {
+		v.Failures[i] = FailedFirst{Test: "unit: x.T" + strconv.Itoa(i)}
+	}
+	if _, err := EncodeVerdict(v); err == nil {
+		t.Error("a verdict naming more than MaxFailures and the rest was written")
+	}
+}

go/delivery/source.go

@@ -269,6 +269,10 @@ type CandidatePull struct {
 	Head
 	Modules []string                   `json:"modules"`
 	Fps     map[string]CandidateModule `json:"fps,omitempty"`
+	// Merged says the pull request merged while another member of its group is still open: it stays in the candidate
+	// with the head it merged at, so the heads hash an Approve covers holds until every member merged, and the checker
+	// judges the members still open by the verdict the group had (issue 1791). A reader of the older shape ignores it.
+	Merged bool `json:"merged,omitempty"`
 }
 
 // CandidateNoneNoModule is the None2 of a candidate's module whose directory the merge preview names no manifest for:
@@ -346,6 +350,13 @@ func (c Candidate) check() error {
 	if len(c.Pulls) == 0 {
 		return fmt.Errorf("delivery: a candidate names no pull request")
 	}
+	open := false
+	for _, p := range c.Pulls {
+		open = open || !p.Merged
+	}
+	if !open {
+		return fmt.Errorf("delivery: every pull request of the candidate merged: the group is no candidate any more")
+	}
 	heads := make([]Head, len(c.Pulls))
 	for i, p := range c.Pulls {
 		heads[i] = p.Head