mesh-delivery/modules/mesh-delivery-checker/internal/check/examiner_test.go

package check

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)

// captured reads a result the build module's own code marshalled (testdata/captured/README).
func captured[T any](t testing.TB, name string) T {
	t.Helper()
	raw, err := os.ReadFile(filepath.Join("testdata", "captured", name+".json"))
	if err != nil {
		t.Fatal(err)
	}
	var v T
	if err := json.Unmarshal(raw, &v); err != nil {
		t.Fatal(err)
	}
	return v
}

type stepCall struct {
	module, node, tool string
	args               map[string]any
}

// fakeSteps stands for the build modules: a handler for each tool answers what the module's code answers.
type fakeSteps struct {
	mu       sync.Mutex
	calls    []stepCall
	handlers map[string]func(c stepCall) (any, error)
}

func (f *fakeSteps) Run(_ context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) {
	c := stepCall{module, node, tool, args}
	f.mu.Lock()
	f.calls = append(f.calls, c)
	h := f.handlers[tool]
	f.mu.Unlock()
	if h == nil {
		return nil, fmt.Errorf("the fake build module has no tool %s", tool)
	}
	v, err := h(c)
	if err != nil {
		return nil, err
	}
	return json.Marshal(v)
}

func (f *fakeSteps) tool(name string) []stepCall {
	f.mu.Lock()
	defer f.mu.Unlock()
	var out []stepCall
	for _, c := range f.calls {
		if strings.HasSuffix(c.tool, "_"+name) {
			out = append(out, c)
		}
	}
	return out
}

type entries []delivery.NodeToolchain

func (e entries) Entries(language string) []delivery.NodeToolchain {
	var out []delivery.NodeToolchain
	for _, x := range e {
		if x.Language == language {
			out = append(out, x)
		}
	}
	return out
}

type langs map[string][]string

func (l langs) BuildModules(repository string) []string {
	if m, ok := l[repository]; ok {
		return m
	}
	return []string{buildrun.GoBuildModule}
}

func goEntries(nodes ...string) entries {
	var e entries
	for _, n := range nodes {
		e = append(e, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "7"}})
		e = append(e, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: "typescript", Version: "node24", Recipe: "3"}})
	}
	return e
}

const digC = "cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33"
const digD = "dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44"

type bench struct {
	ex     *Examiner
	asks   *buildrun.FakeAsks
	steps  *fakeSteps
	kept   *memKV
	prev   *memKV
	fetch  Fetched
	digest map[string]Digested // by level
	test   func(c stepCall) (any, error)
}

// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
func refsOf(v any) []BesideRef {
	raw, _ := json.Marshal(v)
	var refs []BesideRef
	_ = json.Unmarshal(raw, &refs)
	return refs
}

// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"}

// answer runs a check ask as the build module that pulls it does (buildstep.RunAsk), with the fake tools on shanks and the
// bench's kept verdicts, and answers the step it writes.
func (b *bench) answer(a delivery.StepAsk) delivery.Step {
	module, _ := delivery.BuildModuleOf(a.Language)
	const node = "shanks"
	tool := func(name string) buildstep.StepFunc {
		return func(ctx context.Context, args map[string]any) (any, error) {
			raw, err := b.steps.Run(ctx, module, node, toolPrefix[module]+name, args)
			if err != nil {
				return nil, err
			}
			return raw, nil
		}
	}
	steps := buildstep.AskSteps{Fetch: tool("fetch"), Test: tool("test"),
		Kept: func(ctx context.Context, level, digest string) (int, bool, error) {
			k, found, err := (Kept{Store: b.kept}).Lookup(ctx, level, digest)
			return k.Tests, found, err
		}}
	if module == buildrun.GoBuildModule {
		steps.Digest = tool("digest")
	}
	out, err := buildstep.RunAsk(context.Background(), steps, a)
	began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
	step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
	if err != nil {
		step.State, step.Reason = delivery.StepFailed, err.Error()
		return step
	}
	step.Result, _ = json.Marshal(out)
	return step
}

// newBench is an examiner over fake build modules that merge cleanly, digest the captured fixture's packages and pass
// the captured slow fixture's result at both levels, on shanks.
func newBench(t testing.TB) *bench {
	b := &bench{steps: &fakeSteps{}, kept: newMemKV(), prev: newMemKV()}
	b.fetch = captured[Fetched](t, "go-fetch-clean")
	b.fetch.Modules, b.fetch.Changed, b.fetch.Trees = []string{"."}, []string{"README.md"}, nil
	one := captured[Digested](t, "go-digest")
	b.digest = map[string]Digested{
		delivery.LevelUnit:        {Level: "unit", Packages: map[string]string{"fixture/slow": digA}},
		delivery.LevelIntegration: {Level: "integration", Packages: map[string]string{"fixture/slow": digB}},
	}
	_ = one
	slow := captured[Tested](t, "go-test-slow")
	b.test = func(c stepCall) (any, error) {
		r := slow
		r.Level, _ = c.args["level"].(string)
		return r, nil
	}
	b.steps.handlers = map[string]func(stepCall) (any, error){
		"build_go_fetch":   func(stepCall) (any, error) { return b.fetch, nil },
		"build_node_fetch": func(stepCall) (any, error) { return b.fetch, nil },
		"build_go_digest":  func(c stepCall) (any, error) { return echoBeside(b, b.digest[c.args["level"].(string)]), nil },
		"build_go_test":    func(c stepCall) (any, error) { return b.test(c) },
		"build_node_test":  func(c stepCall) (any, error) { return b.test(c) },
	}
	now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
	b.asks = &buildrun.FakeAsks{Answer: b.answer}
	b.ex = &Examiner{Asks: buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 5 * time.Second}, Toolchains: goEntries("shanks"), Kept: Kept{Store: b.kept}, Previews: b.prev,
		Modules: langs{}, Trunks: &Trunks{}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }}
	return b
}

func caseOf(t testing.TB, pulls ...delivery.CandidatePull) *Case {
	t.Helper()
	raw := candidateOf(t, "feat/x", pulls...)
	c, err := delivery.DecodeCandidate(raw)
	if err != nil {
		t.Fatal(err)
	}
	cs := CaseOf(c, map[string]string{repoA: trunkA, repoSDK: trunkS, "novox/mesh-controller": trunkA, "novox/mesh-catalog": trunkA})
	if cs == nil {
		t.Fatal("no case")
	}
	return cs
}

func (b *bench) judge(t testing.TB, cs *Case) Outcome {
	t.Helper()
	out, err := b.ex.Judge(context.Background(), cs)
	if err != nil {
		t.Fatal(err)
	}
	return out
}

func testedWith(t testing.TB, name string, edit func(*Tested)) Tested {
	r := captured[Tested](t, name)
	if edit != nil {
		edit(&r)
	}
	return r
}

func packagesOf(c stepCall) []string {
	var out []string
	if l, ok := c.args["packages"].([]string); ok {
		out = l
	}
	return out
}

func TestACleanMergeIsFetchedAtItsTrunkAndBothLevelsPass(t *testing.T) { // 1679/AC1
	b := newBench(t)
	cs := caseOf(t, pullOf(repoA, 7, head1))
	out := b.judge(t, cs)
	v := out.Verdict
	if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 2 || v.Levels[delivery.LevelIntegration].Run != 2 {
		t.Fatalf("verdict = %+v", v)
	}
	if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun {
		t.Errorf("composition = %+v, want not run", v.Composition)
	}
	if len(v.Machines) != 1 || v.Machines[0] != "shanks" {
		t.Errorf("machines = %v", v.Machines)
	}
	fetch := b.steps.tool("fetch")
	if len(fetch) != 1 || fetch[0].args["repository"] != repoA || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "refs/pull/7/head" {
		t.Fatalf("fetch calls = %+v", fetch)
	}
	if v.Took.Clone <= 0 || v.Took.Unit <= 0 {
		t.Errorf("took = %+v, want the clone and the unit level timed", v.Took)
	}
	// What passed is kept under each level's digest.
	for _, k := range []string{"unit." + digA, "integration." + digB} {
		if _, _, found, _ := b.kept.Get(context.Background(), k); !found {
			t.Errorf("%s was not kept", k)
		}
	}
}

func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
	b := newBench(t)
	b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/slow": digA, "fixture/other": digC}}
	if _, err := (Kept{Store: b.kept}).Keep(context.Background(), "unit", digC, keptValue("fixture/other", 40)); err != nil {
		t.Fatal(err)
	}
	out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	tests := b.steps.tool("test")
	if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || tests[0].args["level"] != "unit" {
		t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, then the integration level", tests)
	}
	if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
		t.Errorf("unit = %+v, want 2 run and 40 kept", u)
	}
}

func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
	b := newBench(t)
	failing := captured[Tested](t, "go-test-failing")
	b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digB}}
	b.test = func(c stepCall) (any, error) {
		if c.args["level"] == "unit" {
			return failing, nil
		}
		return captured[Tested](t, "go-test-slow"), nil
	}
	cs := caseOf(t, pullOf(repoA, 7, head1))
	first := b.judge(t, cs).Verdict
	if first.State != delivery.VerdictFail || first.Failed.Test != "unit: fixture/failing/bad.TestThreeIsFour" {
		t.Fatalf("first = %+v", first)
	}
	if _, _, found, _ := b.kept.Get(context.Background(), "unit."+digB); !found {
		t.Fatal("the package that passed in the failing run was not kept")
	}
	if _, _, found, _ := b.kept.Get(context.Background(), "unit."+digA); found {
		t.Fatal("the package that failed was kept")
	}
	// Asked again with nothing changed: only the failed package runs; the good one is taken.
	b.steps.calls = nil
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	tests := b.steps.tool("test")
	if len(tests) == 0 || strings.Join(packagesOf(tests[0]), ",") != "fixture/failing/bad" {
		t.Fatalf("the re-ask ran %+v, want only fixture/failing/bad", tests)
	}
	// The trunk moved and one digest changed: that package runs, the unchanged one is taken.
	b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digD}}
	b.steps.calls = nil
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	tests = b.steps.tool("test")
	if got := strings.Join(packagesOf(tests[0]), ","); got != "fixture/failing/bad,fixture/failing/good" && got != "fixture/failing/good,fixture/failing/bad" {
		t.Errorf("after a digest changed the run was %q, want the failed package and the changed one", got)
	}
}

func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1292/AC1
	b := newBench(t)
	b.ex.Modules = langs{repoA: {buildrun.NodeBuildModule}}
	b.fetch = captured[Fetched](t, "node-fetch-clean")
	b.test = func(c stepCall) (any, error) {
		return Tested{Level: c.args["level"].(string), Module: "modules/m", Passed: true, Elapsed: 0.4, Bound: 5}, nil
	}
	cs := caseOf(t, pullOf(repoA, 7, head1))
	for i := 0; i < 2; i++ {
		v := b.judge(t, cs).Verdict
		if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Kept != 0 || v.Levels[delivery.LevelUnit].Run != 1 {
			t.Fatalf("round %d: verdict = %+v", i, v)
		}
	}
	if n := len(b.steps.tool("test")); n != 4 {
		t.Errorf("%d node test calls in two checks, want 4 (both levels each time)", n)
	}
	if n := len(b.steps.tool("digest")); n != 0 {
		t.Errorf("the node build module was asked for a digest %d times", n)
	}
	if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 {
		t.Errorf("kept %v for a node package", keys)
	}
}

func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2
	b := newBench(t)
	b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
	b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
	failing := captured[Tested](t, "go-test-failing")
	b.test = func(c stepCall) (any, error) {
		if got := packagesOf(c); len(got) == 1 && got[0] == "m/internal/b" {
			return failing, nil
		}
		return captured[Tested](t, "go-test-slow"), nil
	}
	out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
		t.Fatalf("verdict = %+v", out.Verdict)
	}
	tests := b.steps.tool("test")
	if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
		t.Fatalf("test calls = %+v, want only m/internal/b, once", tests)
	}
	if n := len(b.steps.tool("digest")); n != 1 {
		t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
	}
	if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
		t.Errorf("composition = %+v", out.Verdict.Composition)
	}
}

func TestTheChangesOwnPassingTestRunsFirstAndTheRestFollow(t *testing.T) { // 1289/AC3
	b := newBench(t)
	b.fetch.Changed = []string{"modules/m/internal/b/b_test.go", "modules/m/internal/a/a.go"}
	b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
	out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	if out.Verdict.State != delivery.VerdictPass {
		t.Fatalf("verdict = %+v", out.Verdict)
	}
	var unit []string
	for _, c := range b.steps.tool("test") {
		if c.args["level"] == "unit" {
			unit = append(unit, strings.Join(packagesOf(c), ","))
		}
	}
	if strings.Join(unit, " | ") != "m/internal/b | m/internal/a" {
		t.Errorf("the unit level ran %v, want the changed test's package first, then the rest", unit)
	}
}

func TestAHeadThatDoesNotMergeEndsFailedNamingTheFilesAndRunsNoTest(t *testing.T) { // 1289/AC4
	b := newBench(t)
	b.fetch = captured[Fetched](t, "go-fetch-conflict")
	out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	f := out.Verdict.Failed
	if out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "files in conflict: a.go, sub/b.go") {
		t.Fatalf("verdict = %+v", out.Verdict)
	}
	if n := len(b.steps.tool("digest")) + len(b.steps.tool("test")); n != 0 {
		t.Errorf("%d digest and test steps ran after a conflict", n)
	}
}

func TestALevelWhoseDigestIsNotNamedRunsWholeAndKeepsNothing(t *testing.T) { // 1287/AC4
	b := newBench(t)
	b.steps.handlers["build_go_digest"] = func(stepCall) (any, error) { return nil, fmt.Errorf("go list failed") }
	out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	if out.Verdict.State != delivery.VerdictPass {
		t.Fatalf("verdict = %+v", out.Verdict)
	}
	tests := b.steps.tool("test")
	if len(tests) != 2 || tests[0].args["packages"] != nil {
		t.Errorf("test calls = %+v, want each level once, whole", tests)
	}
	if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 {
		t.Errorf("kept %v with no digest", keys)
	}
}

func TestALevelOverItsBoundIsNamedWithItsSlowestTestsAndKeepsItsVerdict(t *testing.T) { // 1300/AC3
	b := newBench(t)
	b.test = func(c stepCall) (any, error) {
		return testedWith(t, "go-test-slow", func(r *Tested) {
			r.Level = c.args["level"].(string)
			r.OverBound = true
			r.OverWhich = "run"
			r.Elapsed = 7
		}), nil
	}
	out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	v := out.Verdict
	if v.State != delivery.VerdictPass || strings.Join(v.OverBound, ",") != "unit,integration" {
		t.Fatalf("verdict = %+v", v)
	}
	if len(out.Slowest) == 0 || !strings.Contains(out.Slowest[0], "TestTakesAWhile") {
		t.Errorf("slowest = %v", out.Slowest)
	}
}

func TestSkippedTestsAreCountedPerLevel(t *testing.T) { // 1300/AC6, 1251
	b := newBench(t)
	b.test = func(c stepCall) (any, error) {
		return testedWith(t, "go-test-slow", func(r *Tested) { r.Level = c.args["level"].(string); r.Skipped = 3 }), nil
	}
	v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
	if v.Levels[delivery.LevelUnit].Skipped != 3 || v.Levels[delivery.LevelIntegration].Skipped != 3 {
		t.Errorf("levels = %+v", v.Levels)
	}
}

func TestAGroupIsOneVerdictAndEachMembersTestsReadTheOthersAtTheirHeads(t *testing.T) { // 1624/AC1
	b := newBench(t)
	cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
	v := b.judge(t, cs).Verdict
	if v.State != delivery.VerdictPass {
		t.Fatalf("verdict = %+v", v)
	}
	fetch := b.steps.tool("fetch")
	if len(fetch) != 2 {
		t.Fatalf("%d fetches, want one for each member", len(fetch))
	}
	var forA []BesideRef
	for _, f := range fetch {
		if f.args["repository"] == repoA {
			forA, _ = refsOf(f.args["beside"]), true
		}
	}
	found := false
	for _, r := range forA {
		if r.Repository == repoSDK && r.Commit == head2 && r.Ref == "refs/pull/12/head" {
			found = true
		}
	}
	if !found {
		t.Errorf("mesh-delivery's tests were not given the SDK member at its head: %+v", forA)
	}
}

func TestAnExaminerIsAJudgeOfTheChecker(t *testing.T) {
	var _ Judge = (*Examiner)(nil)
}

const (
	manA = "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"
)

func TestACleanMergeWritesItsPreviewWithTheTreesManifestsAndLocks(t *testing.T) { // 1445/AC1
	b := newBench(t)
	b.fetch.Trees = map[string]MergeTree{
		"modules/avahi": {Tree: "7777777777777777777777777777777777777777", Manifest: manA, Locks: map[string]string{"go.sum": "ff00"}},
		"modules/bare":  {Tree: "8888888888888888888888888888888888888888"}, // no manifest digest: left out
	}
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	key, _ := delivery.PreviewKey(repoA, head1, trunkA)
	raw, _, found, _ := b.prev.Get(context.Background(), key)
	if !found {
		t.Fatalf("no preview %s; the bucket holds %v", key, b.prev.vals)
	}
	pv, err := delivery.DecodePreview(raw)
	if err != nil {
		t.Fatal(err)
	}
	m := pv.Modules["modules/avahi"]
	if pv.Merge != b.fetch.Merge || pv.Trunk != trunkA || pv.Head != head1 || len(pv.Modules) != 1 || m.Tree != "7777777777777777777777777777777777777777" || m.Manifest != manA || m.Locks["go.sum"] != "ff00" {
		t.Errorf("preview = %+v", pv)
	}
}

func TestAHeadThatDoesNotMergeWritesAPreviewOfItsConflictsAlone(t *testing.T) { // 1445/AC5
	b := newBench(t)
	b.fetch = captured[Fetched](t, "go-fetch-conflict")
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	key, _ := delivery.PreviewKey(repoA, head1, trunkA)
	raw, _, found, _ := b.prev.Get(context.Background(), key)
	if !found {
		t.Fatalf("no preview for a head that does not merge; the bucket holds %v", b.prev.vals)
	}
	pv, err := delivery.DecodePreview(raw)
	if err != nil || strings.Join(pv.Conflicts, ",") != "a.go,sub/b.go" || pv.Merge != "" || len(pv.Modules) != 0 {
		t.Errorf("preview = %+v, %v, want the files in conflict and no merge", pv, err)
	}
}

func TestAPreviewIsWrittenOnceAndAnExistingKeyStands(t *testing.T) {
	b := newBench(t)
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	b.fetch.Merge = "5656565656565656565656565656565656565656"
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	key, _ := delivery.PreviewKey(repoA, head1, trunkA)
	if b.prev.creates[key] != 1 {
		t.Errorf("the preview was created %d times, want once", b.prev.creates[key])
	}
	raw, _, _, _ := b.prev.Get(context.Background(), key)
	if pv, _ := delivery.DecodePreview(raw); pv.Merge == b.fetch.Merge {
		t.Error("the second merge commit replaced the first")
	}
}

func TestTheIndexerIsToldOfAPreviewOnceWhenItIsCreated(t *testing.T) { // 1445/AC4 (the checker's half)
	b := newBench(t)
	var told []string
	b.ex.Announce = func(_ context.Context, event, key string) { told = append(told, event+" "+key) }
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	key, _ := delivery.PreviewKey(repoA, head1, trunkA)
	if len(told) != 1 || told[0] != EventPreviewWritten+" "+key {
		t.Errorf("told %v, want once of %s", told, key)
	}
}

type fakeBuilds map[string]delivery.BuilderBuild

func (f fakeBuilds) Build(_ context.Context, module, fp2 string) (delivery.BuilderBuild, bool, error) {
	b, ok := f[module+" "+fp2]
	return b, ok, nil
}

func TestTheOutcomeSaysIfEachCandidatesModuleIsBuiltFailedOrWaiting(t *testing.T) { // N7/AC1 (the checker reads the builder's keys)
	b := newBench(t)
	fpA := "src2:" + strings.Repeat("a", 64)
	fpB := "src2:" + strings.Repeat("b", 64)
	fpC := "src2:" + strings.Repeat("c", 64)
	b.ex.Builds = fakeBuilds{
		"avahi " + fpA:      {State: delivery.BuildDone},
		"mesh-tools " + fpB: {State: delivery.BuildFailed, Error: "boom"},
		// netcheck has no key yet
	}
	p := pullOf(repoA, 7, head1)
	p.Modules = []string{"avahi", "mesh-tools", "netcheck", "nofp"}
	p.Fps = map[string]delivery.CandidateModule{"avahi": {FP2: fpA}, "mesh-tools": {FP2: fpB}, "netcheck": {FP2: fpC}, "nofp": {None2: "toolchain not yet reported"}}
	out := b.judge(t, caseOf(t, p))
	var got []string
	for _, n := range out.Builds {
		got = append(got, n.Module+" "+n.State)
	}
	if strings.Join(got, ", ") != "avahi built, mesh-tools failed, netcheck waiting" {
		t.Errorf("builds = %v, want one note for each module that has a fingerprint, sorted", got)
	}
}

// trunksOf are trunk heads of every repository, as the gate's source names them.
func trunksOf(t *testing.T, heads map[string]string) *Trunks {
	t.Helper()
	data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: heads})
	if err != nil {
		t.Fatal(err)
	}
	tr := &Trunks{}
	tr.Put(data, false)
	return tr
}

func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversThem(t *testing.T) { // review D1, ADR 0444, issue 1287
	b := newBench(t)
	// A build module whose digest does not echo the siblings (before DigestVersion 3).
	b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) { return b.digest[c.args["level"].(string)], nil }
	b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA})
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	if got := len(b.steps.tool("fetch")); got != 1 || b.steps.tool("fetch")[0].args["beside"] == nil {
		t.Fatalf("the fetch was not given siblings: %+v", b.steps.tool("fetch"))
	}
	if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 {
		t.Errorf("kept %v for a package whose digest does not say it covers the siblings", keys)
	}
	// And a kept verdict of the same digest from elsewhere is not taken for it: everything runs.
	if _, err := (Kept{Store: b.kept}).Keep(context.Background(), "unit", digA, keptValue("fixture/slow", 2)); err != nil {
		t.Fatal(err)
	}
	b.steps.calls = nil
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	if n := len(b.steps.tool("test")); n != 2 {
		t.Errorf("%d test steps, want both levels run whole", n)
	}
}

// besideDigest stands for the Go build module's digest over a workspace with siblings (DigestVersion 3, lead decision L5):
// each package's digest folds the siblings' commits in, and the answer echoes the lines it folded.
// echoBeside is d with the siblings the fetches placed, as the Go build module's digest echoes them (its DigestResult's
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
func echoBeside(b *bench, d Digested) Digested {
	var lines []string
	for _, f := range b.steps.tool("fetch") {
		refs, _ := refsOf(f.args["beside"]), true
		for _, r := range refs {
			lines = append(lines, r.Repository+" "+r.Commit)
		}
	}
	sort.Strings(lines)
	d.Beside = lines
	return d
}

func besideDigest(b *bench) func(c stepCall) (any, error) {
	return func(c stepCall) (any, error) {
		level := c.args["level"].(string)
		var lines []string
		for _, f := range b.steps.tool("fetch") {
			refs, _ := refsOf(f.args["beside"]), true
			for _, r := range refs {
				lines = append(lines, r.Repository+" "+r.Commit)
			}
		}
		sort.Strings(lines)
		sum := sha256.Sum256([]byte(level + "\n" + strings.Join(lines, "\n")))
		return Digested{Level: level, Packages: map[string]string{"fixture/slow": hex.EncodeToString(sum[:])}, Beside: lines}, nil
	}
}

func TestACoreRepositorysPassingPackageIsKeptUnderADigestCoveringItsSiblings(t *testing.T) { // 1287 AC1-AC3, AC7 with L5
	b := newBench(t)
	b.steps.handlers["build_go_digest"] = besideDigest(b)
	b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA})
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	keys, _ := b.kept.Keys(context.Background())
	if len(keys) != 2 {
		t.Fatalf("kept %v, want the passing package at both levels", keys)
	}
	// The same siblings again: everything is taken, nothing runs.
	b.steps.calls = nil
	v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
	if n := len(b.steps.tool("test")); n != 0 || v.Levels[delivery.LevelUnit].Run != 0 || v.Levels[delivery.LevelUnit].Kept == 0 {
		t.Fatalf("%d test steps, unit %+v, want none run and the kept verdict taken", n, v.Levels[delivery.LevelUnit])
	}
	// A sibling's commit moves: a new digest, so the package runs again and is kept under it.
	b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS2, "novox/mesh-catalog": trunkA})
	b.steps.calls = nil
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	if n := len(b.steps.tool("test")); n != 2 {
		t.Errorf("%d test steps after a sibling moved, want both levels run", n)
	}
	if keys, _ := b.kept.Keys(context.Background()); len(keys) != 4 {
		t.Errorf("kept %v, want a second key per level under the new digest", keys)
	}
}

func TestADigestThatEchoesOtherSiblingsKeepsNothing(t *testing.T) { // 1287 with L5: an echo that differs is not trusted
	b := newBench(t)
	inner := besideDigest(b)
	b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
		v, err := inner(c)
		d := v.(Digested)
		d.Beside = d.Beside[:len(d.Beside)-1]
		return d, err
	}
	b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA})
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 {
		t.Errorf("kept %v under a digest that does not cover every sibling", keys)
	}
}

type pkgPaths map[string][]string

func (p pkgPaths) PackagePaths(repository string) []string { return p[repository] }

// groupPackageBench is a group of a TypeScript member (mesh-delivery) and the mesh-sdk member, whose merge changes sdkChanged.
func groupPackageBench(t *testing.T, sdkChanged []string) *bench {
	b := newBench(t)
	b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}, repoSDK: {buildrun.GoBuildModule, buildrun.NodeBuildModule}}
	b.ex.Packages = pkgPaths{repoSDK: {"src", "test", "package.json", "package-lock.json", "tsconfig.json"}}
	fetch := func(c stepCall) (any, error) {
		f := b.fetch
		if c.args["repository"] == repoSDK {
			f.Changed = sdkChanged
		}
		return f, nil
	}
	b.steps.handlers["build_go_fetch"], b.steps.handlers["build_node_fetch"] = fetch, fetch
	return b
}

func nodeFetchOf(b *bench, repo string) stepCall {
	for _, c := range b.steps.tool("fetch") {
		if c.module == buildrun.NodeBuildModule && c.args["repository"] == repo {
			return c
		}
	}
	return stepCall{}
}

func TestATypeScriptMemberTakesTheGroupsPackageWhenTheSDKMemberChangesItsPaths(t *testing.T) { // ADR 0500 §5, C11b
	b := groupPackageBench(t, []string{"go/delivery/x.go", "src/state.ts"})
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2)))
	want := map[string]any{"repository": repoSDK, "commit": head2, "ref": "refs/pull/12/head"}
	if got := nodeFetchOf(b, repoA).args["package"]; !reflect.DeepEqual(got, want) {
		t.Errorf("mesh-delivery's Node fetch was given the package %v, want %v", got, want)
	}
	if got := nodeFetchOf(b, repoSDK).args["package"]; got != nil {
		t.Errorf("the mesh-sdk member's own fetch was given %v: it builds its package itself", got)
	}
}

func TestATypeScriptMemberResolvesTheRegistryWhenTheSDKMemberLeavesItsPaths(t *testing.T) { // ADR 0500 §5: the manifest alone, or Go alone
	for _, changed := range [][]string{{"go/delivery/x.go"}, {"module.json"}, {"srcx/a.ts"}} {
		b := groupPackageBench(t, changed)
		b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2)))
		if got := nodeFetchOf(b, repoA).args["package"]; got != nil {
			t.Errorf("%v: the Node fetch was given %v, want the registry", changed, got)
		}
	}
	// A lone TypeScript pull request: no group package either.
	b := groupPackageBench(t, []string{"src/state.ts"})
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	if got := nodeFetchOf(b, repoA).args["package"]; got != nil {
		t.Errorf("a lone pull request's Node fetch was given %v", got)
	}
}

func TestNoStepOfACheckCarriesPublish(t *testing.T) { // ADR 0500 §4, 1309 (the checker's half): only the builder's trunk build publishes
	b := groupPackageBench(t, []string{"src/state.ts"})
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2)))
	if len(b.steps.calls) == 0 {
		t.Fatal("no step ran")
	}
	for _, c := range b.steps.calls {
		if _, ok := c.args["publish"]; ok {
			t.Errorf("%s was given publish: %v", c.tool, c.args)
		}
		if strings.HasSuffix(c.tool, "_build") || strings.HasSuffix(c.tool, "_pack") {
			t.Errorf("the checker asked %s: a check builds no artifact of its own", c.tool)
		}
	}
}

// 1890/AC1, 1887/A5a: an ask whose step never ends (the build node stopped with it running, and nothing redelivered it in
// time) ends the check loudly at the asker's bound; the checker holds no place to free.
func TestAnAskWhoseStepNeverEndsEndsTheCheckLoudly(t *testing.T) {
	b := newBench(t)
	// A node took the ask and its step stays running: the bound runs from its start (review round 2 item 1).
	b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
		return delivery.Step{ID: a.ID, Tool: "check ask", Node: "shanks", State: delivery.StepRunning, Started: time.Now()}
	}
	b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond}
	out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || !strings.Contains(f.Words, "on shanks did not end within") {
		t.Fatalf("verdict = %+v", out.Verdict)
	}
	if n := len(b.asks.Asks()); n != 1 || b.asks.Asks()[0].Kind != delivery.AskCheck || b.asks.Asks()[0].Check == nil {
		t.Fatalf("asks = %+v, want one check ask", b.asks.Asks())
	}
}

// 1887/A5a, ADR 0536 §4: a check is one ask per pull request and build module on the check stream, a re-judge after a
// trunk move one on the recheck stream, each carrying both levels with the kept verdicts taken by the build module.
func TestACheckIsOneAskAndAReJudgeGoesOnTheRecheckStream(t *testing.T) {
	b := newBench(t)
	b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
	asks := b.asks.Asks()
	if len(asks) != 1 || asks[0].Kind != delivery.AskCheck || asks[0].Language != delivery.AskGo || asks[0].Asker != delivery.CheckerModule {
		t.Fatalf("asks = %+v, want one Go check ask", asks)
	}
	if j := asks[0].Check; j == nil || !j.Kept || strings.Join(j.Levels, ",") != "unit,integration" || j.Fetch["repository"] != repoA {
		t.Fatalf("the job = %+v", asks[0].Check)
	}
	cs := caseOf(t, pullOf(repoA, 7, head1))
	cs.Kind = KindTrunk
	b.judge(t, cs)
	if asks := b.asks.Asks(); len(asks) != 2 || asks[1].Kind != delivery.AskRecheck {
		t.Fatalf("asks = %+v, want the re-judge on recheck", asks)
	}
}

// Review round 2 item 1: a check ask that waits in its stream longer than the step bound (no node has a free slot during a
// large rebuild) is not failed: the check goes on when a node takes it, and no second ask is put.
func TestACheckAskWaitingInItsStreamIsNotBounded(t *testing.T) {
	b := newBench(t)
	answer := b.asks.Answer
	b.asks.Answer = nil
	b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond}
	done := make(chan Outcome, 1)
	go func() { done <- b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) }()
	for deadline := time.Now().Add(2 * time.Second); len(b.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
		if time.Now().After(deadline) {
			t.Fatal("no ask was put")
		}
	}
	time.Sleep(100 * time.Millisecond) // five bounds in the stream
	select {
	case out := <-done:
		t.Fatalf("the check ended while its ask waited: %+v", out.Verdict)
	default:
	}
	b.asks.End(answer(b.asks.Asks()[0])) // a node took it and ran it
	if out := <-done; out.Verdict.State != delivery.VerdictPass || len(b.asks.Asks()) != 1 {
		t.Fatalf("verdict %+v after %d asks, want a pass from the one ask", out.Verdict, len(b.asks.Asks()))
	}
}

// 1917: the digest is taken inside the check ask, as the build module's own digest step answers it (no step key is asked
// of it), so a second check of the same heads keeps every package the first one passed: "N kept", nothing run again.
func TestASecondCheckOfTheSameHeadsKeepsWhatTheFirstPassed(t *testing.T) {
	b := newBench(t)
	first := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
	u := first.Levels[delivery.LevelUnit]
	if first.State != delivery.VerdictPass || u.Run == 0 || u.Kept != 0 {
		t.Fatalf("the first check = %+v", first)
	}
	b.steps.calls = nil
	second := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
	if s := second.Levels[delivery.LevelUnit]; second.State != delivery.VerdictPass || s.Kept != u.Run || s.Run != 0 {
		t.Fatalf("the second check's unit level = %+v, want %d kept and none run", s, u.Run)
	}
	if n := len(b.steps.tool("test")); n != 0 {
		t.Errorf("%d test steps ran on the second check", n)
	}
}