mesh-delivery/modules/mesh-delivery-checker/internal/check/checker_test.go

package check

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
	"git.novox.be/novox/mesh-delivery/sdk/buildstep"
	"git.novox.be/novox/mesh-sdk/go/keys"
)

const hqRepo = "novox/hq"

const (
	repoA   = "novox/mesh-delivery"
	repoSDK = "novox/mesh-sdk"
	head1   = "1111111111111111111111111111111111111111"
	head2   = "2222222222222222222222222222222222222222"
	trunkA  = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
	trunkA2 = "abababababababababababababababababababab"
	trunkS  = "5555555555555555555555555555555555555555"
	trunkS2 = "5656565656565656565656565656565656565656"
)

// candidateOf is a candidate of a group with the given pull requests, as the indexer writes one.
func candidateOf(t testing.TB, group string, pulls ...meshcatalog.CandidatesByGroupValuePullsItem) []byte {
	t.Helper()
	heads := make([]meshdeliveryclient.Head, len(pulls))
	for i, p := range pulls {
		heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
	}
	hash, err := meshdeliveryclient.HeadsHash(heads)
	if err != nil {
		t.Fatal(err)
	}
	c := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}
	data, err := json.Marshal(c)
	if err != nil {
		t.Fatal(err)
	}
	return data
}

func pullOf(repo string, number int, head string) meshcatalog.CandidatesByGroupValuePullsItem {
	return meshcatalog.CandidatesByGroupValuePullsItem{Repository: repo, Number: int64(number), Head: head, Modules: []string{}}
}

// newRepoPullOf is a pull request of a new repository of the mesh: its trunk not named yet, its first module added.
func newRepoPullOf(repo string, number int, head string) meshcatalog.CandidatesByGroupValuePullsItem {
	p := pullOf(repo, number, head)
	p.Modules = []string{"mod-u"}
	return p
}

// judgeFunc is a Judge written as a function; it counts its calls.
type judgeFunc struct {
	mu    sync.Mutex
	calls int
	f     func(ctx context.Context, c *Case, call int) (Outcome, error)
}

func (j *judgeFunc) Judge(ctx context.Context, c *Case) (Outcome, error) {
	j.mu.Lock()
	j.calls++
	n := j.calls
	j.mu.Unlock()
	return j.f(ctx, c, n)
}

func (j *judgeFunc) count() int { j.mu.Lock(); defer j.mu.Unlock(); return j.calls }

func passing() meshdeliverychecker.VerdictsByGroupHeadsValue {
	return meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictPass, Levels: map[string]meshdeliverychecker.VerdictsByGroupHeadsValueLevelsValue{
		buildstep.LevelUnit: {Run: 2, Kept: 3}, buildstep.LevelIntegration: {Run: 1}}}
}

func passJudge() *judgeFunc {
	return &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
}

type rig struct {
	ch       *Checker
	verdicts *memKV
	owed     *memKV
	trunks   *Trunks
}

func newRig(t testing.TB, j Judge) *rig {
	t.Helper()
	r := &rig{verdicts: newMemKV(), owed: newMemKV(), trunks: &Trunks{}}
	r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
	r.ch = &Checker{Verdicts: r.verdicts, Owed: r.owed, Judge: j, Trunks: r.trunks, Link: "https://view.invalid/checks", Now: time.Now}
	return r
}

func (r *rig) setTrunks(t testing.TB, heads map[string]string) bool {
	t.Helper()
	return r.trunks.Set(heads)
}

func (r *rig) verdict(t testing.TB, key string) meshdeliverychecker.VerdictsByGroupHeadsValue {
	t.Helper()
	raw, _, found, _ := r.verdicts.Get(context.Background(), key)
	if !found {
		t.Fatalf("no verdict %s; the bucket holds %v", key, r.verdicts.vals)
	}
	v, err := meshdeliverycheckerclient.DecodeVerdict(raw)
	if err != nil {
		t.Fatal(err)
	}
	return v
}

func keyOf(t testing.TB, group string, value []byte) string {
	t.Helper()
	c, err := meshcatalog.DecodeCandidatesByGroupValue(value)
	if err != nil {
		t.Fatal(err)
	}
	k, err := meshdeliverycheckerclient.VerdictKey(group, c.Heads)
	if err != nil {
		t.Fatal(err)
	}
	return k
}

func TestACandidateNotYetCheckedGetsItsVerdictKeyCreatedOnceAndIsJudged(t *testing.T) { // N6/AC1, 1678
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	key := keyOf(t, "feat/x", cand)
	if v := r.verdict(t, key); v.State != meshdeliverycheckerclient.VerdictPass || v.Group != "feat/x" || v.Trunk[repoA] != trunkA {
		t.Fatalf("verdict = %+v", v)
	}
	if r.verdicts.creates[key] != 1 || j.count() != 1 {
		t.Errorf("creates = %d, judged = %d, want one each", r.verdicts.creates[key], j.count())
	}
	// The same candidate heard again, with nothing changed, is not judged again.
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	if j.count() != 1 {
		t.Errorf("judged %d times after the same candidate again, want once", j.count())
	}
	// The candidate of a whole try's group is the try's: the try judges it, never the checker as a pull request's check.
	r.ch.Candidate(ctx, "try-1793913602402800439", candidateOf(t, "try-1793913602402800439", pullOf(repoA, 7, head1)))
	r.ch.Wait()
	if j.count() != 1 {
		t.Errorf("a try's candidate was judged as a check: %d judgments, want one", j.count())
	}
}

// racyKV lets another instance create the key just before this one's create.
type racyKV struct {
	*memKV
	other []byte
}

func (r racyKV) Create(ctx context.Context, key string, value []byte) (uint64, error) {
	if r.other != nil {
		_, _ = r.memKV.Create(ctx, key, r.other)
	}
	return r.memKV.Create(ctx, key, value)
}

func TestASecondInstancesCreateIsRefusedAndItDoesNothing(t *testing.T) { // N6/AC1
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	key := keyOf(t, "feat/x", cand)
	other, _ := meshdeliverycheckerclient.EncodeVerdict(meshdeliverychecker.VerdictsByGroupHeadsValue{Group: "feat/x", Heads: strings.SplitN(key, ".", 2)[1], State: meshdeliverycheckerclient.VerdictRunning})
	r.ch.Verdicts = racyKV{r.verdicts, other}
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	if j.count() != 0 {
		t.Errorf("the instance that lost the create judged %d times, want none", j.count())
	}
	if v := r.verdict(t, key); v.State != meshdeliverycheckerclient.VerdictRunning {
		t.Errorf("the other instance's key was changed: %+v", v)
	}
}

func TestARestartWithARunningKeyJudgesItAgain(t *testing.T) { // N6/AC2
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	key := keyOf(t, "feat/x", cand)
	left, _ := meshdeliverycheckerclient.EncodeVerdict(meshdeliverychecker.VerdictsByGroupHeadsValue{Group: "feat/x", Heads: strings.SplitN(key, ".", 2)[1], Trunk: map[string]string{repoA: trunkA}, State: meshdeliverycheckerclient.VerdictRunning})
	if _, err := r.verdicts.Create(ctx, key, left); err != nil {
		t.Fatal(err)
	}
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	if j.count() != 1 || r.verdict(t, key).State != meshdeliverycheckerclient.VerdictPass {
		t.Fatalf("judged %d times, verdict %+v, want the left check judged once and passed", j.count(), r.verdict(t, key))
	}
}

func TestACheckPastItsBoundFailsNamingTheStuckStep(t *testing.T) { // N6/AC3
	ctx := context.Background()
	j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) {
		c.Progress.Set("test the unit level of mesh-delivery", "shanks")
		<-ctx.Done()
		return Outcome{}, ctx.Err()
	}}
	r := newRig(t, j)
	r.ch.Bound = 30 * time.Millisecond
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	v := r.verdict(t, keyOf(t, "feat/x", cand))
	if v.State != meshdeliverycheckerclient.VerdictFail || v.Failed.Machine != "shanks" || !strings.Contains(v.Failed.Words, "stuck at: test the unit level of mesh-delivery") {
		t.Fatalf("verdict = %+v, want a failure naming shanks and the stuck step", v)
	}
}

// findingsKept is a FindingWriter that keeps what is written, by ID.
type findingsKept map[string]Finding

func (f findingsKept) Put(x Finding) error    { f[x.ID] = x; return nil }
func (f findingsKept) Delete(x Finding) error { delete(f, x.ID); return nil }

// A check that could not hear a part of the mesh says `could not run`, naming the part, and releases its claim; only the
// running of that part asks it again, once per heads and part at one fingerprint; not asked again within an hour it is the
// finding check-could-not-run, which clears when the heads are checked again, replaced, or their pull request closes
// (design 47 test 58, issue 1373; issue 1912: no check fails for want of a build node).
func TestACheckThatCouldNotRunIsAskedAgainWhenItsPartRuns(t *testing.T) { // N6/AC4, 47/58
	ctx := context.Background()
	unheard := Part{Node: "g14", Module: "forge"}
	newCase := func(t *testing.T, fails int) (*rig, *judgeFunc, findingsKept, *time.Time) {
		j := &judgeFunc{f: func(_ context.Context, _ *Case, n int) (Outcome, error) {
			if n <= fails {
				return Outcome{Verdict: failure("the forge", "novox/mesh-delivery#7: go.mod cannot be read: no responders (the forge was "+UnheardWords+")"), Unheard: unheard}, nil
			}
			return Outcome{Verdict: passing()}, nil
		}}
		r := newRig(t, j)
		found, now := findingsKept{}, time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC)
		r.ch.CouldNotRun, r.ch.Findings, r.ch.Now = newMemKV(), found, func() time.Time { return now }
		return r, j, found, &now
	}
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	key := keyOf(t, "feat/x", cand)
	for name, run := range map[string]func(t *testing.T){
		"it says could not run, naming the part, and only its running asks it again, once per fingerprint": func(t *testing.T) {
			r, j, _, _ := newCase(t, 2)
			r.ch.Candidate(ctx, "feat%2Fx", cand)
			r.ch.Wait()
			if v := r.verdict(t, key); !IsCouldNotRun(v) || !strings.Contains(v.Failed.Words, "could not run") {
				t.Fatalf("verdict = %+v, want could not run", v)
			}
			if rec, _, ok := r.ch.couldNotRunOf(ctx, key); !ok || rec.Part != unheard || rec.AskedAt != "" {
				t.Fatalf("could-not-run.%s = %+v (%v), want the part %v", key, rec, ok, unheard)
			}
			r.ch.Reconsider(ctx) // nothing ran: not asked again
			r.ch.NodesChanged(ctx)
			r.ch.PartRunning(ctx, "g14", "mail", "src2:1") // another part ran
			r.ch.Wait()
			if j.count() != 1 {
				t.Fatalf("judged %d times before its part ran, want once", j.count())
			}
			r.ch.PartRunning(ctx, "g14", "forge", "src2:1")
			r.ch.Wait()
			r.ch.PartRunning(ctx, "g14", "forge", "src2:1") // the same part at the same fingerprint asks nothing more
			r.ch.Wait()
			if j.count() != 2 {
				t.Fatalf("judged %d times after its part ran twice at one fingerprint, want twice", j.count())
			}
			if rec, _, _ := r.ch.couldNotRunOf(ctx, key); rec.AskedAt != runningAt("g14", "forge", "src2:1") {
				t.Fatalf("asked-at = %q after it could not run again", rec.AskedAt)
			}
			r.ch.PartRunning(ctx, "g14", "forge", "src2:2") // a new build of the part asks once more
			r.ch.Wait()
			if j.count() != 3 || r.verdict(t, key).State != meshdeliverycheckerclient.VerdictPass {
				t.Fatalf("judged %d times, verdict %+v, want a pass on the third", j.count(), r.verdict(t, key))
			}
			if _, _, ok := r.ch.couldNotRunOf(ctx, key); ok {
				t.Error("checked again, the record that it could not run is kept")
			}
		},
		"not asked again within the hour it is a finding, cleared when it is checked again": func(t *testing.T) {
			r, _, found, now := newCase(t, 1)
			r.ch.Candidate(ctx, "feat%2Fx", cand)
			r.ch.Wait()
			if due := r.ch.CouldNotRunDue(ctx, now.Add(59*time.Minute)); !due.Equal(now.Add(CouldNotRunBound)) || len(found) != 0 {
				t.Fatalf("before its bound: due %v, findings %v", due, found)
			}
			if due := r.ch.CouldNotRunDue(ctx, now.Add(CouldNotRunBound)); !due.IsZero() || found[key].Kind != CouldNotRunKind || !strings.Contains(found[key].Summary, "forge on g14") {
				t.Fatalf("at its bound: due %v, findings %+v, want %s naming the part", due, found, CouldNotRunKind)
			}
			if observed, err := r.ch.RaisedCouldNotRun(ctx); err != nil || len(observed) != 1 || observed[0].Kind != CouldNotRunKind || observed[0].ID != key {
				t.Fatalf("observed at its bound = %+v, %v, want its one finding, for the finding writer's statement", observed, err)
			}
			r.ch.PartRunning(ctx, "g14", "forge", "src2:1")
			r.ch.Wait()
			if observed, _ := r.ch.RaisedCouldNotRun(ctx); len(found) != 0 || len(observed) != 0 {
				t.Errorf("checked again, the finding stays: %v, observed %v", found, observed)
			}
		},
		"its heads replaced, or its pull request closed, the record and the finding go": func(t *testing.T) {
			r, _, found, now := newCase(t, 1)
			r.ch.Candidate(ctx, "feat%2Fx", cand)
			r.ch.Wait()
			r.ch.CouldNotRunDue(ctx, now.Add(CouldNotRunBound))
			r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head2)))
			r.ch.Wait()
			if _, _, ok := r.ch.couldNotRunOf(ctx, key); ok || len(found) != 0 {
				t.Fatalf("heads replaced: record kept %v, findings %v", ok, found)
			}
			r2, _, found2, now2 := newCase(t, 1)
			r2.ch.Candidate(ctx, "feat%2Fx", cand)
			r2.ch.Wait()
			r2.ch.CouldNotRunDue(ctx, now2.Add(CouldNotRunBound))
			r2.ch.Forget("feat%2Fx")
			if _, _, ok := r2.ch.couldNotRunOf(ctx, key); ok || len(found2) != 0 {
				t.Fatalf("pull request closed: record kept %v, findings %v", ok, found2)
			}
		},
	} {
		t.Run(name, run)
	}
}

// Issue 1422: a pull request of a repository the gate's source.main names no trunk head of (one never merged through the
// gate) is never dropped silently. It is not judged, its verdict says it cannot be judged and why, its status is error with
// those words, checker_status names it, and the log says it once per head. Once its trunk head is known it is judged whole;
// in a group, the members whose trunk is known are judged and the others' status says why they are not.
func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testing.T) { // N6/AC6, ADR 0484 clause 11, 1422
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	var logged []string
	r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
	const unknown = "novox/unknown"
	cand := candidateOf(t, "feat/y", newRepoPullOf(unknown, 4, head1))
	key := keyOf(t, "feat/y", cand)
	r.ch.Candidate(ctx, "feat/y", cand)
	r.ch.Wait()
	r.ch.Reconsider(ctx) // looked at again: nothing is written or said twice
	r.ch.NodesChanged(ctx)
	r.ch.Wait()
	v := r.verdict(t, key)
	if j.count() != 0 || !IsUntrunked(v) || !strings.Contains(v.Failed.Words, "the trunk head of "+unknown+" is not known") {
		t.Fatalf("judged %d times, verdict %+v, want none judged and a verdict that says it cannot be judged", j.count(), v)
	}
	if s := owedState(t, r, unknown, 4, head1); s != "error" {
		t.Errorf("%s#4 owes %q, want error", unknown, s)
	}
	if n := countWith(logged, "cannot be judged"); n != 1 {
		t.Errorf("said %d times in the log, want once for the head: %q", n, logged)
	}
	reader := func() Reader {
		ckey, _ := meshcatalogclient.CandidatesKey("feat/y")
		return Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
			Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}
	}
	st, err := reader().Status()
	if err != nil {
		t.Fatal(err)
	}
	if why := st.CannotBeJudged["feat/y"]; !strings.Contains(why, unknown) || len(st.Waiting[KindNew]) != 0 || len(st.Unjudged) != 0 {
		t.Errorf("cannot_be_judged %v, waiting %v, unjudged %v: want feat/y named as cannot be judged, not as new", st.CannotBeJudged, st.Waiting, st.Unjudged)
	}
	if rows, _, err := reader().Verdicts("feat/y"); err != nil || len(rows) != 1 || !strings.Contains(rows[0].Reports[unknown+"#4"], "cannot be judged") {
		t.Errorf("checker_verdicts %+v (%v), want the report of %s#4 to say it cannot be judged", rows, err, unknown)
	}

	// Its trunk head becomes known: the heads are judged whole, the same key rewritten, the status success.
	r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, unknown: trunkA2})
	r.ch.TrunkMoved(ctx)
	r.ch.Wait()
	if v := r.verdict(t, key); j.count() != 1 || v.State != meshdeliverycheckerclient.VerdictPass || v.Trunk[unknown] != trunkA2 {
		t.Errorf("judged %d times, verdict %+v, want one pass on %s", j.count(), v, trunkA2)
	}
	if s := owedState(t, r, unknown, 4, head1); s != "success" {
		t.Errorf("%s#4 owes %q once its trunk is known, want success", unknown, s)
	}

	// Issue 1733: the gate asks the Approve only on a verdict, so no candidate is left without one. A group with a member
	// whose trunk is not known is not judged in part: its verdict fails naming that repository, and every status says so; a
	// running verdict left for heads that cannot be judged is replaced; a verdict key deleted is judged again on its delete.
	for _, row := range []struct {
		name    string
		pulls   []meshcatalog.CandidatesByGroupValuePullsItem
		running bool // a running verdict is left under the key first
		deleted bool // the key is deleted after its verdict
		judged  int
		want    string // the verdict's state, and for a failure the repository it names
	}{
		{name: "a member the checker cannot claim", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 9, head2), pullOf("novox/other", 2, head2)}, want: "novox/other"},
		{name: "a running verdict left for heads that cannot be judged", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{newRepoPullOf(unknown, 5, head2)}, running: true, want: unknown},
		{name: "a verdict key deleted", pulls: []meshcatalog.CandidatesByGroupValuePullsItem{pullOf(repoA, 5, head2)}, deleted: true, judged: 2, want: meshdeliverycheckerclient.VerdictPass},
	} {
		t.Run(row.name, func(t *testing.T) {
			j := passJudge()
			r := newRig(t, j)
			cand := candidateOf(t, "feat/z", row.pulls...)
			key := keyOf(t, "feat/z", cand)
			if row.running {
				group, heads, _ := meshdeliverycheckerclient.ParseVerdictKey(key)
				raw, _ := meshdeliverycheckerclient.EncodeVerdict(meshdeliverychecker.VerdictsByGroupHeadsValue{Group: group, Heads: heads, State: meshdeliverycheckerclient.VerdictRunning})
				_, _ = r.verdicts.Create(ctx, key, raw)
			}
			r.ch.Candidate(ctx, "feat%2Fz", cand)
			r.ch.Wait()
			if row.deleted {
				_ = r.verdicts.Delete(ctx, key)
				r.ch.VerdictDeleted(ctx, key) // the watch of the verdicts hands the delete over
				r.ch.Wait()
			}
			v := r.verdict(t, key)
			if row.want == meshdeliverycheckerclient.VerdictPass {
				if v.State != meshdeliverycheckerclient.VerdictPass || j.count() != row.judged {
					t.Errorf("verdict %s after %d judgings, want a pass after %d", v.State, j.count(), row.judged)
				}
				return
			}
			if !IsUntrunked(v) || !strings.Contains(v.Failed.Words, row.want) || j.count() != 0 {
				t.Errorf("verdict %+v after %d judgings, want a failure naming %s and none judged", v, j.count(), row.want)
			}
			for _, p := range row.pulls {
				if s := owedState(t, r, p.Repository, int(p.Number), p.Head); s != "error" {
					t.Errorf("%s#%d owes %q, want error", p.Repository, p.Number, s)
				}
			}
		})
	}

	// ADR 0664 §5 (issue 1748): a candidate taken out (its pull request merged or closed) ends the check running for it with a
	// cause its asks' wait names, never a plain cancel, so its asker drops the asks it put on the build nodes.
	t.Run("a candidate taken out ends its check with a named cause", func(t *testing.T) {
		ch := &Checker{}
		ch.init()
		run, end := context.WithCancelCause(context.Background())
		ch.live["g.heads"] = liveCheck{group: "g", end: end}
		// The check's asker is waiting on an ask of its run when the candidate goes (review 1, finding 7).
		drops := &checkDrops{}
		fake := &buildrun.FakeAsks{}
		done := make(chan struct{})
		go func() {
			defer close(done)
			(buildrun.Asker{Seats: testSeats(), Publisher: fake, Waiter: fake, Bound: time.Hour, Dropper: drops}).Wait(run, buildstep.StepAsk{ID: "mvl0frv18d2000", Language: buildstep.AskGo, Kind: buildstep.AskCheck})
		}()
		ch.Forget("g")
		<-done
		var s superseded
		if cause := context.Cause(run); !errors.As(cause, &s) || s.by != "" || errors.Is(cause, context.Canceled) {
			t.Errorf("the check was ended with %v, want superseded with no heads, not a plain cancel", cause)
		}
		if len(drops.got) != 1 || !strings.Contains(drops.got[0], "mvl0frv18d2000") || !strings.Contains(drops.got[0], "no candidate holds these heads") {
			t.Errorf("the asker dropped %q, want its ask withdrawn with the check's cause", drops.got)
		}
	})
}

// checkDrops keeps the asks an asker dropped: "<id> <why>".
type checkDrops struct{ got []string }

func (d *checkDrops) DropAsk(_ context.Context, a buildstep.StepAsk, why string) error {
	d.got = append(d.got, a.ID+" "+why)
	return nil
}

func countWith(lines []string, words string) int {
	n := 0
	for _, l := range lines {
		if strings.Contains(l, words) {
			n++
		}
	}
	return n
}

// 1887/A10, ADR 0527 §2: hq is judged as any repository, its checks Go tests, and its pull request gets a verdict and a
// mesh/check; in a group, hq's member is judged with the others.
func TestAnHqPullRequestGetsAVerdictAndAStatus(t *testing.T) {
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA2})
	r.ch.Candidate(ctx, "feat/hq", candidateOf(t, "feat/hq", pullOf(hqRepo, 3, head1)))
	r.ch.Wait()
	if j.count() != 1 || len(r.verdicts.vals) != 1 {
		t.Fatalf("judged %d times, %d verdicts, want one", j.count(), len(r.verdicts.vals))
	}
	if s := owedState(t, r, hqRepo, 3, head1); s != "success" {
		t.Errorf("hq#3 owes %q, want success", s)
	}
	r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(hqRepo, 4, head2), pullOf(repoA, 7, head2)))
	r.ch.Wait()
	if s := owedState(t, r, hqRepo, 4, head2); s != "success" {
		t.Errorf("hq#4 in a group owes %q, want success", s)
	}
}

func TestATrunkMoveRewritesTheSameVerdictKeyAndTheHeadsStatus(t *testing.T) { // 1300/AC7
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	key := keyOf(t, "feat/x", cand)
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	_, rev1, _, _ := r.verdicts.Get(ctx, key)
	owedKey, _ := CheckerOwedKey(repoA, 7, head1)
	_, orev1, _, _ := r.owed.Get(ctx, owedKey)
	if !r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) {
		t.Fatal("the trunks did not change")
	}
	r.ch.Reconsider(ctx)
	r.ch.Wait()
	_, rev2, _, _ := r.verdicts.Get(ctx, key)
	if j.count() != 2 || rev2 <= rev1 || r.verdict(t, key).Trunk[repoA] != trunkA2 {
		t.Fatalf("judged %d times, revisions %d then %d, trunk %v", j.count(), rev1, rev2, r.verdict(t, key).Trunk)
	}
	if r.verdicts.creates[key] != 1 {
		t.Errorf("the key was created %d times, want the same key rewritten", r.verdicts.creates[key])
	}
	if _, orev2, _, _ := r.owed.Get(ctx, owedKey); orev2 <= orev1 {
		// the line is the same when nothing differs; the status is rewritten when the words change, here the trunk is in them
		t.Logf("the status keeps revision %d (its words did not change)", orev2)
	}
	// A new head is another key, and the older head's status is not touched.
	older, _, _, _ := r.owed.Get(ctx, owedKey)
	cand2 := candidateOf(t, "feat/x", pullOf(repoA, 7, head2))
	r.ch.Candidate(ctx, "feat%2Fx", cand2)
	r.ch.Wait()
	if again, _, _, _ := r.owed.Get(ctx, owedKey); string(again) != string(older) {
		t.Errorf("the older head's status was rewritten")
	}
	if n := len(r.owed.vals); n != 2 {
		t.Errorf("%d statuses, want one for each head", n)
	}
}

func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T) { // 1300/AC5 (unit part)
	ctx := context.Background()
	var seen []string
	var r *rig
	j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) {
		key, _ := CheckerOwedKey(repoA, 7, head1)
		raw, _, _, _ := r.owed.Get(ctx, key)
		o, _ := DecodeCheckerOwed(raw)
		seen = append(seen, o.State)
		c.series(repoA+"#7", []string{"web/pkg series 0.1 → 0.2"}, []meshdeliverychecker.VerdictsByGroupHeadsValueSeriesItem{{Module: "web", Artifact: "pkg", From: "0.1", To: "0.2"}})
		return Outcome{Verdict: meshdeliverychecker.VerdictsByGroupHeadsValue{State: meshdeliverycheckerclient.VerdictFail, Failed: &FailedFirst{Test: "m/a.TestX"}},
			OutOfStep: map[string][]string{repoA + "#7": {"go/seats/x/contract.go"}}}, nil
	}}
	r = newRig(t, j)
	says := &verdictsSaid{}
	r.ch.Says = says
	r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
	r.ch.Wait()
	key, _ := CheckerOwedKey(repoA, 7, head1)
	raw, _, _, _ := r.owed.Get(ctx, key)
	o, err := DecodeCheckerOwed(raw)
	if err != nil {
		t.Fatal(err)
	}
	if len(seen) != 1 || seen[0] != "pending" || o.State != "failure" || o.Context != meshdeliveryclient.StatusContextCheck || !strings.HasPrefix(o.Target, "https://view.invalid/checks/") {
		t.Errorf("while running %v, at the end %+v", seen, o)
	}
	// DLV-C2 (design 26 *Out of step*): the check's end says one verdict event for the pull request, naming its out-of-step
	// files; the running status says none.
	if len(says.said) != 1 || says.said[0].Repository != repoA || says.said[0].Pull != 7 || says.said[0].Head != head1 ||
		says.said[0].Verdict != meshdeliverycheckerclient.VerdictFail || strings.Join(says.said[0].OutOfStep, ",") != "go/seats/x/contract.go" ||
		// The join (A2j finding 2): and its series changes, as the seat's event names them.
		len(says.said[0].Series) != 1 || says.said[0].Series[0] != (meshdeliverychecker.VerdictHolderEventSeriesItem{Module: "web", Artifact: "pkg", From: "0.1", To: "0.2"}) {
		t.Errorf("verdict events %+v", says.said)
	}
}

// verdictsSaid is a VerdictSayer that keeps what it was told.
type verdictsSaid struct {
	mu   sync.Mutex
	said []VerdictEvent
}

func (s *verdictsSaid) Say(_ context.Context, v VerdictEvent) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.said = append(s.said, v)
	return nil
}

// No check of heads no candidate holds is left running (issue 1890): at start the sweep ends a key left running; while the
// checker runs, a check whose group moved to other heads is ended at once, unjudged, and writes no status; heads held again
// are checked whole; and a group's check starts only once its heads settled, so a partial set of members is never checked.
func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) {
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	live := keyOf(t, "feat/x", cand)
	dead, _ := meshdeliverycheckerclient.VerdictKey("feat/gone", "0123456789ab")
	for _, k := range []string{live, dead} {
		group, heads, _ := meshdeliverycheckerclient.ParseVerdictKey(k)
		raw, _ := meshdeliverycheckerclient.EncodeVerdict(meshdeliverychecker.VerdictsByGroupHeadsValue{Group: group, Heads: heads, State: meshdeliverycheckerclient.VerdictRunning})
		_, _ = r.verdicts.Create(ctx, k, raw)
	}
	r.ch.init()
	var c meshcatalog.CandidatesByGroupValue
	_ = json.Unmarshal(cand, &c)
	r.ch.known["feat%2Fx"] = c
	if err := r.ch.Sweep(ctx); err != nil {
		t.Fatal(err)
	}
	if v := r.verdict(t, dead); !IsEnded(v) || !strings.Contains(v.Failed.Words, "closed at start") {
		t.Errorf("the dead key = %+v", v)
	}
	if v := r.verdict(t, live); v.State != meshdeliverycheckerclient.VerdictRunning {
		t.Errorf("the live key = %+v, want it left for the check to take over", v)
	}

	// A member joins while the check of the lone member's heads runs: that check is ended, its judge's run cancelled.
	started, ended := make(chan struct{}), make(chan error, 1)
	j = &judgeFunc{f: func(run context.Context, _ *Case, call int) (Outcome, error) {
		if call == 1 {
			close(started)
			<-run.Done()
			ended <- context.Cause(run)
			return Outcome{}, run.Err()
		}
		return Outcome{Verdict: passing()}, nil
	}}
	r = newRig(t, j)
	var logged []string
	var logMu sync.Mutex
	r.ch.Logf = func(f string, a ...any) { logMu.Lock(); logged = append(logged, fmt.Sprintf(f, a...)); logMu.Unlock() }
	alone, both := cand, candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 3, head2))
	r.ch.Candidate(ctx, "feat%2Fx", alone)
	<-started
	r.ch.Candidate(ctx, "feat%2Fx", both)
	var bothC meshcatalog.CandidatesByGroupValue
	_ = json.Unmarshal(both, &bothC)
	select {
	case err := <-ended:
		if err == nil || !strings.Contains(err.Error(), "superseded by "+GateHeads(bothC)) {
			t.Errorf("the old check's run ended with %v", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("the check of the heads the candidate left still runs")
	}
	r.ch.Wait()
	if v := r.verdict(t, live); !IsEnded(v) || v.Failed.Words != EndedWords+"superseded by "+GateHeads(bothC) {
		t.Errorf("the superseded verdict = %+v", v.Failed)
	}
	if v := r.verdict(t, keyOf(t, "feat/x", both)); v.State != meshdeliverycheckerclient.VerdictPass {
		t.Errorf("the group's verdict = %s", v.State)
	}
	okey, _ := CheckerOwedKey(repoA, 7, head1)
	if raw, _, _, _ := r.owed.Get(ctx, okey); !strings.Contains(string(raw), `"success"`) {
		t.Errorf("the head's status = %s, want the group's success, never the ended check's", raw)
	}
	logMu.Lock()
	said := 0
	for _, l := range logged {
		if strings.Contains(l, "ended: superseded") {
			said++
		}
	}
	logMu.Unlock()
	if said != 1 {
		t.Errorf("the end was said %d times: %v", said, logged)
	}
	// The member leaves again: the ended heads are held again and checked whole.
	r.ch.Candidate(ctx, "feat%2Fx", alone)
	r.ch.Wait()
	if v := r.verdict(t, live); v.State != meshdeliverycheckerclient.VerdictPass || j.count() != 3 {
		t.Errorf("the heads held again = %s after %d judgings, want a pass of a third", v.State, j.count())
	}

	// Members announced one after another: only the settled, whole set is checked.
	j = passJudge()
	r = newRig(t, j)
	r.ch.Settle = time.Hour
	sdk := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2))
	whole := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2), pullOf(repoA, 8, head1))
	r.ch.Candidate(ctx, "feat%2Fy", sdk)
	r.ch.Candidate(ctx, "feat%2Fy", whole)
	r.ch.Reconsider(ctx) // a trunk move or a turn while the heads settle starts nothing
	if j.count() != 0 {
		t.Fatalf("judged %d times before the heads settled", j.count())
	}
	r.ch.settleNow()
	r.ch.Wait()
	if _, _, found, _ := r.verdicts.Get(ctx, keyOf(t, "feat/y", sdk)); found || j.count() != 1 {
		t.Errorf("the partial set has a verdict (%v) or the judge ran %d times, want the whole set alone", found, j.count())
	}
	if v := r.verdict(t, keyOf(t, "feat/y", whole)); v.State != meshdeliverycheckerclient.VerdictPass {
		t.Errorf("the whole set = %s", v.State)
	}
}

func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683 (the checker's half of AC5)
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	var told []string
	r.ch.Announce = func(_ context.Context, event, key string) { told = append(told, event+" "+key) }
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	key, _ := CheckerOwedKey(repoA, 7, head1)
	want := []string{EventOwedWritten + " " + key, EventOwedWritten + " " + key} // pending, then success
	if len(told) != 2 || told[0] != want[0] || told[1] != want[1] {
		t.Fatalf("told %v, want the status said when pending and when success", told)
	}
	r.ch.Candidate(ctx, "feat%2Fx", cand) // nothing changed: nothing written, nothing said
	r.ch.Wait()
	if len(told) != 2 {
		t.Errorf("told %v after a check that changed nothing", told)
	}
}

// Sweep 2026-10-30 item 7: a pull request of a repository the mesh does not deliver (an operator's own, on the same
// forge) touches no module and its trunk the gate never names. It is outside the mesh: no verdict, no status owed (one
// owed before is taken away, so the forge module stops writing it into a repository the mesh may not write), said once.
func TestACandidateOutsideTheMeshIsJudgedNothingAndOwesNothing(t *testing.T) {
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	var logged []string
	r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
	const theirs = "someone/own-app"
	okey, _ := CheckerOwedKey(theirs, 55, head1)
	before, _ := EncodeCheckerOwed(meshdeliverychecker.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: meshdeliveryclient.StatusContextCheck, State: "error", Text: "cannot be judged"})
	if _, err := r.owed.Create(ctx, okey, before); err != nil {
		t.Fatal(err)
	}
	cand := candidateOf(t, "feat/theirs", pullOf(theirs, 55, head1))
	key := keyOf(t, "feat/theirs", cand)
	r.ch.Candidate(ctx, "feat/theirs", cand)
	r.ch.Wait()
	r.ch.Reconsider(ctx)
	r.ch.Wait()
	if _, _, found, _ := r.verdicts.Get(ctx, key); found || j.count() != 0 {
		t.Errorf("a candidate outside the mesh was judged (%d) or given a verdict (%v)", j.count(), found)
	}
	if s := owedState(t, r, theirs, 55, head1); s != "" {
		t.Errorf("%s#55 still owes %q on the forge", theirs, s)
	}
	if n := countWith(logged, "outside the mesh"); n != 1 {
		t.Errorf("said %d times in the log, want once: %q", n, logged)
	}
	ckey, _ := meshcatalogclient.CandidatesKey("feat/theirs")
	st, err := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
		Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}.Status()
	if err != nil {
		t.Fatal(err)
	}
	if st.Outside["feat/theirs"] == "" || len(st.CannotBeJudged) != 0 || len(st.Unjudged) != 0 {
		t.Errorf("status: outside %v, cannot be judged %v, unjudged %v", st.Outside, st.CannotBeJudged, st.Unjudged)
	}
}