mesh-delivery/modules/mesh-delivery-checker/internal/check/checker_test.go

package check

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

const hqRepo = "novox/hq"

const (
	repoA   = "novox/mesh-delivery"
	repoSDK = "novox/mesh-sdk"
	head1   = "1111111111111111111111111111111111111111"
	head2   = "2222222222222222222222222222222222222222"
	trunkA  = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
	trunkA2 = "abababababababababababababababababababab"
	trunkS  = "5555555555555555555555555555555555555555"
	trunkS2 = "5656565656565656565656565656565656565656"
)

// candidateOf is a candidate of a group with the given pull requests, as the indexer writes one.
func candidateOf(t testing.TB, group string, pulls ...delivery.CandidatePull) []byte {
	t.Helper()
	heads := make([]delivery.Head, len(pulls))
	for i, p := range pulls {
		heads[i] = p.Head
	}
	hash, err := delivery.HeadsHash(heads)
	if err != nil {
		t.Fatal(err)
	}
	c := delivery.Candidate{V: delivery.Version, Group: group, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
	data, err := delivery.EncodeCandidate(c)
	if err != nil {
		t.Fatal(err)
	}
	return data
}

func pullOf(repo string, number int, head string) delivery.CandidatePull {
	return delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: head}, Modules: []string{}}
}

// judgeFunc is a Judge written as a function; it counts its calls.
type judgeFunc struct {
	mu    sync.Mutex
	calls int
	f     func(ctx context.Context, c *Case, call int) (Outcome, error)
}

func (j *judgeFunc) Judge(ctx context.Context, c *Case) (Outcome, error) {
	j.mu.Lock()
	j.calls++
	n := j.calls
	j.mu.Unlock()
	return j.f(ctx, c, n)
}

func (j *judgeFunc) count() int { j.mu.Lock(); defer j.mu.Unlock(); return j.calls }

func passing() delivery.Verdict {
	return delivery.Verdict{State: delivery.VerdictPass, Levels: map[string]delivery.LevelResult{
		delivery.LevelUnit: {Run: 2, Kept: 3}, delivery.LevelIntegration: {Run: 1}}}
}

func passJudge() *judgeFunc {
	return &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
}

type rig struct {
	ch       *Checker
	verdicts *memKV
	owed     *memKV
	trunks   *Trunks
}

func newRig(t testing.TB, j Judge) *rig {
	t.Helper()
	r := &rig{verdicts: newMemKV(), owed: newMemKV(), trunks: &Trunks{}}
	r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
	r.ch = &Checker{Verdicts: r.verdicts, Owed: r.owed, Judge: j, Trunks: r.trunks, Link: "https://view.invalid/checks", Now: time.Now}
	return r
}

func (r *rig) setTrunks(t testing.TB, heads map[string]string) bool {
	t.Helper()
	data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: heads})
	if err != nil {
		t.Fatal(err)
	}
	return r.trunks.Put(data, false)
}

func (r *rig) verdict(t testing.TB, key string) delivery.Verdict {
	t.Helper()
	raw, _, found, _ := r.verdicts.Get(context.Background(), key)
	if !found {
		t.Fatalf("no verdict %s; the bucket holds %v", key, r.verdicts.vals)
	}
	v, err := delivery.DecodeVerdict(raw)
	if err != nil {
		t.Fatal(err)
	}
	return v
}

func keyOf(t testing.TB, group string, value []byte) string {
	t.Helper()
	c, err := delivery.DecodeCandidate(value)
	if err != nil {
		t.Fatal(err)
	}
	k, err := delivery.CheckerVerdictKey(group, c.HeadsHash)
	if err != nil {
		t.Fatal(err)
	}
	return k
}

func TestACandidateNotYetCheckedGetsItsVerdictKeyCreatedOnceAndIsJudged(t *testing.T) { // N6/AC1, 1678
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	key := keyOf(t, "feat/x", cand)
	if v := r.verdict(t, key); v.State != delivery.VerdictPass || v.Group != "feat/x" || v.Trunk[repoA] != trunkA {
		t.Fatalf("verdict = %+v", v)
	}
	if r.verdicts.creates[key] != 1 || j.count() != 1 {
		t.Errorf("creates = %d, judged = %d, want one each", r.verdicts.creates[key], j.count())
	}
	// The same candidate heard again, with nothing changed, is not judged again.
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	if j.count() != 1 {
		t.Errorf("judged %d times after the same candidate again, want once", j.count())
	}
}

// racyKV lets another instance create the key just before this one's create.
type racyKV struct {
	*memKV
	other []byte
}

func (r racyKV) Create(ctx context.Context, key string, value []byte) (uint64, error) {
	if r.other != nil {
		_, _ = r.memKV.Create(ctx, key, r.other)
	}
	return r.memKV.Create(ctx, key, value)
}

func TestASecondInstancesCreateIsRefusedAndItDoesNothing(t *testing.T) { // N6/AC1
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	key := keyOf(t, "feat/x", cand)
	other, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/x", Heads: strings.SplitN(key, ".", 2)[1], State: delivery.VerdictRunning})
	r.ch.Verdicts = racyKV{r.verdicts, other}
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	if j.count() != 0 {
		t.Errorf("the instance that lost the create judged %d times, want none", j.count())
	}
	if v := r.verdict(t, key); v.State != delivery.VerdictRunning {
		t.Errorf("the other instance's key was changed: %+v", v)
	}
}

func TestARestartWithARunningKeyJudgesItAgain(t *testing.T) { // N6/AC2
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	key := keyOf(t, "feat/x", cand)
	left, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/x", Heads: strings.SplitN(key, ".", 2)[1], Trunk: map[string]string{repoA: trunkA}, State: delivery.VerdictRunning})
	if _, err := r.verdicts.Create(ctx, key, left); err != nil {
		t.Fatal(err)
	}
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	if j.count() != 1 || r.verdict(t, key).State != delivery.VerdictPass {
		t.Fatalf("judged %d times, verdict %+v, want the left check judged once and passed", j.count(), r.verdict(t, key))
	}
}

func TestACheckPastItsBoundFailsNamingTheStuckStep(t *testing.T) { // N6/AC3
	ctx := context.Background()
	j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) {
		c.Progress.Set("test the unit level of mesh-delivery", "shanks")
		<-ctx.Done()
		return Outcome{}, ctx.Err()
	}}
	r := newRig(t, j)
	r.ch.Bound = 30 * time.Millisecond
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	v := r.verdict(t, keyOf(t, "feat/x", cand))
	if v.State != delivery.VerdictFail || v.Failed.Machine != "shanks" || !strings.Contains(v.Failed.Words, "stuck at: test the unit level of mesh-delivery") {
		t.Fatalf("verdict = %+v, want a failure naming shanks and the stuck step", v)
	}
}

// A check whose forge was not heard is judged again when the nodes change (issue 1912: no check fails for want of a build
// node any more; its asks wait in the streams).
func TestAnUnheardForgeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // N6/AC4
	ctx := context.Background()
	j := &judgeFunc{f: func(_ context.Context, _ *Case, n int) (Outcome, error) {
		if n == 1 {
			return Outcome{Verdict: failure("the forge", "novox/mesh-delivery#7: go.mod cannot be read: no responders (the forge was "+UnheardWords+")")}, nil
		}
		return Outcome{Verdict: passing()}, nil
	}}
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	key := keyOf(t, "feat/x", cand)
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	if v := r.verdict(t, key); !IsNodesFailure(v) {
		t.Fatalf("verdict = %+v, want a failure of the nodes", v)
	}
	r.ch.Reconsider(ctx) // nothing changed: not judged again
	r.ch.Wait()
	if j.count() != 1 {
		t.Fatalf("judged %d times with the nodes unchanged, want once", j.count())
	}
	r.ch.NodesChanged(ctx)
	r.ch.Wait()
	if j.count() != 2 || r.verdict(t, key).State != delivery.VerdictPass {
		t.Fatalf("after the nodes changed: judged %d times, verdict %+v", j.count(), r.verdict(t, key))
	}
}

// Issue 1422: a pull request of a repository the gate's source.main names no trunk head of (one never merged through the
// gate) is never dropped silently. It is not judged, its verdict says it cannot be judged and why, its status is error with
// those words, checker_status names it, and the log says it once per head. Once its trunk head is known it is judged whole;
// in a group, the members whose trunk is known are judged and the others' status says why they are not.
func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testing.T) { // N6/AC6, ADR 0484 clause 11, 1422
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	var logged []string
	r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
	const unknown = "novox/unknown"
	cand := candidateOf(t, "feat/y", pullOf(unknown, 4, head1))
	key := keyOf(t, "feat/y", cand)
	r.ch.Candidate(ctx, "feat/y", cand)
	r.ch.Wait()
	r.ch.Reconsider(ctx) // looked at again: nothing is written or said twice
	r.ch.NodesChanged(ctx)
	r.ch.Wait()
	v := r.verdict(t, key)
	if j.count() != 0 || !IsUntrunked(v) || !strings.Contains(v.Failed.Words, "the trunk head of "+unknown+" is not known") {
		t.Fatalf("judged %d times, verdict %+v, want none judged and a verdict that says it cannot be judged", j.count(), v)
	}
	if s := owedState(t, r, unknown, 4, head1); s != "error" {
		t.Errorf("%s#4 owes %q, want error", unknown, s)
	}
	if n := countWith(logged, "cannot be judged"); n != 1 {
		t.Errorf("said %d times in the log, want once for the head: %q", n, logged)
	}
	reader := func() Reader {
		ckey, _ := delivery.CandidatesKey("feat/y")
		return Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
			Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}
	}
	st, err := reader().Status()
	if err != nil {
		t.Fatal(err)
	}
	if why := st.CannotBeJudged["feat/y"]; !strings.Contains(why, unknown) || len(st.Waiting[KindNew]) != 0 || len(st.Unjudged) != 0 {
		t.Errorf("cannot_be_judged %v, waiting %v, unjudged %v: want feat/y named as cannot be judged, not as new", st.CannotBeJudged, st.Waiting, st.Unjudged)
	}
	if rows, _, err := reader().Verdicts("feat/y"); err != nil || len(rows) != 1 || !strings.Contains(rows[0].Reports[unknown+"#4"], "cannot be judged") {
		t.Errorf("checker_verdicts %+v (%v), want the report of %s#4 to say it cannot be judged", rows, err, unknown)
	}

	// Its trunk head becomes known: the heads are judged whole, the same key rewritten, the status success.
	r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, unknown: trunkA2})
	r.ch.TrunkMoved(ctx)
	r.ch.Wait()
	if v := r.verdict(t, key); j.count() != 1 || v.State != delivery.VerdictPass || v.Trunk[unknown] != trunkA2 {
		t.Errorf("judged %d times, verdict %+v, want one pass on %s", j.count(), v, trunkA2)
	}
	if s := owedState(t, r, unknown, 4, head1); s != "success" {
		t.Errorf("%s#4 owes %q once its trunk is known, want success", unknown, s)
	}

	// Issue 1733: the gate asks the Approve only on a verdict, so no candidate is left without one. A group with a member
	// whose trunk is not known is not judged in part: its verdict fails naming that repository, and every status says so; a
	// running verdict left for heads that cannot be judged is replaced; a verdict key deleted is judged again on its delete.
	for _, row := range []struct {
		name    string
		pulls   []delivery.CandidatePull
		running bool // a running verdict is left under the key first
		deleted bool // the key is deleted after its verdict
		judged  int
		want    string // the verdict's state, and for a failure the repository it names
	}{
		{name: "a member the checker cannot claim", pulls: []delivery.CandidatePull{pullOf(repoA, 9, head2), pullOf("novox/other", 2, head2)}, want: "novox/other"},
		{name: "a running verdict left for heads that cannot be judged", pulls: []delivery.CandidatePull{pullOf(unknown, 5, head2)}, running: true, want: unknown},
		{name: "a verdict key deleted", pulls: []delivery.CandidatePull{pullOf(repoA, 5, head2)}, deleted: true, judged: 2, want: delivery.VerdictPass},
	} {
		t.Run(row.name, func(t *testing.T) {
			j := passJudge()
			r := newRig(t, j)
			cand := candidateOf(t, "feat/z", row.pulls...)
			key := keyOf(t, "feat/z", cand)
			if row.running {
				group, heads, _ := delivery.ParseCheckerVerdictKey(key)
				raw, _ := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, State: delivery.VerdictRunning})
				_, _ = r.verdicts.Create(ctx, key, raw)
			}
			r.ch.Candidate(ctx, "feat%2Fz", cand)
			r.ch.Wait()
			if row.deleted {
				_ = r.verdicts.Delete(ctx, key)
				r.ch.VerdictDeleted(ctx, key) // the watch of the verdicts hands the delete over
				r.ch.Wait()
			}
			v := r.verdict(t, key)
			if row.want == delivery.VerdictPass {
				if v.State != delivery.VerdictPass || j.count() != row.judged {
					t.Errorf("verdict %s after %d judgings, want a pass after %d", v.State, j.count(), row.judged)
				}
				return
			}
			if !IsUntrunked(v) || !strings.Contains(v.Failed.Words, row.want) || j.count() != 0 {
				t.Errorf("verdict %+v after %d judgings, want a failure naming %s and none judged", v, j.count(), row.want)
			}
			for _, p := range row.pulls {
				if s := owedState(t, r, p.Repository, p.Number, p.Head.Head); s != "error" {
					t.Errorf("%s#%d owes %q, want error", p.Repository, p.Number, s)
				}
			}
		})
	}
}

func countWith(lines []string, words string) int {
	n := 0
	for _, l := range lines {
		if strings.Contains(l, words) {
			n++
		}
	}
	return n
}

// 1887/A10, ADR 0527 §2: hq is judged as any repository, its checks Go tests, and its pull request gets a verdict and a
// mesh/check; in a group, hq's member is judged with the others.
func TestAnHqPullRequestGetsAVerdictAndAStatus(t *testing.T) {
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA2})
	r.ch.Candidate(ctx, "feat/hq", candidateOf(t, "feat/hq", pullOf(hqRepo, 3, head1)))
	r.ch.Wait()
	if j.count() != 1 || len(r.verdicts.vals) != 1 {
		t.Fatalf("judged %d times, %d verdicts, want one", j.count(), len(r.verdicts.vals))
	}
	if s := owedState(t, r, hqRepo, 3, head1); s != "success" {
		t.Errorf("hq#3 owes %q, want success", s)
	}
	r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(hqRepo, 4, head2), pullOf(repoA, 7, head2)))
	r.ch.Wait()
	if s := owedState(t, r, hqRepo, 4, head2); s != "success" {
		t.Errorf("hq#4 in a group owes %q, want success", s)
	}
}

func TestATrunkMoveRewritesTheSameVerdictKeyAndTheHeadsStatus(t *testing.T) { // 1300/AC7
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	key := keyOf(t, "feat/x", cand)
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	_, rev1, _, _ := r.verdicts.Get(ctx, key)
	owedKey, _ := delivery.CheckerOwedKey(repoA, 7, head1)
	_, orev1, _, _ := r.owed.Get(ctx, owedKey)
	if !r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) {
		t.Fatal("the trunks did not change")
	}
	r.ch.Reconsider(ctx)
	r.ch.Wait()
	_, rev2, _, _ := r.verdicts.Get(ctx, key)
	if j.count() != 2 || rev2 <= rev1 || r.verdict(t, key).Trunk[repoA] != trunkA2 {
		t.Fatalf("judged %d times, revisions %d then %d, trunk %v", j.count(), rev1, rev2, r.verdict(t, key).Trunk)
	}
	if r.verdicts.creates[key] != 1 {
		t.Errorf("the key was created %d times, want the same key rewritten", r.verdicts.creates[key])
	}
	if _, orev2, _, _ := r.owed.Get(ctx, owedKey); orev2 <= orev1 {
		// the line is the same when nothing differs; the status is rewritten when the words change, here the trunk is in them
		t.Logf("the status keeps revision %d (its words did not change)", orev2)
	}
	// A new head is another key, and the older head's status is not touched.
	older, _, _, _ := r.owed.Get(ctx, owedKey)
	cand2 := candidateOf(t, "feat/x", pullOf(repoA, 7, head2))
	r.ch.Candidate(ctx, "feat%2Fx", cand2)
	r.ch.Wait()
	if again, _, _, _ := r.owed.Get(ctx, owedKey); string(again) != string(older) {
		t.Errorf("the older head's status was rewritten")
	}
	if n := len(r.owed.vals); n != 2 {
		t.Errorf("%d statuses, want one for each head", n)
	}
}

func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T) { // 1300/AC5 (unit part)
	ctx := context.Background()
	var seen []string
	var r *rig
	j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) {
		key, _ := delivery.CheckerOwedKey(repoA, 7, head1)
		raw, _, _, _ := r.owed.Get(ctx, key)
		o, _ := delivery.DecodeCheckerOwed(raw)
		seen = append(seen, o.State)
		return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "m/a.TestX"}}}, nil
	}}
	r = newRig(t, j)
	r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
	r.ch.Wait()
	key, _ := delivery.CheckerOwedKey(repoA, 7, head1)
	raw, _, _, _ := r.owed.Get(ctx, key)
	o, err := delivery.DecodeCheckerOwed(raw)
	if err != nil {
		t.Fatal(err)
	}
	if len(seen) != 1 || seen[0] != "pending" || o.State != "failure" || o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Target, "https://view.invalid/checks/") {
		t.Errorf("while running %v, at the end %+v", seen, o)
	}
}

// No check of heads no candidate holds is left running (issue 1890): at start the sweep ends a key left running; while the
// checker runs, a check whose group moved to other heads is ended at once, unjudged, and writes no status; heads held again
// are checked whole; and a group's check starts only once its heads settled, so a partial set of members is never checked.
func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) {
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	live := keyOf(t, "feat/x", cand)
	dead, _ := delivery.CheckerVerdictKey("feat/gone", "0123456789ab")
	for _, k := range []string{live, dead} {
		group, heads, _ := delivery.ParseCheckerVerdictKey(k)
		raw, _ := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, State: delivery.VerdictRunning})
		_, _ = r.verdicts.Create(ctx, k, raw)
	}
	r.ch.init()
	var c delivery.Candidate
	_ = json.Unmarshal(cand, &c)
	r.ch.known["feat%2Fx"] = c
	if err := r.ch.Sweep(ctx); err != nil {
		t.Fatal(err)
	}
	if v := r.verdict(t, dead); !IsEnded(v) || !strings.Contains(v.Failed.Words, "closed at start") {
		t.Errorf("the dead key = %+v", v)
	}
	if v := r.verdict(t, live); v.State != delivery.VerdictRunning {
		t.Errorf("the live key = %+v, want it left for the check to take over", v)
	}

	// A member joins while the check of the lone member's heads runs: that check is ended, its judge's run cancelled.
	started, ended := make(chan struct{}), make(chan error, 1)
	j = &judgeFunc{f: func(run context.Context, _ *Case, call int) (Outcome, error) {
		if call == 1 {
			close(started)
			<-run.Done()
			ended <- context.Cause(run)
			return Outcome{}, run.Err()
		}
		return Outcome{Verdict: passing()}, nil
	}}
	r = newRig(t, j)
	var logged []string
	var logMu sync.Mutex
	r.ch.Logf = func(f string, a ...any) { logMu.Lock(); logged = append(logged, fmt.Sprintf(f, a...)); logMu.Unlock() }
	alone, both := cand, candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 3, head2))
	r.ch.Candidate(ctx, "feat%2Fx", alone)
	<-started
	r.ch.Candidate(ctx, "feat%2Fx", both)
	var bothC delivery.Candidate
	_ = json.Unmarshal(both, &bothC)
	select {
	case err := <-ended:
		if err == nil || !strings.Contains(err.Error(), "superseded by "+GateHeads(bothC)) {
			t.Errorf("the old check's run ended with %v", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("the check of the heads the candidate left still runs")
	}
	r.ch.Wait()
	if v := r.verdict(t, live); !IsEnded(v) || v.Failed.Words != EndedWords+"superseded by "+GateHeads(bothC) {
		t.Errorf("the superseded verdict = %+v", v.Failed)
	}
	if v := r.verdict(t, keyOf(t, "feat/x", both)); v.State != delivery.VerdictPass {
		t.Errorf("the group's verdict = %s", v.State)
	}
	okey, _ := delivery.CheckerOwedKey(repoA, 7, head1)
	if raw, _, _, _ := r.owed.Get(ctx, okey); !strings.Contains(string(raw), `"success"`) {
		t.Errorf("the head's status = %s, want the group's success, never the ended check's", raw)
	}
	logMu.Lock()
	said := 0
	for _, l := range logged {
		if strings.Contains(l, "ended: superseded") {
			said++
		}
	}
	logMu.Unlock()
	if said != 1 {
		t.Errorf("the end was said %d times: %v", said, logged)
	}
	// The member leaves again: the ended heads are held again and checked whole.
	r.ch.Candidate(ctx, "feat%2Fx", alone)
	r.ch.Wait()
	if v := r.verdict(t, live); v.State != delivery.VerdictPass || j.count() != 3 {
		t.Errorf("the heads held again = %s after %d judgings, want a pass of a third", v.State, j.count())
	}

	// Members announced one after another: only the settled, whole set is checked.
	j = passJudge()
	r = newRig(t, j)
	r.ch.Settle = time.Hour
	sdk := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2))
	whole := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2), pullOf(repoA, 8, head1))
	r.ch.Candidate(ctx, "feat%2Fy", sdk)
	r.ch.Candidate(ctx, "feat%2Fy", whole)
	r.ch.Reconsider(ctx) // a trunk move or a turn while the heads settle starts nothing
	if j.count() != 0 {
		t.Fatalf("judged %d times before the heads settled", j.count())
	}
	r.ch.settleNow()
	r.ch.Wait()
	if _, _, found, _ := r.verdicts.Get(ctx, keyOf(t, "feat/y", sdk)); found || j.count() != 1 {
		t.Errorf("the partial set has a verdict (%v) or the judge ran %d times, want the whole set alone", found, j.count())
	}
	if v := r.verdict(t, keyOf(t, "feat/y", whole)); v.State != delivery.VerdictPass {
		t.Errorf("the whole set = %s", v.State)
	}
}

func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683 (the checker's half of AC5)
	ctx, j := context.Background(), passJudge()
	r := newRig(t, j)
	var told []string
	r.ch.Announce = func(_ context.Context, event, key string) { told = append(told, event+" "+key) }
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	r.ch.Candidate(ctx, "feat%2Fx", cand)
	r.ch.Wait()
	key, _ := delivery.CheckerOwedKey(repoA, 7, head1)
	want := []string{EventOwedWritten + " " + key, EventOwedWritten + " " + key} // pending, then success
	if len(told) != 2 || told[0] != want[0] || told[1] != want[1] {
		t.Fatalf("told %v, want the status said when pending and when success", told)
	}
	r.ch.Candidate(ctx, "feat%2Fx", cand) // nothing changed: nothing written, nothing said
	r.ch.Wait()
	if len(told) != 2 {
		t.Errorf("told %v after a check that changed nothing", told)
	}
}