mesh-delivery/modules/mesh-delivery-checker/internal/check/read_test.go

package check

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

type fakeState map[string][]byte

func (f fakeState) read() StateRead {
	return StateRead{
		Keys: func() ([]string, error) {
			var k []string
			for key := range f {
				k = append(k, key)
			}
			return k, nil
		},
		Get: func(key string) (json.RawMessage, error) {
			if v, ok := f[key]; ok {
				return json.RawMessage(v), nil
			}
			return nil, nil
		},
	}
}

// readerOf is a reader over a verdict the checker wrote through its own code and the candidate it judged.
func readerOf(t *testing.T) (Reader, string) {
	t.Helper()
	r := newRig(t, passJudge())
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(hqRepo, 3, head2))
	r.ch.Candidate(context.Background(), "feat%2Fx", cand)
	r.ch.Wait()
	ckey, _ := delivery.CandidatesKey("feat/x")
	verdicts, kept := fakeState{}, fakeState{}
	for k, v := range r.verdicts.vals {
		verdicts[k] = v
	}
	if _, err := (Kept{Store: r.verdicts}).Keep(context.Background(), "unit", digA, keptValue("m/a", 3)); err != nil {
		t.Fatal(err)
	}
	kept["unit."+digA] = r.verdicts.vals["unit."+digA]
	return Reader{VerdictState: verdicts.read(), KeptState: kept.read(), CandidateState: fakeState{ckey: cand}.read()}, keyOf(t, "feat/x", cand)
}

func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
	rd, key := readerOf(t)
	vs, _, err := rd.Verdicts("")
	if err != nil || len(vs) != 1 || vs[0].Key != key {
		t.Fatalf("verdicts = %+v, %v", vs, err)
	}
	if _, err := delivery.DecodeVerdict(vs[0].Verdict); err != nil {
		t.Errorf("the verdict is not as stored: %v", err)
	}
	if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
		t.Errorf("reports = %v", vs[0].Reports)
	}
	if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
		t.Errorf("another group's verdicts = %+v", only)
	}
}

// 1887/A10: an hq pull request is answered as any other, its report its verdict's (ADR 0527 §2), never a note that a script
// judges it.
func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) {
	rd, _ := readerOf(t)
	hq, err := rd.Pull(hqRepo, 3)
	if err != nil || hq.Note != "" || hq.Verdict == nil {
		t.Errorf("hq pull = %+v, %v", hq, err)
	}
	p, err := rd.Pull(repoA, 7)
	if err != nil || p.Note != "" || !strings.HasPrefix(p.Reports["novox/mesh-delivery#7"], "pass ") {
		t.Errorf("pull = %+v, %v", p, err)
	}
	if _, err := rd.Pull(repoA, 99); err == nil {
		t.Error("a pull request no candidate holds was answered")
	}
}

func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
	rd, _ := readerOf(t)
	all, _, err := rd.Kept("", "")
	if err != nil || len(all) != 1 {
		t.Fatalf("kept = %v, %v", all, err)
	}
	one, _, err := rd.Kept("unit", digA)
	if err != nil || len(one) != 1 {
		t.Fatalf("one kept = %v, %v", one, err)
	}
	for _, raw := range one {
		if k, err := delivery.DecodeKept(raw); err != nil || k.Package != "m/a" {
			t.Errorf("kept value = %+v, %v", k, err)
		}
	}
	if none, _, _ := rd.Kept("integration", ""); len(none) != 0 {
		t.Errorf("integration = %v", none)
	}
	if _, _, err := rd.Kept("unit", digB); err == nil {
		t.Error("a digest nothing kept was answered")
	}
}

// 1680 live acceptance: checker_previews shows the preview key of an open pull request, as stored, narrowed by repository and head.
func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
	pv := delivery.Preview{Repository: repoA, Head: head1, Trunk: head2, Merge: "m1", Modules: map[string]delivery.PreviewModule{}}
	data, err := delivery.EncodePreview(pv)
	if err != nil {
		t.Fatal(err)
	}
	key, _ := delivery.PreviewKey(repoA, head1, head2)
	rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
	got, bad, _, err := rd.Previews(repoA, head1[:8])
	if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
		t.Fatalf("previews = %+v, %v", got, err)
	}
	if len(bad) != 1 || bad[0] != "junk" {
		t.Errorf("unreadable = %v", bad)
	}
	if got, _, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
		t.Errorf("another repository's = %+v", got)
	}
	if _, _, _, err := (Reader{}).Previews("", ""); err == nil {
		t.Error("answered with no state to read")
	}
}

// checker_status reads the steps in flight by node and the candidates without a verdict; since R4g nothing holds the checker
// and it has no capacity of its own (ADR 0525 §4, §7; 1685 closed by the hold's deletion).
func TestTheStatusToolReadsRunningSteps(t *testing.T) {
	running, _ := delivery.EncodeStep(delivery.Step{ID: "abc1", Tool: "build_go_test", Node: "shanks", State: delivery.StepRunning, Started: time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)})
	passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed})
	cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	ckey, _ := delivery.CandidatesKey("feat/x")
	rd := Reader{
		StepStates:     map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()},
		CandidateState: fakeState{ckey: cand}.read(),
		VerdictState:   fakeState{}.read(),
		HaveState:      fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
		HearingState: fakeState{"dead-letters": []byte(`{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],` +
			`"why":"DEAD_LETTERS is not heard, so an ask a build seat gives up on is waited for until its bound: SEAT_NODE_BUILD_GO: permissions violation",` +
			`"at":"2026-10-26T13:00:00Z"}`)}.read(),
		Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
	}
	st, err := rd.Status()
	if err != nil {
		t.Fatal(err)
	}
	if len(st.Running) != 1 || st.Running[0].Node != "shanks" || st.RunningByNode["shanks"] != 1 || st.RunningByNode["ace"] != 0 {
		t.Errorf("running = %+v %v", st.Running, st.RunningByNode)
	}
	if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
		t.Errorf("unjudged = %v", st.Unjudged)
	}
	// Issue 734, review round 1: a checker that does not hear DEAD_LETTERS says so in its status, with why.
	if st.DeadLetters.Heard || !strings.Contains(st.DeadLetters.Why, "permissions violation") {
		t.Errorf("dead letters = %+v, want not heard, saying why", st.DeadLetters)
	}
	// What cannot be read is said, not guessed.
	st, err = Reader{}.Status()
	if err != nil || len(st.Notes) == 0 || st.DeadLetters.Heard {
		t.Errorf("empty status = %+v, %v", st, err)
	}
}

func haveOf(t *testing.T, phase delivery.Phase) []byte {
	t.Helper()
	raw, err := delivery.EncodeHave(delivery.Have{Phase: phase, Health: delivery.HealthOK})
	if err != nil {
		t.Fatal(err)
	}
	return raw
}

func stepOf(t *testing.T, id, node string, started time.Time) []byte {
	t.Helper()
	raw, err := delivery.EncodeStep(delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: delivery.StepRunning, Started: started})
	if err != nil {
		t.Fatal(err)
	}
	return raw
}

// 1890: a key a build module left running (it stopped, restarted or was unassigned before it wrote the step's end) holds
// no place in the checker, whose wait for it ended at the step bound; the status says it is left running and why, and
// counts only the steps that can still run.
func TestTheStatusCountsOnlyStepsThatCanStillRunAndSaysWhyTheOthersAreLeft(t *testing.T) { // 1890/AC2, AC3
	now := time.Date(2026, 10, 23, 7, 0, 0, 0, time.UTC)
	rd := Reader{
		StepStates: map[string]StateRead{
			"mesh-build-go": fakeState{
				"g14.fresh1":   stepOf(t, "fresh1", "g14", now.Add(-5*time.Minute)),
				"g14.old1":     stepOf(t, "old1", "g14", now.Add(-33*time.Hour)),
				"ace.gone1":    stepOf(t, "gone1", "ace", now.Add(-2*time.Minute)),
				"shanks.down1": stepOf(t, "down1", "shanks", now.Add(-time.Minute)),
			}.read(),
		},
		HaveState: fakeState{
			"g14.mesh-build-go":    haveOf(t, delivery.PhaseRunning),
			"shanks.mesh-build-go": haveOf(t, delivery.PhaseFailed),
		}.read(),
		StepBound: 30 * time.Minute,
		Now:       func() time.Time { return now },
	}
	st, err := rd.Status()
	if err != nil {
		t.Fatal(err)
	}
	if len(st.Running) != 1 || st.Running[0].ID != "fresh1" || len(st.RunningByNode) != 1 || st.RunningByNode["g14"] != 1 {
		t.Fatalf("running = %+v by node %v", st.Running, st.RunningByNode)
	}
	why := map[string]string{}
	for _, l := range st.LeftRunning {
		why[l.ID] = l.Why
	}
	if len(why) != 3 {
		t.Fatalf("left running = %+v", st.LeftRunning)
	}
	for id, want := range map[string]string{"old1": "past the step bound of 30m0s", "gone1": "mesh-build-go is not on ace", "down1": "mesh-build-go on shanks is failed"} {
		if !strings.Contains(why[id], want) {
			t.Errorf("why %s = %q, want it to say %q", id, why[id], want)
		}
	}
}

// 1887/A10: a group of hq's pull requests alone is judged by the checker since R4g, so without a verdict it is unjudged.
func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
	cand := candidateOf(t, "pb-x", pullOf(hqRepo, 3, head2))
	ckey, _ := delivery.CandidatesKey("pb-x")
	st, err := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{}.read()}.Status()
	if err != nil {
		t.Fatal(err)
	}
	if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
		t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
	}
	// ADR 0548 §3: a candidate whose check an approved merge's turn holds off is named with the turn.
	held := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
	hkey, _ := delivery.CandidatesKey("feat/x")
	st, err = Reader{CandidateState: fakeState{hkey: held}.read(), VerdictState: fakeState{}.read(),
		Trunks:    func() (map[string]string, error) { return map[string]string{repoA: trunkA}, nil },
		TurnState: fakeState{turnKey(t, repoA): turnOf(t, repoA, "fix/other", "0123456789ab")}.read()}.Status()
	if err != nil {
		t.Fatal(err)
	}
	if why := st.HeldByTurn["feat/x"]; !strings.Contains(why, repoA) || !strings.Contains(why, "fix/other") {
		t.Errorf("held_by_turn = %v, want feat/x held by the merge of fix/other on %s", st.HeldByTurn, repoA)
	}
}

// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
type slowBucket struct {
	vals                fakeState
	reads, lists        atomic.Int64
	inFlight, maxFlight atomic.Int64
}

func (b *slowBucket) get(key string) (json.RawMessage, error) {
	b.reads.Add(1)
	now := b.inFlight.Add(1)
	for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
	}
	time.Sleep(time.Millisecond)
	b.inFlight.Add(-1)
	return b.vals.read().Get(key)
}

func (b *slowBucket) read() StateRead {
	return StateRead{Keys: func() ([]string, error) { b.lists.Add(1); return b.vals.read().Keys() }, Get: b.get}
}

func (b *slowBucket) Keys() ([]string, error)                 { return b.read().Keys() }
func (b *slowBucket) Get(key string) (json.RawMessage, error) { return b.get(key) }
func (b *slowBucket) Create(string, []byte) error             { return nil }

// 1827: a read tool lists the keys of a bucket of thousands and reads only the values its answer needs, sixteen at a time, at
// most MaxReads, saying when it cut (on the live shape of 2026-10-26: kept 7990 keys, steps 10034, tries 1000s).
func TestReadToolsReadOnlyWhatTheyNeedInParallelAndSayWhenCut(t *testing.T) { // 1827
	kept := fakeState{}
	for i := 0; i < 1800; i++ {
		level := map[bool]string{true: "unit", false: "integration"}[i%9 != 0]
		k, _ := delivery.KeptKey(level, fmt.Sprintf("%064x", i))
		kept[k] = []byte(`{"v":1}`)
	}
	previews := fakeState{}
	for i := 0; i < 1500; i++ {
		repo := map[bool]string{true: repoA, false: repoSDK}[i < 300]
		k, _ := delivery.PreviewKey(repo, fmt.Sprintf("%08x%032x", i, 0), trunkA)
		previews[k] = []byte(`{"v":1}`)
	}
	for _, row := range []struct {
		name         string
		call         func(*slowBucket) (rows int, cut string, err error)
		rows         int
		reads, lists int64
		cut          bool
	}{
		{"kept of one key", func(b *slowBucket) (int, string, error) {
			m, c, err := Reader{KeptState: b.read()}.Kept("unit", fmt.Sprintf("%064x", 1))
			return len(m), c, err
		}, 1, 1, 0, false},
		{"kept of a level", func(b *slowBucket) (int, string, error) {
			m, c, err := Reader{KeptState: b.read()}.Kept("integration", "")
			return len(m), c, err
		}, 200, 200, 1, false},
		{"kept of everything", func(b *slowBucket) (int, string, error) {
			m, c, err := Reader{KeptState: b.read()}.Kept("", "")
			return len(m), c, err
		}, MaxReads, MaxReads, 1, true},
		{"previews of a repository", func(b *slowBucket) (int, string, error) {
			p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoSDK, "")
			return len(p), c, err
		}, 1200, 1000, 1, true},
		{"previews of a repository and a head", func(b *slowBucket) (int, string, error) {
			p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoA, fmt.Sprintf("%08x", 7))
			return len(p), c, err
		}, 1, 1, 1, false},
	} {
		b := &slowBucket{}
		b.vals = kept
		if strings.HasPrefix(row.name, "previews") {
			b.vals = previews
		}
		start := time.Now()
		rows, cut, err := row.call(b)
		if err != nil {
			t.Fatalf("%s: %v", row.name, err)
		}
		if rows != min(row.rows, MaxReads) || b.reads.Load() != row.reads || b.lists.Load() != row.lists || (cut != "") != row.cut {
			t.Errorf("%s: %d rows, %d reads, %d lists, cut %q; want %d, %d, %d, cut %v", row.name, rows, b.reads.Load(), b.lists.Load(), cut, min(row.rows, MaxReads), row.reads, row.lists, row.cut)
		}
		if row.reads > 20 && b.maxFlight.Load() < 2 {
			t.Errorf("%s: values were read one at a time", row.name)
		}
		t.Logf("%s: %d values in %v (one after the other: %v)", row.name, b.reads.Load(), time.Since(start), time.Duration(b.reads.Load())*time.Millisecond)
	}

	// The status reads the newest steps of a bucket of 1500 first, at most MaxReads, and finds the one that runs.
	now := time.Date(2026, 10, 26, 7, 0, 0, 0, time.UTC)
	steps := fakeState{}
	for i := 0; i < 1500; i++ {
		id := fmt.Sprintf("mvo5%04dabcdef", i)
		step := delivery.Step{ID: id, Tool: "t", Node: "g14", State: delivery.StepPassed}
		if i == 1499 {
			step.State, step.Started = delivery.StepRunning, now.Add(-time.Minute)
		}
		raw, _ := delivery.EncodeStep(step)
		steps["g14."+id] = raw
	}
	sb := &slowBucket{vals: steps}
	st, err := Reader{StepStates: map[string]StateRead{"mesh-build-go": sb.read()}, HaveState: fakeState{"g14.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
		Now: func() time.Time { return now }}.Status()
	if err != nil || sb.reads.Load() != MaxReads || len(st.Running) != 1 || sb.maxFlight.Load() < 2 {
		t.Errorf("status: %d reads (want %d), running %+v, in flight %d, %v", sb.reads.Load(), MaxReads, st.Running, sb.maxFlight.Load(), err)
	}

	// A new try reads the newest TryScan of 1500 ended tries, not all, and still finds the same head asked twice.
	tries := fakeState{}
	for i := 0; i < 1500; i++ {
		raw, _ := encodeTry(Try{ID: fmt.Sprintf("try-%d", 1700000000000000000+i), Repository: repoA, Head: head1, Trunk: trunkA, State: TryPass})
		tries[fmt.Sprintf("try-%d", 1700000000000000000+i)] = raw
	}
	tb := &slowBucket{vals: tries}
	if _, err := AskTry(tb, tryTrunks, TryAsk{Repository: repoA, Head: head2, Trunk: trunkA}, now); err != nil || tb.reads.Load() != TryScan {
		t.Errorf("AskTry read %d of 1500 tries (want %d), %v", tb.reads.Load(), TryScan, err)
	}
}