mesh-delivery/modules/mesh-delivery-checker/internal/check/rank_test.go

package check

import (
	"context"
	"fmt"
	"runtime"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// The checker's ranked start of checks (ADR 0536 §4, issue 1897): a freed place goes to a head never judged (or asked again
// after a failure of the nodes) before a head asked again only because its trunk moved; among equals the oldest asked first.
// A running check is never stopped.

// rankRig is a checker of one check at a time whose judge records the order checks start in and holds the first check
// after hold is set until release is closed.
type rankRig struct {
	*rig
	mu      sync.Mutex
	started []*Case
	hold    bool
	held    chan struct{} // closed when the held check started
	release chan struct{}
	errs    []error // ctx errors the held check saw when released
}

func newRankRig(t *testing.T) *rankRig {
	rr := &rankRig{held: make(chan struct{}), release: make(chan struct{})}
	j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) {
		rr.mu.Lock()
		rr.started = append(rr.started, c)
		hold := rr.hold
		rr.hold = false
		rr.mu.Unlock()
		if hold {
			close(rr.held)
			<-rr.release
			rr.mu.Lock()
			rr.errs = append(rr.errs, ctx.Err())
			rr.mu.Unlock()
		}
		return Outcome{Verdict: passing()}, nil
	}}
	rr.rig = newRig(t, j)
	rr.ch.Limit = 1
	now := time.Date(2026, 10, 23, 14, 0, 0, 0, time.UTC)
	var clock sync.Mutex
	rr.ch.Now = func() time.Time {
		clock.Lock()
		defer clock.Unlock()
		now = now.Add(time.Second)
		return now
	}
	return rr
}

func group(n int) string { return fmt.Sprintf("feat/g%d", n) }

// judged puts n candidates in and judges them on the current trunk.
func (rr *rankRig) judged(t *testing.T, n int) {
	for i := 1; i <= n; i++ {
		rr.ch.Candidate(context.Background(), group(i), candidateOf(t, group(i), pullOf(repoA, i, head1)))
	}
	rr.ch.Wait()
	rr.mu.Lock()
	rr.started = nil
	rr.mu.Unlock()
}

// waitFor returns when n checks wait for a place.
func (rr *rankRig) waitFor(t *testing.T, n int) {
	t.Helper()
	deadline := time.Now().Add(5 * time.Second)
	for {
		if w := len(rr.ch.Waiting()); w == n {
			return
		} else if time.Now().After(deadline) {
			t.Fatalf("%d checks wait, want %d", w, n)
		}
		runtime.Gosched()
	}
}

// moveTrunkHoldingOne moves the trunk, so every judged candidate is asked again; the first re-judge to start is held.
func (rr *rankRig) moveTrunkHoldingOne(t *testing.T, judged int) {
	rr.mu.Lock()
	rr.hold = true
	rr.mu.Unlock()
	rr.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
	rr.ch.TrunkMoved(context.Background())
	<-rr.held
	rr.waitFor(t, judged-1)
}

func TestANewHeadStartsBeforeTheReJudgesOfATrunkMove(t *testing.T) { // 1897/AC1
	rr := newRankRig(t)
	rr.judged(t, 4)
	rr.moveTrunkHoldingOne(t, 4) // one runs, three re-judges wait
	rr.ch.Candidate(context.Background(), group(9), candidateOf(t, group(9), pullOf(repoA, 9, head2)))
	rr.waitFor(t, 4)
	close(rr.release)
	rr.ch.Wait()
	if len(rr.started) != 5 {
		t.Fatalf("%d checks started, want 5", len(rr.started))
	}
	if g := rr.started[1].Group; g != group(9) {
		t.Errorf("after the running check, %s started, want the new head's %s; order %v", g, group(9), groups(rr.started))
	}
}

func TestTheReJudgeQueuedFirstStartsFirst(t *testing.T) { // 1897/AC2
	rr := newRankRig(t)
	rr.judged(t, 3)
	rr.moveTrunkHoldingOne(t, 3) // one runs, two re-judges wait
	close(rr.release)
	rr.ch.Wait()
	if len(rr.started) != 3 {
		t.Fatalf("%d checks started, want 3", len(rr.started))
	}
	if a, b := rr.started[1], rr.started[2]; !a.Queued.Before(b.Queued) {
		t.Errorf("%s (queued %s) started before %s (queued %s)", a.Group, a.Queued.Format(time.TimeOnly), b.Group, b.Queued.Format(time.TimeOnly))
	}
}

func TestANewHeadDoesNotStopARunningReJudge(t *testing.T) { // 1897/AC3
	rr := newRankRig(t)
	rr.judged(t, 1)
	rr.moveTrunkHoldingOne(t, 1) // the re-judge runs
	rr.ch.Candidate(context.Background(), group(9), candidateOf(t, group(9), pullOf(repoA, 9, head2)))
	rr.waitFor(t, 1)
	close(rr.release)
	rr.ch.Wait()
	if len(rr.errs) != 1 || rr.errs[0] != nil {
		t.Fatalf("the running re-judge saw %v, want it never stopped", rr.errs)
	}
	if len(rr.started) != 2 || rr.started[0].Group != group(1) || rr.started[1].Group != group(9) {
		t.Errorf("started %v, want the re-judge to end before the new head starts", groups(rr.started))
	}
	key := keyOf(t, group(1), candidateOf(t, group(1), pullOf(repoA, 1, head1)))
	if v := rr.verdict(t, key); v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 {
		t.Errorf("the re-judge's verdict = %+v, want its pass on the moved trunk", v)
	}
}

func TestWaitingChecksAreNamedByKind(t *testing.T) { // 1897/AC4, the checker's own view
	rr := newRankRig(t)
	rr.judged(t, 2)
	rr.moveTrunkHoldingOne(t, 2)
	rr.ch.Candidate(context.Background(), group(9), candidateOf(t, group(9), pullOf(repoA, 9, head2)))
	rr.waitFor(t, 2)
	kinds := map[string]string{}
	for _, w := range rr.ch.Waiting() {
		kinds[w.Group] = w.Kind
	}
	close(rr.release)
	rr.ch.Wait()
	running := rr.started[0].Group
	other := group(1)
	if running == other {
		other = group(2)
	}
	if kinds[group(9)] != KindNew || kinds[other] != KindTrunk || len(kinds) != 2 {
		t.Errorf("waiting = %v, want %s %s and %s %s", kinds, group(9), KindNew, other, KindTrunk)
	}
}

func groups(cs []*Case) []string {
	out := make([]string, len(cs))
	for i, c := range cs {
		out[i] = c.Group
	}
	return out
}

func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4
	ctx := context.Background()
	j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
		if c.Group == group(2) {
			return Outcome{Verdict: failure("checker", "no "+NodesWords+": every node is offline")}, nil
		}
		return Outcome{Verdict: passing()}, nil
	}}
	r := newRig(t, j)
	cands := fakeState{}
	put := func(n int, head string) {
		cand := candidateOf(t, group(n), pullOf(repoA, n, head))
		ckey, _ := delivery.CandidatesKey(group(n))
		cands[ckey] = cand
		r.ch.Candidate(ctx, group(n), cand)
	}
	put(1, head1) // judged on the old trunk: a re-judge waits
	put(2, head1) // failed for the nodes: waits for them
	r.ch.Wait()
	r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
	put(4, head1) // judged on the new trunk: stands
	r.ch.Wait()
	cand3 := candidateOf(t, group(3), pullOf(repoA, 3, head1)) // never judged
	ckey3, _ := delivery.CandidatesKey(group(3))
	cands[ckey3] = cand3
	verdicts := fakeState{}
	for k, v := range r.verdicts.vals {
		verdicts[k] = v
	}
	rd := Reader{CandidateState: cands.read(), VerdictState: verdicts.read(),
		Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA2, repoSDK: trunkS}, nil }}
	st, err := rd.Status()
	if err != nil {
		t.Fatal(err)
	}
	want := map[string][]string{KindNew: {group(3)}, KindNodes: {group(2)}, KindTrunk: {group(1)}}
	if fmt.Sprint(st.Waiting) != fmt.Sprint(want) {
		t.Errorf("waiting = %v, want %v", st.Waiting, want)
	}
}