mesh-delivery/modules/mesh-delivery-indexer/internal/index/pending_test.go

package index

import (
	"context"
	"encoding/json"
	"errors"
	"reflect"
	"slices"
	"strings"
	"testing"

	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
	"git.novox.be/novox/mesh-sdk/go/buscompose"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/process"
)

// takeInBed is an indexer holding the catalogue's avahi and the indexer's own module, both with a fingerprint, its
// pending state and the take-in events it says; base is the catalogue merge it was built from.
type takeInBed struct {
	ix      *Indexer
	mods    *memStore
	pending *memStore
	said    []takein.TakeIn
	bridge  map[string][]byte // what the forge bridge answers (bridge_merged_message), by repository@merge
	base    forgemsg.Forge
	logs    *strings.Builder
}

func newTakeInBed(t *testing.T) *takeInBed {
	t.Helper()
	ix, mods, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")},
		mod{Module, renamed(t, "avahi", Module)})
	if err := ix.Apply(context.Background(), f); err != nil {
		t.Fatal(err)
	}
	reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
	reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
	b := &takeInBed{ix: ix, mods: mods, pending: newMem(), bridge: map[string][]byte{}, base: f, logs: &strings.Builder{}}
	ix.Verdicts = newMem()
	b.pending.maxValue = buscompose.StateMaxValueBytes
	ix.Pending = b.pending
	ix.Message = func(_ context.Context, repository, merge string) ([]byte, error) {
		if data, ok := b.bridge[repository+"@"+merge]; ok {
			return data, nil
		}
		return nil, errors.New("the gate holds no merge of " + repository + " with that commit")
	}
	ix.SayTakeIn = func(_ context.Context, t takein.TakeIn) error { b.said = append(b.said, t); return nil }
	logf := ix.Logf
	ix.Logf = func(format string, args ...any) {
		b.logs.WriteString(format + "\n")
		if logf != nil {
			logf(format, args...)
		}
	}
	own := b.fp(t, Module)
	if own == "" || b.fp(t, "avahi") == "" {
		t.Fatalf("the bed's modules have no fingerprint")
	}
	ix.OwnBuild = func(context.Context) (string, error) { return own, nil }
	ix.OwnBuildState = newMem()
	ix.FixOwnBuild(context.Background()) // fixed once, at the start
	return b
}

func (b *takeInBed) fp(t *testing.T, module string) string {
	t.Helper()
	m, _ := moduleOf(t, b.mods, module)
	return m.Fp2
}

func (b *takeInBed) commit(t *testing.T, module string) string {
	t.Helper()
	m, _ := moduleOf(t, b.mods, module)
	return m.Commit
}

// merge is a catalogue merge at commit c whose trees of avahi and the indexer are given ("" keeps the base's).
func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge {
	f := b.base
	f.Number, f.Merge, f.Head = number, c, c
	f.Trees = map[string]string{}
	for dir, tree := range b.base.Trees {
		f.Trees[dir] = tree
	}
	if avahi != "" {
		f.Trees["modules/avahi"] = avahi
	}
	if indexer != "" {
		f.Trees["modules/"+Module] = indexer
	}
	return f
}

func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) {
	t.Helper()
	b.publish(t, f)
	if _, _, err := b.ix.Arrive(context.Background(), seq, f); err != nil {
		t.Fatalf("arrive %d: %v", seq, err)
	}
}

// publish lets the bed's forge bridge answer the merged message f, as bridge_merged_message does.
func (b *takeInBed) publish(t *testing.T, f forgemsg.Forge) {
	t.Helper()
	data, err := forgemsg.EncodeForge(f)
	if err != nil {
		t.Fatal(err)
	}
	b.bridge[f.Repository+"@"+f.Merge] = data
}

func (b *takeInBed) keys() []string {
	k, _ := b.pending.Keys(context.Background())
	slices.Sort(k)
	return k
}

func (b *takeInBed) events() []string {
	var out []string
	for _, s := range b.said {
		out = append(out, s.Kind+" "+s.Key)
	}
	return out
}

func sha(c byte) string { return strings.Repeat(string(c), 40) }

func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
	ctx := context.Background()
	t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) {
		b := newTakeInBed(t)
		b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer
		b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
		b.ix.FixOwnBuild(ctx)
		b.said = nil
		_, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
		if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') {
			t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi"))
		}
		// The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the
		// build is unknown, from the own-build state.
		if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") ||
			b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" {
			t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said)
		} else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil {
			t.Errorf("the unknown-build event is refused by its own contract: %v", err)
		}
		answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{})
		if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() {
			t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err)
		}
		// Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once,
		// as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever).
		b.ix.Pending = &memStore{unbound: process.ErrNotYet}
		if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
			t.Fatal(err)
		}
		if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') {
			t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi"))
		}
	})
	t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) {
		b := newTakeInBed(t)
		own, avahi0 := b.fp(t, Module), b.fp(t, "avahi")
		// A merge with no change to the indexer and nothing pending is taken in alone, as before.
		b.arrive(t, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
		if avahi1 := b.fp(t, "avahi"); avahi1 == avahi0 || len(b.keys()) != 0 || len(b.said) != 0 {
			t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in", avahi1, avahi0, b.keys(), b.events())
		}
		avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals)
		// T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending.
		b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
		named := b.fp(t, Module)
		if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') {
			t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"))
		}
		if g := graphOf(t, b.ix, Module); g.FP != named {
			t.Errorf("graph of the indexer names %s, want %s", g.FP, named)
		}
		if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 {
			t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1)
		}
		if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named {
			t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named)
		}
		// T5: the merge delivered again writes no second key and says nothing again.
		b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
		if len(b.keys()) != 1 || len(b.said) != 1 {
			t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events())
		}
		// A later merge joins: the newest governs and still names the new indexer.
		b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
		if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 {
			t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events())
		}
		// A revert brings the indexer back to the running build: everything is taken in, oldest first.
		b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+Module]))
		if len(b.keys()) != 0 || b.fp(t, Module) != own || b.commit(t, "avahi") != sha('e') {
			t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, Module), b.commit(t, "avahi"))
		}
		if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) {
			t.Errorf("said %v, want %v", b.events(), want)
		}
	})
	t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) {
		// A successor's manifest invokes a verb of the node-engine's seat that the seat list lacks and the revision's
		// definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending;
		// accepted with them, and the revision waits for the build it names.
		const verb, contracts = "revision-verb", "modules/seat-contracts"
		for _, row := range []struct {
			name    string
			defined bool
			refused bool
		}{{"definitions the revision does not carry", false, true}, {"definitions the revision carries", true, false}} {
			b := newTakeInBed(t)
			f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
			f.Manifests = map[string]string{}
			for dir, raw := range b.base.Manifests {
				f.Manifests[dir] = raw
			}
			dir := "modules/" + Module
			f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+Module+`"`,
				`"module": "`+Module+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1)
			if row.defined {
				f.Trees[contracts] = "tree-seat-contracts"
				f.Locks = map[string]map[string]string{}
				for d, l := range b.base.Locks {
					f.Locks[d] = l
				}
				f.Locks[contracts] = map[string]string{}
				f.Manifests[contracts] = `{"module":"seat-contracts","version":"1","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`
				b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) {
					if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) {
						t.Errorf("listed %v of %s at %s", dirs, repository, commit)
					}
					return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil
				}
				b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
					switch p {
					case contracts + "/seats/node-engine/seat.json":
						return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"` + verb + `"}]}`), nil
					}
					return nil, nil
				}
			}
			b.arrive(t, 2, f)
			waits := PendingWaits(readPending(t, b))
			refused := ""
			if len(b.said) > 0 {
				refused = b.said[0].Refused
			}
			if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) {
				t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused)
			}
			if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) {
				t.Errorf("%s: the seats are not put back after the dry run", row.name)
			}
		}
	})
	t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) {
		b := newTakeInBed(t)
		b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
		g := b.merge(30, sha('f'), "", "")
		g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x"
		g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
		b.arrive(t, 5, g)
		named := b.fp(t, Module)
		// The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
		// meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again.
		mk := func(ask func(context.Context) (string, error)) *Indexer {
			n := &Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
				Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: b.ix.Message, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum}
			n.toolchains = b.ix.toolchains
			n.OwnBuild = ask
			n.FixOwnBuild(ctx)
			return n
		}
		// A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the
		// wait was said before; and indexer_pending names the known build and since when.
		oldBuild, _ := b.ix.OwnBuild(ctx)
		oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil })
		b.logs.Reset()
		if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") ||
			!strings.Contains(b.logs.String(), "at the start, this program runs build") {
			t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String())
		}
		if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() {
			t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err)
		}
		asked := 0
		next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") })
		if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") {
			t.Errorf("a hand ask while the own build is unknown: %+v", r)
		}
		if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
			t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys())
		}
		// The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start.
		next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil }
		if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
			t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys())
		}
		// The module started again fixes the build at its start and takes the revision in.
		next = mk(func(context.Context) (string, error) { return named, nil })
		if _, _, err := next.TakeInAtStart(ctx); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') {
			t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi"))
		}
	})
	t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) {
		b := newTakeInBed(t)
		b.ix.Taken = newMem()
		// The commits in the order they were made: one is in another when it comes first.
		order := "a756bc"
		b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
			return strings.IndexByte(order, commit[0]) <= strings.IndexByte(order, in[0]), nil
		}
		member := func(repo string, number int, c string) forgemsg.Forge {
			f := b.merge(number, c, "", "")
			f.Group = "feat/take-in"
			f.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}}
			if repo != catalogueRepo {
				f.Repository, f.Trees, f.Manifests, f.Locks = repo, map[string]string{}, nil, nil
			}
			f.Head = map[int]string{40: sha('1'), 41: sha('2')}[number]
			return f
		}
		b.arrive(t, 1, member(catalogueRepo, 40, sha('7')))
		if b.commit(t, "avahi") == sha('7') || !slices.Equal(b.events(), []string{"pending g.feat/take-in"}) {
			t.Fatalf("first member: avahi at %s, said %v: want held, the group said pending", b.commit(t, "avahi"), b.events())
		}
		// A merge in another repository is taken in alone; one in a member's repository waits behind the group.
		alone := b.merge(50, sha('5'), "", "")
		alone.Repository, alone.Trees, alone.Manifests, alone.Locks = "novox/hq", map[string]string{}, nil, nil
		b.arrive(t, 2, alone)
		b.arrive(t, 3, b.merge(51, sha('6'), "tree-avahi-9", ""))
		if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(1), takein.PendingKey(3)}) {
			t.Fatalf("pending %v: want the member and the merge behind it", b.keys())
		}
		end := forgemsg.ForgeGroupEnd{Group: "feat/take-in", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('7')}, {Repository: "novox/mesh-sdk", Number: 41, Merge: sha('b')}}}
		if _, _, err := b.ix.EndGroup(ctx, end); err == nil {
			t.Fatal("an end before its last member is taken in")
		}
		b.arrive(t, 4, member("novox/mesh-sdk", 41, sha('b')))
		if _, _, err := b.ix.EndGroup(ctx, end); err != nil {
			t.Fatal(err)
		}
		if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('6') {
			t.Fatalf("end: pending %v, avahi at %s: want the group and the merge behind it taken in", b.keys(), b.commit(t, "avahi"))
		}
		if want := []string{"pending g.feat/take-in", "pending g.feat/take-in", "taken g.feat/take-in"}; !slices.Equal(b.events(), want) {
			t.Errorf("said %v, want %v", b.events(), want)
		}
		// R1: the end published again after the take-in changes nothing and is not delivered again.
		if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") {
			t.Errorf("an end again: %q %v", words, err)
		}
		// T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending.
		b.said = nil
		g := member(catalogueRepo, 40, sha('c'))
		g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees
		g.Members = g.Members[:1]
		b.arrive(t, 9, g)
		if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil {
			t.Fatal(err)
		}
		if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
			t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want)
		}
	})
	t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) {
		b := newTakeInBed(t)
		b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
		named := b.fp(t, Module)
		b.ix.ownMu.Lock()
		b.ix.ownFP = named
		b.ix.ownMu.Unlock()
		// The other indexer claims the merge between this one's read and its claim.
		b.pending.beforeWrite = func() {
			v := b.pending.vals[takein.PendingKey(2)]
			var p Pending
			_ = json.Unmarshal(v, &p)
			p.Taking = "the other indexer"
			data, _ := json.Marshal(p)
			b.pending.put(takein.PendingKey(2), data)
		}
		if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
			t.Fatal(err)
		}
		if b.commit(t, "avahi") == sha('c') || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is lost") {
			t.Errorf("avahi at %s, pending %v: want nothing taken in by the loser, and it said", b.commit(t, "avahi"), b.keys())
		}
	})
	t.Run("T5 a take-in whose store fails once leaves no claim behind, and is taken in after", func(t *testing.T) {
		claims := func(b *takeInBed) []string {
			var out []string
			for _, k := range b.keys() {
				var p Pending
				_ = json.Unmarshal(b.pending.vals[k], &p)
				if p.Taking != "" {
					out = append(out, k)
				}
			}
			return out
		}
		once := func(match func(key string) bool) func(string) error {
			done := false
			return func(key string) error {
				if !done && match(key) {
					done = true
					return errors.New("the store failed once")
				}
				return nil
			}
		}
		for _, c := range []struct {
			name string
			fail func(b *takeInBed)
		}{
			// The second merge's claim fails: the first's, already made, is released.
			{"a claim lost", func(b *takeInBed) {
				b.pending.failWrite = once(func(k string) bool { return k == takein.PendingKey(3) })
			}},
			// The take-in itself fails on a module's write after both claims: both are released.
			{"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }},
		} {
			b := newTakeInBed(t)
			b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
			b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
			named := b.fp(t, Module)
			b.ix.ownMu.Lock()
			b.ix.ownFP = named
			b.ix.ownMu.Unlock()
			c.fail(b)
			_, _, _ = b.ix.TakeInAtStart(ctx)
			if got := claims(b); len(got) != 0 || len(b.keys()) != 2 {
				t.Fatalf("%s: pending %v, claimed %v: want both pending and no claim left", c.name, b.keys(), got)
			}
			// The next take-in, not at a start, is not refused by a claim left behind: it takes both in.
			b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
			if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('e') {
				t.Errorf("%s: after: pending %v, avahi at %s: want all taken in", c.name, b.keys(), b.commit(t, "avahi"))
			}
		}
	})
	t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
		b := newTakeInBed(t)
		b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
		entries, err := ReadPending(ctx, b.pending, nil)
		if err != nil {
			t.Fatal(err)
		}
		// The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it.
		x := b.ix.dry().ix
		for _, node := range []string{"novox", "ace"} {
			key, _ := ToolchainKey(node, "go")
			x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.27.2", "1"), false)
		}
		ready, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready)
		next, _, err := x.computeOver(ctx, ready)
		if err != nil || next == "" || next == b.said[0].WaitsFor {
			t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor)
		}
		b.ix.ownMu.Lock()
		b.ix.ownFP = next
		b.ix.ownMu.Unlock()
		reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.27.2")
		if len(b.keys()) != 1 {
			t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys())
		}
		reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.27.2")
		if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
			t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi"))
		}
	})
	t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) {
		b := newTakeInBed(t)
		b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
		b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
		var answer takein.PendingAnswer
		for _, tool := range PendingTools(b.pending, nil, nil, nil) {
			if tool.Name == PendingTool {
				out, err := tool.Run(map[string]any{})
				if err != nil {
					t.Fatal(err)
				}
				answer = out.(takein.PendingAnswer)
			}
		}
		if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{"avahi", Module}) ||
			answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 {
			t.Fatalf("indexer_pending = %+v", answer)
		}
		b.said = nil
		if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3"); !r.Stored {
			t.Fatalf("drop: %+v", r)
		}
		if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) {
			t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said)
		}
		b.said = nil
		if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored {
			t.Fatalf("hand take-in: %+v", r)
		}
		if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
			t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said)
		}
	})
	t.Run("1585 a merge over the value cap is held by reference, a new indexer with an empty memory takes it in through the forge bridge, and a covered pull request leaves its candidate", func(t *testing.T) {
		big := func(f forgemsg.Forge) forgemsg.Forge {
			f.Words = strings.Repeat("x", 300*1024)
			f.Branch = "feat/g"
			return f
		}
		b := newTakeInBed(t)
		avahi0 := b.fp(t, "avahi")
		// The forge bridge is not asked at all while the merges are in memory: a 300 KiB merge, over the bucket's cap whole,
		// is held by reference and a lone one taken in.
		bridgeDown := func(context.Context, string, string) ([]byte, error) {
			return nil, errors.New("the bridge does not answer")
		}
		answers := b.ix.Message
		b.ix.Message = bridgeDown
		b.arrive(t, 1, big(b.merge(20, sha('b'), "tree-avahi-2", "")))
		if b.fp(t, "avahi") == avahi0 || b.commit(t, "avahi") != sha('b') || len(b.keys()) != 0 || len(b.ix.held) != 0 {
			t.Fatalf("a big merge alone: avahi at %s, pending %v, in memory %d: want it taken in and forgotten", b.commit(t, "avahi"), b.keys(), len(b.ix.held))
		}
		// Two pull requests of one group are open; both then merge (the indexer changed: held for the new build).
		for n, head := range map[int]string{21: sha('1'), 22: sha('2')} {
			if err := b.ix.Apply(ctx, pull(catalogueRepo, n, "feat/g", head, "modules/avahi")); err != nil {
				t.Fatal(err)
			}
		}
		key, _ := CandidatesKey("feat/g")
		if _, _, found, _ := b.ix.Store.Get(ctx, key); !found {
			t.Fatal("the group has no candidate")
		}
		b.arrive(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
		b.arrive(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
		raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2))
		if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') {
			t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi"))
		}
		named := b.fp(t, Module)
		// The new indexer starts with nothing in memory and the bridge down: the newest merge is said lost with the bridge's
		// words (event, state), the older one is covered and not lost, and nothing is taken in.
		next := &Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
			Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: bridgeDown, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf,
			GoSum: b.ix.GoSum, OwnBuild: func(context.Context) (string, error) { return named, nil }}
		next.toolchains = b.ix.toolchains
		next.FixOwnBuild(ctx)
		b.said = nil
		if _, _, err := next.TakeInAtStart(ctx); err != nil {
			t.Fatal(err)
		}
		waits := PendingWaits(mustReadPending(t, b))
		if len(b.said) != 1 || !slices.Equal(b.said[0].Lost, []string{takein.PendingKey(3)}) || len(waits) != 1 || !slices.Equal(waits[0].Lost, []string{takein.PendingKey(3)}) || b.commit(t, "avahi") != sha('b') {
			t.Fatalf("lost: said %+v, waits %+v, avahi at %s: want m.3 lost on the event and in indexer_pending, nothing taken in", b.said, waits, b.commit(t, "avahi"))
		}
		if v, _, _, _ := b.pending.Get(ctx, takein.PendingKey(3)); !strings.Contains(string(v), "the bridge does not answer") {
			t.Errorf("m.3 = %s: want the bridge's words", v)
		}
		if err := next.PendingAgain(ctx); err != nil || len(b.said) != 1 {
			t.Errorf("lost again: err %v, said %v: want it said once", err, b.events())
		}
		if r := next.TakeInPendingByHand(ctx, takein.PendingKey(2), "lead@shanks", "try"); r.Stored || b.commit(t, "avahi") != sha('b') {
			t.Errorf("hand take-in of a lost merge: %+v, avahi at %s: want nothing taken in", r, b.commit(t, "avahi"))
		}
		// The bridge answers: the newest merge is read through it, the older one is covered, and the revision is taken in.
		b.publish(t, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) // the answer is the message as published, small here
		next.Message = answers
		b.bridge[catalogueRepo+"@"+sha('d')], _ = forgemsg.EncodeForge(big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
		b.logs.Reset()
		if err := next.PendingAgain(ctx); err != nil {
			t.Fatal(err)
		}
		if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || !strings.Contains(b.logs.String(), "is covered by") {
			t.Fatalf("taken in: pending %v, avahi at %s, journal %q: want both merges taken in, the older covered", b.keys(), b.commit(t, "avahi"), b.logs.String())
		}
		if _, _, found, _ := b.ix.Store.Get(ctx, key); found {
			t.Errorf("the group's candidate stays after both its pull requests merged (the covered one is not taken out)")
		}
		// A hand take-in of a merge by its commit asks the bridge and runs the message through Arrive.
		b.ix.Message = answers
		b.publish(t, big(b.merge(30, sha('e'), "tree-avahi-6", b.base.Trees["modules/"+Module])))
		if r := b.ix.TakeInMerge(ctx, catalogueRepo, sha('e'), 9, "lead@shanks", "held by the cap"); !r.Stored || b.commit(t, "avahi") != sha('e') {
			t.Fatalf("merge take-in: %+v, avahi at %s: want it taken in", r, b.commit(t, "avahi"))
		}
		if r := b.ix.TakeInMerge(ctx, catalogueRepo, sha('9'), 10, "lead@shanks", "no such"); r.Stored || !strings.Contains(r.Answer, "gate holds no merge") {
			t.Errorf("merge take-in of a merge the bridge does not know: %+v", r)
		}
		// An old value that holds the message whole reads as before.
		old := b.merge(31, sha('f'), "tree-avahi-7", b.base.Trees["modules/"+Module])
		rawOld, _ := forgemsg.EncodeForge(old)
		data, _ := encodePending(Pending{Forge: rawOld, Since: b.ix.now(), LastWriter: b.fp(t, Module)})
		if err := b.pending.Create(ctx, takein.PendingKey(11), data); err != nil {
			t.Fatal(err)
		}
		if err := b.ix.PendingAgain(ctx); err != nil || b.commit(t, "avahi") != sha('f') {
			t.Errorf("an old value with the message whole: err %v, avahi at %s: want it taken in", err, b.commit(t, "avahi"))
		}
	})
	t.Run("the dry run names every field of the indexer", func(t *testing.T) {
		// dry (pending.go) builds an indexer field by field: a field added to Indexer is named there, or this fails.
		named := map[string]bool{}
		for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections",
			"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
			"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn",
			"Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Holds", "Counted", "AskEngines", "HoldBounds", "OpenTries", "TryRecovery", "OwnBuildState"} {
			named[f] = true
		}
		ty := reflect.TypeOf(Indexer{})
		for i := range ty.NumField() {
			if f := ty.Field(i); f.IsExported() && !named[f.Name] {
				t.Errorf("Indexer.%s is not named by dry", f.Name)
			}
		}
	})
}

func mustReadPending(t *testing.T, b *takeInBed) []PendingEntry {
	t.Helper()
	entries, err := ReadPending(context.Background(), b.pending, nil)
	if err != nil {
		t.Fatal(err)
	}
	return entries
}

// judgeSeatsHave says whether the catalogue's seat list holds the verb on the seat now.
func judgeSeatsHave(seat, verb string) bool {
	s, _ := catalogue.SeatNamed(seat)
	return slices.ContainsFunc(s.Serves, func(v catalogue.Verb) bool { return v.Name == verb })
}

// readPending is the bed's pending state as indexer_pending reads it.
func readPending(t *testing.T, b *takeInBed) []PendingEntry {
	t.Helper()
	entries, err := ReadPending(context.Background(), b.pending, nil)
	if err != nil {
		t.Fatal(err)
	}
	return entries
}