mesh-catalog/modules/nats/internal/keeper/keeper_test.go

package keeper

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/asks"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
)

// fakeBus holds objects by stream, as a bus would, and refuses what a test tells it to.
type fakeBus struct {
	mu      sync.Mutex
	held    map[string]Settings
	fill    map[string]uint64
	refuse  map[string]error
	created []string
	updated []string
	deleted []string
	snaps   [][]string
}

func newFakeBus() *fakeBus {
	return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}}
}

func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	s, ok := b.held[o.Stream()]
	return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil
}

func (b *fakeBus) Create(_ context.Context, o Object) error {
	b.mu.Lock()
	defer b.mu.Unlock()
	if err := b.refuse[o.Stream()]; err != nil {
		return err
	}
	b.held[o.Stream()] = o.Settings
	b.created = append(b.created, o.Stream())
	return nil
}

func (b *fakeBus) Update(_ context.Context, o Object) error {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.held[o.Stream()] = o.Settings
	b.updated = append(b.updated, o.Stream())
	return nil
}

func (b *fakeBus) Delete(_ context.Context, o Object) error {
	b.mu.Lock()
	defer b.mu.Unlock()
	delete(b.held, o.Stream())
	b.deleted = append(b.deleted, o.Stream())
	return nil
}

func (b *fakeBus) Snapshot(_ context.Context, streams []string) (string, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.snaps = append(b.snaps, streams)
	return "/snapshots/" + strings.Join(streams, "+") + ".tar", nil
}

// memStore is a key-value state in memory.
type memStore struct {
	mu   sync.Mutex
	kv   map[string][]byte
	revs map[string]uint64
	rev  uint64
}

func newMemStore() *memStore { return &memStore{kv: map[string][]byte{}, revs: map[string]uint64{}} }

func (s *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	v, ok := s.kv[state+"/"+key]
	return v, s.revs[state+"/"+key], ok, nil
}

func (s *memStore) Put(state, key string, value []byte) (uint64, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.rev++
	s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev
	return s.rev, nil
}

func (s *memStore) Delete(state, key string) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	delete(s.kv, state+"/"+key)
	return nil
}

func (s *memStore) Keys(state string) ([]string, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	var out []string
	for k := range s.kv {
		if st, key, _ := strings.Cut(k, "/"); st == state {
			out = append(out, key)
		}
	}
	sort.Strings(out)
	return out, nil
}

// said is what a keeper said: its events and its conditions, in order.
type said struct {
	mu     sync.Mutex
	events []string
	bodies []Event
	conds  []string
}

func (s *said) emit(event string, body Event) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.events = append(s.events, event+" "+body.Module+" "+body.Local)
	s.bodies = append(s.bodies, body)
	return nil
}

func (s *said) say(c asks.Condition) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.conds = append(s.conds, c.State+" "+c.Key)
	return nil
}

func (s *said) reset() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.events, s.bodies, s.conds = nil, nil, nil
}

func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) {
	bus, store, out := newFakeBus(), newMemStore(), &said{}
	at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)
	k := &Keeper{Bus: bus, Store: store, Emit: out.emit, Say: out.say,
		View: func() (FirstView, error) { return view, nil },
		Now:  func() time.Time { return at }}
	return k, bus, store, out
}

func entry(name, storage, template string, options map[string]string) catalogue.BusStateEntry {
	e := catalogue.BusStateEntry{Name: name, Storage: storage, Template: template}
	if len(options) > 0 {
		e.Options = map[string]json.RawMessage{}
		for k, v := range options {
			e.Options[k] = json.RawMessage(v)
		}
	}
	return e
}

// What one entry is on the bus (design 25 § 12.1, CAT-N3): each template's objects, named `<module>_<local>`, its
// binding, and the fixed settings no option changes; and the entries the keeper refuses itself, naming why.
func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) {
	ledger := entry("records", catalogue.StorageLedger, catalogue.TemplateLedger, map[string]string{"max-mib": "16"})
	ledger.Types = []catalogue.LedgerType{{Name: "decision", Indexes: []catalogue.LedgerIndex{{Name: "by-topic", On: []string{"topic"}}}}}
	bare := entry("raw", catalogue.StorageStream, "", nil)
	bare.Configuration = map[string]json.RawMessage{"when-full": json.RawMessage(`"drop-oldest"`), "max-mib": json.RawMessage("8"), "max-age-days": json.RawMessage("3")}
	for _, row := range []struct {
		name    string
		entry   catalogue.BusStateEntry
		objects string
		binding string
		refused string
	}{
		{name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}),
			objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}",
			binding: "bucket mesh-knowledge_drafts"},
		{name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil),
			objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}",
			binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"},
		{name: "work queue", entry: entry("jobs", catalogue.StorageStream, catalogue.TemplateWorkQueue, map[string]string{"max-mib": "4", "duplicates": "120"}),
			objects: "stream mesh-knowledge_jobs [mesh.bus.mesh-knowledge.jobs.>] {4 MiB, age 0, history 0, new, size 0, work-queue, duplicates 120}",
			binding: "stream mesh-knowledge_jobs on mesh.bus.mesh-knowledge.jobs"},
		{name: "log", entry: entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil),
			objects: "stream mesh-knowledge_log [mesh.bus.mesh-knowledge.log.>] {1024 MiB, age 0, history 0, new, size 262144}",
			binding: "stream mesh-knowledge_log on mesh.bus.mesh-knowledge.log"},
		{name: "objects", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "32"}),
			objects: "object-store mesh-knowledge_files {32 MiB, age 0, history 0, new, size 0}",
			binding: "objects mesh-knowledge_files"},
		{name: "ledger", entry: ledger,
			objects: "stream mesh-knowledge_records [mesh.ledger.mesh-knowledge.records.history.>] {16 MiB, age 0, history 0, new, size 262144}; " +
				"key-value mesh-knowledge_records_state {64 MiB, age 0, history 1, new, size 262144}; " +
				"key-value mesh-knowledge_records_index-by-topic {64 MiB, age 0, history 1, new, size 262144}; " +
				"stream mesh-knowledge_records_snapshot [mesh.ledger.mesh-knowledge.records.snapshot.>] {16 MiB, age 0, history 1, old, size 0}",
			binding: "history mesh-knowledge_records on mesh.ledger.mesh-knowledge.records.history, state mesh-knowledge_records_state, indexes map[by-topic:mesh-knowledge_records_index-by-topic], snapshot mesh.ledger.mesh-knowledge.records.snapshot"},
		{name: "bare stream", entry: bare,
			objects: "stream mesh-knowledge_raw [mesh.bus.mesh-knowledge.raw.>] {8 MiB, age 259200, history 0, old, size 0}",
			binding: "stream mesh-knowledge_raw on mesh.bus.mesh-knowledge.raw"},
		{name: "a local name holding the separator", entry: entry("my_drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil),
			refused: `the local name "my_drafts" is empty or holds "_"`},
		{name: "objects with no cap", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, nil),
			refused: "gives no max-mib"},
		{name: "a bare stream with no when-full", entry: entry("raw", catalogue.StorageStream, "", nil),
			refused: "must give when-full"},
		{name: "a template on the wrong storage type", entry: entry("x", catalogue.StorageKeyValue, catalogue.TemplateLog, nil),
			refused: "is none the bus module makes"},
	} {
		t.Run(row.name, func(t *testing.T) {
			p, err := PlanOf("mesh-knowledge", row.entry)
			if row.refused != "" {
				if err == nil || !strings.Contains(err.Error(), row.refused) {
					t.Fatalf("planned %+v (%v): refused, naming %q", p, err, row.refused)
				}
				return
			}
			if err != nil {
				t.Fatal(err)
			}
			var objs []string
			for _, o := range p.Objects {
				s := o.Settings
				w := fmt.Sprintf("%s %s", o.Kind, o.Name)
				if len(o.Subjects) > 0 {
					w += fmt.Sprintf(" %v", o.Subjects)
				}
				w += fmt.Sprintf(" {%d MiB, age %d, history %d, %s, size %d", s.MaxBytes/MiB, s.MaxAgeSeconds, s.History, s.Discard, s.MaxMsgSize)
				if s.WorkQueue {
					w += fmt.Sprintf(", work-queue, duplicates %d", s.DuplicatesSeconds)
				}
				objs = append(objs, w+"}")
			}
			if got := strings.Join(objs, "; "); got != row.objects {
				t.Errorf("objects\n got %s\nwant %s", got, row.objects)
			}
			b := p.Binding
			var binding string
			switch {
			case b.History != "":
				binding = fmt.Sprintf("history %s on %s, state %s, indexes %v, snapshot %s", b.History, b.Subject, b.State, b.Indexes, b.Snapshot)
			case b.Stream != "":
				binding = fmt.Sprintf("stream %s on %s", b.Stream, b.Subject)
			case b.Bucket != "":
				binding = "bucket " + b.Bucket
			case b.ObjectStore != "":
				binding = "objects " + b.ObjectStore
			}
			if binding != row.binding || b.Local != row.entry.Name || b.Storage != row.entry.Storage {
				t.Errorf("binding %s (%+v), want %s", binding, b, row.binding)
			}
		})
	}
}

// The keeper makes what an accepted verdict asks for, once, and says it (design 25 § 12.2, ADR 0595 §4; CAT-N2, N3):
// nothing before its first view; a replay not newer than what it holds is not applied; a refusal of its grant waits and
// is made on the backoff, never said refused; a refusal of the entry is said and raised, and cleared by a later make.
func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
	drafts := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil)
	log := entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil)
	for _, row := range []struct {
		name string
		run  func(t *testing.T, k *Keeper, bus *fakeBus, out *said)
	}{
		{"created once, binding said, nothing before the first view", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			k.mu.Lock()
			k.viewed = false
			k.mu.Unlock()
			k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}})
			if len(bus.created) != 0 {
				t.Fatalf("created %v before the first view", bus.created)
			}
			if _, err := k.AskFirstView(0); err != nil {
				t.Fatal(err)
			}
			k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
			k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
			if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want {
				t.Errorf("created %v, want %s", bus.created, want)
			}
			if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want {
				t.Errorf("said %v, want %s", out.events, want)
			}
			if b := out.bodies[1]; b.Binding.Stream != "m_log" || b.Binding.Subject != "mesh.bus.m.log" || b.Revision != 3 || b.Asked["m_log"].MaxBytes == 0 {
				t.Errorf("created carries %+v: the binding, the revision and the settings asked", b)
			}
			// In the field names of the seat mesh-broker's message said (mesh-contracts), beside binding and revision.
			raw, _ := json.Marshal(out.bodies[1])
			for _, w := range []string{`"module":"m"`, `"local-name":"log"`, `"storage-type":"stream"`, `"template":"log"`, `"bus-name":"m_log"`, `"binding":{`, `"revision":3`} {
				if !strings.Contains(string(raw), w) {
					t.Errorf("the body %s lacks %s", raw, w)
				}
			}
		}},
		{"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}})
			k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil})
			if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
				t.Errorf("held %+v: the replay at 4 took back what 9 asked", a)
			}
		}},
		{"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			bus.refuse["m_log"] = fmt.Errorf("%w: permissions violation", ErrRetry)
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
			if w := k.Waiting(); len(w) != 1 || len(out.events) != 0 {
				t.Fatalf("waiting %v, said %v: waits, says nothing", w, out.events)
			}
			delete(bus.refuse, "m_log")
			k.CheckAll()
			if len(k.Waiting()) != 0 || fmt.Sprint(out.events) != "[created m log]" {
				t.Errorf("waiting %v, said %v: made at the next check", k.Waiting(), out.events)
			}
		}},
		{"a refused create is said and raised, cleared by a later create", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			bus.refuse["KV_m_drafts"] = errors.New("insufficient resources")
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
			if fmt.Sprint(out.events) != "[refused m drafts]" || fmt.Sprint(out.conds) != "[raised nats.refused.m_drafts]" ||
				!strings.Contains(out.bodies[0].Why, "insufficient resources") {
				t.Fatalf("said %v %v %+v", out.events, out.conds, out.bodies)
			}
			delete(bus.refuse, "KV_m_drafts")
			out.reset()
			k.CheckAll()
			if fmt.Sprint(out.events) != "[created m drafts]" || fmt.Sprint(out.conds) != "[cleared nats.refused.m_drafts]" {
				t.Errorf("said %v %v: created, the refusal cleared", out.events, out.conds)
			}
		}},
		{"an entry the keeper cannot plan is refused, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{entry("a_b", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
			if len(bus.created) != 0 || fmt.Sprint(out.events) != "[refused m a_b]" {
				t.Errorf("created %v, said %v", bus.created, out.events)
			}
		}},
		// CAT-N4: check and drift (ADR 0595 §5, §6): a difference that only grows is applied, said by nothing; any other
		// is said and raised, never fixed quietly, and cleared once the bus holds what is asked again.
		{"a held entry that grows is applied without a person", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
			out.reset()
			bigger := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "2048"})
			k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{bigger}})
			if fmt.Sprint(bus.updated) != "[m_log]" || bus.held["m_log"].MaxBytes != 2048*MiB || len(out.events)+len(out.conds) != 0 {
				t.Errorf("updated %v to %d, said %v %v: grown quietly", bus.updated, bus.held["m_log"].MaxBytes, out.events, out.conds)
			}
		}},
		{"a held entry that would shrink is a drift, said, raised and cleared", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
			out.reset()
			smaller := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})
			k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{smaller}})
			if len(bus.updated) != 0 || fmt.Sprint(out.events) != "[drifted m log]" || fmt.Sprint(out.conds) != "[raised nats.drifted.m_log]" ||
				!strings.Contains(out.bodies[0].Why, "max-bytes asked 16777216, held 1073741824") {
				t.Fatalf("updated %v, said %v %v %+v: the drift said, nothing changed", bus.updated, out.events, out.conds, out.bodies)
			}
			if l := k.CheckAll(); len(l) != 1 || l[0].Drift == "" || len(out.events) != 1 {
				t.Errorf("check answers %+v and said %v: the drift once", l, out.events)
			}
			k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{log}})
			if fmt.Sprint(out.events) != "[drifted m log drift-cleared m log]" || out.conds[len(out.conds)-1] != "cleared nats.drifted.m_log" {
				t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds)
			}
		}},
		{"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			p, _ := PlanOf("m", drafts)
			bus.held["KV_m_drafts"] = p.Objects[0].Settings
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
			if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" {
				t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events)
			}
		}},
		// CAT-N5: orphans (design 25 § 12.2-12.3): an empty manifest-accepted asks nothing now, so what was made is an
		// orphan, said and raised, kept until a drop; asked again it is kept and its binding said again; a module missing
		// from the first view (set aside to wait) keeps its key; one never made leaves nothing behind.
		{"an empty verdict orphans what was made, and asking again keeps it", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
			out.reset()
			k.Heard(Accepted{Module: "m", Revision: 2})
			a, held := k.Held()["m"]
			if !held || !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 || fmt.Sprint(out.events) != "[orphaned m drafts]" ||
				fmt.Sprint(out.conds) != "[raised nats.orphaned.m_drafts]" {
				t.Fatalf("held %v %+v, deleted %v, said %v %v: orphaned, kept, raised", held, a, bus.deleted, out.events, out.conds)
			}
			k.Heard(Accepted{Module: "m", Revision: 3})
			k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}})
			if fmt.Sprint(out.events) != "[orphaned m drafts created m drafts]" || out.conds[len(out.conds)-1] != "cleared nats.orphaned.m_drafts" {
				t.Errorf("said %v %v: orphaned once, then created again and cleared", out.events, out.conds)
			}
		}},
		{"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
			k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil }
			if _, err := k.AskFirstView(0); err != nil {
				t.Fatal(err)
			}
			if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
				t.Errorf("held %+v: kept as asked, never orphaned by a first view", a)
			}
			bus.refuse["KV_n_x"] = errors.New("no")
			k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
			k.Heard(Accepted{Module: "n", Revision: 2})
			if _, held := k.Held()["n"]; held {
				t.Errorf("n's key is still held: nothing of it was made, so its empty verdict ends it")
			}
		}},
		// CAT-N6: fill (ADR 0595 §6): an entry that refuses when full raises `full` at the raise mark and clears only
		// below the clear mark; one that drops its oldest is never read; the health names the pass.
		{"full at the raise mark, cleared only below the clear mark", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			events := entry("events", catalogue.StorageStream, catalogue.TemplateEvents, map[string]string{"max-mib": "1"})
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, events}})
			out.reset()
			bus.fill["m_events"] = 1 * uint64(MiB)
			var steps []string
			for _, mib := range []uint64{40, 48, 46, 45, 44} {
				bus.fill["KV_m_drafts"] = mib * uint64(MiB)
				k.FillPass()
				steps = append(steps, fmt.Sprintf("%d:%d", mib, len(out.events)))
			}
			if fmt.Sprint(steps) != "[40:0 48:1 46:1 45:1 44:2]" || fmt.Sprint(out.events) != "[full m drafts full-cleared m drafts]" ||
				fmt.Sprint(out.conds) != "[raised nats.full.m_drafts cleared nats.full.m_drafts]" {
				t.Errorf("said %v %v: full at 48 of 64 MiB (75%%), kept at 46 and 45 (71%%, 70%%), cleared at 44 (68%%)", out.events, out.conds)
			}
			h := HealthOf(k.Status(), k.Held(), 0, k.now())
			if !h.Healthy || h.Pass == "none" {
				t.Errorf("health %+v: healthy, naming its pass", h)
			}
		}},
		{"the health is unhealthy past its bound, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			at := k.now()
			k.FillPass()
			later := at.Add(4 * time.Minute)
			if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "last fill pass is 4m0s old") {
				t.Errorf("health %+v: unhealthy, its pass too old", h)
			}
			if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") {
				t.Errorf("health %+v: names the verdicts waiting", h)
			}
			k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") }
			k.mu.Lock()
			k.status.FirstView.Answered = time.Time{}
			k.mu.Unlock()
			if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second {
				t.Fatalf("asked again in %s (%v): on the backoff", next, err)
			}
			if h := HealthOf(k.Status(), nil, 0, at); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") {
				t.Errorf("health %+v: unhealthy, naming the verb", h)
			}
		}},
		// CAT-N8: a person's requests: a drop removes an orphan alone, only on the operator's warrant binding exactly
		// that drop, after a snapshot, said dropped and recorded; a recreate rebuilds a drifted entry the same way.
		{"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			store := k.Store.(*memStore)
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
			k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}})
			var acts []string
			k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
			ask := asks.Ask{ID: "bus-1", Who: asks.Operator, Expires: k.now().Add(time.Hour)}
			var w *asks.Warrant
			k.Warrant = func(string) (*asks.Ask, *asks.Warrant, error) { return &ask, w, nil }
			put := func(r Request) Request {
				raw, _ := json.Marshal(r)
				_, _ = store.Put(StateRequests, r.ID, raw)
				k.ActOnRequests()
				raw, _, _, _ = store.Get(StateRequests, r.ID)
				var got Request
				_ = json.Unmarshal(raw, &got)
				return got
			}
			if got := put(Request{ID: "r0", Verb: VerbDrop, Module: "m", Local: "log", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "drop removes an orphan alone") {
				t.Errorf("dropping an asked entry: %+v", got)
			}
			if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !got.Done.IsZero() || len(bus.deleted) != 0 {
				t.Fatalf("acted with no warrant: %+v, deleted %v", got, bus.deleted)
			}
			binds := func(act asks.Act) string { d, _ := asks.ActDigest(act); return d }
			choose := func(act asks.Act) {
				ask.Options = []asks.Option{{ID: "go", Label: "Remove", Level: asks.Destroy, Binds: binds(act)}}
				w = &asks.Warrant{Ask: "bus-1", Asker: Module, Outcome: asks.OutcomeChosen, By: &asks.Person{Who: asks.Operator},
					Option: "go", Label: "Remove", Level: asks.Destroy, At: k.now(), AskDigest: ask.Digest()}
			}
			choose(asks.Act{"verb": "drop", "module": "m", "local": "log"})
			if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "bound") || len(bus.deleted) != 0 {
				t.Fatalf("acted on a warrant for another act: %+v, deleted %v", got, bus.deleted)
			}
			choose(asks.Act{"verb": "drop", "module": "m", "local": "drafts"})
			out.reset()
			got := put(Request{ID: "r2", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", Why: "unused"})
			if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts]" ||
				fmt.Sprint(out.events) != "[dropped m drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_drop: dropped drafts of m") ||
				!strings.Contains(acts[0], "person.jo") {
				t.Errorf("ended %+v, snapshots %v, deleted %v, said %v, recorded %v", got, bus.snaps, bus.deleted, out.events, acts)
			}
			if _, made := k.Held()["m"].Made["drafts"]; made {
				t.Error("the dropped entry is still held")
			}
		}},
		{"a recreate rebuilds a drifted entry after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			store := k.Store.(*memStore)
			k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
			k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})}})
			binds, _ := asks.ActDigest(asks.Act{"verb": "recreate", "module": "m", "local": "log"})
			ask := asks.Ask{ID: "bus-2", Who: asks.Operator, Expires: k.now().Add(time.Hour), Options: []asks.Option{{ID: "go", Label: "Rebuild", Level: asks.Destroy, Binds: binds}}}
			w := &asks.Warrant{Ask: "bus-2", Asker: Module, Outcome: asks.OutcomeChosen, By: &asks.Person{Who: asks.Operator},
				Option: "go", Label: "Rebuild", Level: asks.Destroy, At: k.now(), AskDigest: ask.Digest()}
			k.Warrant = func(string) (*asks.Ask, *asks.Warrant, error) { return &ask, w, nil }
			out.reset()
			raw, _ := json.Marshal(Request{ID: "r3", Verb: VerbRecreate, Module: "m", Local: "log", Ask: "bus-2"})
			_, _ = store.Put(StateRequests, "r3", raw)
			k.ActOnRequests()
			if bus.held["m_log"].MaxBytes != 16*MiB || fmt.Sprint(out.events) != "[recreated m log]" ||
				fmt.Sprint(out.conds) != "[cleared nats.drifted.m_log]" || k.Held()["m"].Made["log"].Drift != "" {
				t.Errorf("held %+v, said %v %v: rebuilt as asked, the drift cleared", bus.held["m_log"], out.events, out.conds)
			}
		}},
		// CAT-N2b: nats_first_view_again asks the first view at once, by hand, recorded as a hand act.
		{"the first view asked again by hand", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
			store := k.Store.(*memStore)
			var acts []string
			k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
			k.View = func() (FirstView, error) {
				return FirstView{Revision: 8, Modules: []Accepted{{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}, nil
			}
			raw, _ := json.Marshal(Request{ID: "r9", Verb: VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"})
			_, _ = store.Put(StateRequests, "r9", raw)
			k.ActOnRequests()
			raw, _, _, _ = store.Get(StateRequests, "r9")
			var got Request
			_ = json.Unmarshal(raw, &got)
			if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts]" ||
				len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked mesh-delivery-indexer.indexer_bus_state again by hand") {
				t.Errorf("ended %+v, created %v, recorded %v", got, bus.created, acts)
			}
		}},
	} {
		t.Run(row.name, func(t *testing.T) {
			k, bus, _, out := newKeeper(FirstView{Revision: 1})
			if _, err := k.AskFirstView(0); err != nil {
				t.Fatal(err)
			}
			row.run(t, k, bus, out)
		})
	}
}