Files
mesh-catalog/modules/nats/internal/keeper/keeper_test.go
T
jschoubben 93bfdd9e2f CAT-N5: orphans: an empty accepted manifest asks nothing, so what was made is orphaned and kept until a drop
An entry made for a module whose newest accepted manifest no longer asks for
it is said orphaned and raised, never removed; asked again it is kept, its
binding said again with created and the condition cleared. A module missing
from a first view keeps its key (a set-aside manifest is not in it); an entry
never made leaves nothing, so the empty verdict ends that module's key.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
2026-11-04 12:34:59 +01:00

414 lines
18 KiB
Go

package keeper
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// fakeBus holds objects by stream, as a bus would, and refuses what a test tells it to.
type fakeBus struct {
mu sync.Mutex
held map[string]Settings
fill map[string]uint64
refuse map[string]error
created []string
updated []string
deleted []string
snaps [][]string
}
func newFakeBus() *fakeBus {
return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}}
}
func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) {
b.mu.Lock()
defer b.mu.Unlock()
s, ok := b.held[o.Stream()]
return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil
}
func (b *fakeBus) Create(_ context.Context, o Object) error {
b.mu.Lock()
defer b.mu.Unlock()
if err := b.refuse[o.Stream()]; err != nil {
return err
}
b.held[o.Stream()] = o.Settings
b.created = append(b.created, o.Stream())
return nil
}
func (b *fakeBus) Update(_ context.Context, o Object) error {
b.mu.Lock()
defer b.mu.Unlock()
b.held[o.Stream()] = o.Settings
b.updated = append(b.updated, o.Stream())
return nil
}
func (b *fakeBus) Delete(_ context.Context, o Object) error {
b.mu.Lock()
defer b.mu.Unlock()
delete(b.held, o.Stream())
b.deleted = append(b.deleted, o.Stream())
return nil
}
func (b *fakeBus) Snapshot(_ context.Context, streams []string) (string, error) {
b.mu.Lock()
defer b.mu.Unlock()
b.snaps = append(b.snaps, streams)
return "/snapshots/" + strings.Join(streams, "+") + ".tar", nil
}
// memStore is a key-value state in memory.
type memStore struct {
mu sync.Mutex
kv map[string][]byte
revs map[string]uint64
rev uint64
}
func newMemStore() *memStore { return &memStore{kv: map[string][]byte{}, revs: map[string]uint64{}} }
func (s *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.kv[state+"/"+key]
return v, s.revs[state+"/"+key], ok, nil
}
func (s *memStore) Put(state, key string, value []byte) (uint64, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.rev++
s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev
return s.rev, nil
}
func (s *memStore) Delete(state, key string) error {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.kv, state+"/"+key)
return nil
}
func (s *memStore) Keys(state string) ([]string, error) {
s.mu.Lock()
defer s.mu.Unlock()
var out []string
for k := range s.kv {
if st, key, _ := strings.Cut(k, "/"); st == state {
out = append(out, key)
}
}
sort.Strings(out)
return out, nil
}
// said is what a keeper said: its events and its conditions, in order.
type said struct {
mu sync.Mutex
events []string
bodies []Event
conds []string
}
func (s *said) emit(event string, body Event) error {
s.mu.Lock()
defer s.mu.Unlock()
s.events = append(s.events, event+" "+body.Module+" "+body.Local)
s.bodies = append(s.bodies, body)
return nil
}
func (s *said) say(c asks.Condition) error {
s.mu.Lock()
defer s.mu.Unlock()
s.conds = append(s.conds, c.State+" "+c.Key)
return nil
}
func (s *said) reset() {
s.mu.Lock()
defer s.mu.Unlock()
s.events, s.bodies, s.conds = nil, nil, nil
}
func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) {
bus, store, out := newFakeBus(), newMemStore(), &said{}
at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)
k := &Keeper{Bus: bus, Store: store, Emit: out.emit, Say: out.say,
View: func() (FirstView, error) { return view, nil },
Now: func() time.Time { return at }}
return k, bus, store, out
}
func entry(name, storage, template string, options map[string]string) catalogue.BusStateEntry {
e := catalogue.BusStateEntry{Name: name, Storage: storage, Template: template}
if len(options) > 0 {
e.Options = map[string]json.RawMessage{}
for k, v := range options {
e.Options[k] = json.RawMessage(v)
}
}
return e
}
// What one entry is on the bus (design 25 § 12.1, CAT-N3): each template's objects, named `<module>_<local>`, its
// binding, and the fixed settings no option changes; and the entries the keeper refuses itself, naming why.
func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) {
ledger := entry("records", catalogue.StorageLedger, catalogue.TemplateLedger, map[string]string{"max-mib": "16"})
ledger.Types = []catalogue.LedgerType{{Name: "decision", Indexes: []catalogue.LedgerIndex{{Name: "by-topic", On: []string{"topic"}}}}}
bare := entry("raw", catalogue.StorageStream, "", nil)
bare.Configuration = map[string]json.RawMessage{"when-full": json.RawMessage(`"drop-oldest"`), "max-mib": json.RawMessage("8"), "max-age-days": json.RawMessage("3")}
for _, row := range []struct {
name string
entry catalogue.BusStateEntry
objects string
binding string
refused string
}{
{name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}),
objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}",
binding: "bucket mesh-knowledge_drafts"},
{name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil),
objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}",
binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"},
{name: "work queue", entry: entry("jobs", catalogue.StorageStream, catalogue.TemplateWorkQueue, map[string]string{"max-mib": "4", "duplicates": "120"}),
objects: "stream mesh-knowledge_jobs [mesh.bus.mesh-knowledge.jobs.>] {4 MiB, age 0, history 0, new, size 0, work-queue, duplicates 120}",
binding: "stream mesh-knowledge_jobs on mesh.bus.mesh-knowledge.jobs"},
{name: "log", entry: entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil),
objects: "stream mesh-knowledge_log [mesh.bus.mesh-knowledge.log.>] {1024 MiB, age 0, history 0, new, size 262144}",
binding: "stream mesh-knowledge_log on mesh.bus.mesh-knowledge.log"},
{name: "objects", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "32"}),
objects: "object-store mesh-knowledge_files {32 MiB, age 0, history 0, new, size 0}",
binding: "objects mesh-knowledge_files"},
{name: "ledger", entry: ledger,
objects: "stream mesh-knowledge_records [mesh.ledger.mesh-knowledge.records.history.>] {16 MiB, age 0, history 0, new, size 262144}; " +
"key-value mesh-knowledge_records_state {64 MiB, age 0, history 1, new, size 262144}; " +
"key-value mesh-knowledge_records_index-by-topic {64 MiB, age 0, history 1, new, size 262144}; " +
"stream mesh-knowledge_records_snapshot [mesh.ledger.mesh-knowledge.records.snapshot.>] {16 MiB, age 0, history 1, old, size 0}",
binding: "history mesh-knowledge_records on mesh.ledger.mesh-knowledge.records.history, state mesh-knowledge_records_state, indexes map[by-topic:mesh-knowledge_records_index-by-topic], snapshot mesh.ledger.mesh-knowledge.records.snapshot"},
{name: "bare stream", entry: bare,
objects: "stream mesh-knowledge_raw [mesh.bus.mesh-knowledge.raw.>] {8 MiB, age 259200, history 0, old, size 0}",
binding: "stream mesh-knowledge_raw on mesh.bus.mesh-knowledge.raw"},
{name: "a local name holding the separator", entry: entry("my_drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil),
refused: `the local name "my_drafts" is empty or holds "_"`},
{name: "objects with no cap", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, nil),
refused: "gives no max-mib"},
{name: "a bare stream with no when-full", entry: entry("raw", catalogue.StorageStream, "", nil),
refused: "must give when-full"},
{name: "a template on the wrong storage type", entry: entry("x", catalogue.StorageKeyValue, catalogue.TemplateLog, nil),
refused: "is none the bus module makes"},
} {
t.Run(row.name, func(t *testing.T) {
p, err := PlanOf("mesh-knowledge", row.entry)
if row.refused != "" {
if err == nil || !strings.Contains(err.Error(), row.refused) {
t.Fatalf("planned %+v (%v): refused, naming %q", p, err, row.refused)
}
return
}
if err != nil {
t.Fatal(err)
}
var objs []string
for _, o := range p.Objects {
s := o.Settings
w := fmt.Sprintf("%s %s", o.Kind, o.Name)
if len(o.Subjects) > 0 {
w += fmt.Sprintf(" %v", o.Subjects)
}
w += fmt.Sprintf(" {%d MiB, age %d, history %d, %s, size %d", s.MaxBytes/MiB, s.MaxAgeSeconds, s.History, s.Discard, s.MaxMsgSize)
if s.WorkQueue {
w += fmt.Sprintf(", work-queue, duplicates %d", s.DuplicatesSeconds)
}
objs = append(objs, w+"}")
}
if got := strings.Join(objs, "; "); got != row.objects {
t.Errorf("objects\n got %s\nwant %s", got, row.objects)
}
b := p.Binding
var binding string
switch {
case b.History != "":
binding = fmt.Sprintf("history %s on %s, state %s, indexes %v, snapshot %s", b.History, b.Subject, b.State, b.Indexes, b.Snapshot)
case b.Stream != "":
binding = fmt.Sprintf("stream %s on %s", b.Stream, b.Subject)
case b.Bucket != "":
binding = "bucket " + b.Bucket
case b.ObjectStore != "":
binding = "objects " + b.ObjectStore
}
if binding != row.binding || b.Local != row.entry.Name || b.Storage != row.entry.Storage {
t.Errorf("binding %s (%+v), want %s", binding, b, row.binding)
}
})
}
}
// The keeper makes what an accepted verdict asks for, once, and says it (design 25 § 12.2, ADR 0595 §4; CAT-N2, N3):
// nothing before its first view; a replay not newer than what it holds is not applied; a refusal of its grant waits and
// is made on the backoff, never said refused; a refusal of the entry is said and raised, and cleared by a later make.
func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
drafts := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil)
log := entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil)
for _, row := range []struct {
name string
run func(t *testing.T, k *Keeper, bus *fakeBus, out *said)
}{
{"created once, binding said, nothing before the first view", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.mu.Lock()
k.viewed = false
k.mu.Unlock()
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}})
if len(bus.created) != 0 {
t.Fatalf("created %v before the first view", bus.created)
}
if _, err := k.AskFirstView(0); err != nil {
t.Fatal(err)
}
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want {
t.Errorf("created %v, want %s", bus.created, want)
}
if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want {
t.Errorf("said %v, want %s", out.events, want)
}
if b := out.bodies[1]; b.Binding.Stream != "m_log" || b.Binding.Subject != "mesh.bus.m.log" || b.Revision != 3 || b.Asked["m_log"].MaxBytes == 0 {
t.Errorf("created carries %+v: the binding, the revision and the settings asked", b)
}
}},
{"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}})
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil})
if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
t.Errorf("held %+v: the replay at 4 took back what 9 asked", a)
}
}},
{"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
bus.refuse["m_log"] = fmt.Errorf("%w: permissions violation", ErrRetry)
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
if w := k.Waiting(); len(w) != 1 || len(out.events) != 0 {
t.Fatalf("waiting %v, said %v: waits, says nothing", w, out.events)
}
delete(bus.refuse, "m_log")
k.CheckAll()
if len(k.Waiting()) != 0 || fmt.Sprint(out.events) != "[created m log]" {
t.Errorf("waiting %v, said %v: made at the next check", k.Waiting(), out.events)
}
}},
{"a refused create is said and raised, cleared by a later create", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
bus.refuse["KV_m_drafts"] = errors.New("insufficient resources")
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
if fmt.Sprint(out.events) != "[refused m drafts]" || fmt.Sprint(out.conds) != "[raised nats.refused.m_drafts]" ||
!strings.Contains(out.bodies[0].Why, "insufficient resources") {
t.Fatalf("said %v %v %+v", out.events, out.conds, out.bodies)
}
delete(bus.refuse, "KV_m_drafts")
out.reset()
k.CheckAll()
if fmt.Sprint(out.events) != "[created m drafts]" || fmt.Sprint(out.conds) != "[cleared nats.refused.m_drafts]" {
t.Errorf("said %v %v: created, the refusal cleared", out.events, out.conds)
}
}},
{"an entry the keeper cannot plan is refused, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{entry("a_b", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[refused m a_b]" {
t.Errorf("created %v, said %v", bus.created, out.events)
}
}},
// CAT-N4: check and drift (ADR 0595 §5, §6): a difference that only grows is applied, said by nothing; any other
// is said and raised, never fixed quietly, and cleared once the bus holds what is asked again.
{"a held entry that grows is applied without a person", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
out.reset()
bigger := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "2048"})
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{bigger}})
if fmt.Sprint(bus.updated) != "[m_log]" || bus.held["m_log"].MaxBytes != 2048*MiB || len(out.events)+len(out.conds) != 0 {
t.Errorf("updated %v to %d, said %v %v: grown quietly", bus.updated, bus.held["m_log"].MaxBytes, out.events, out.conds)
}
}},
{"a held entry that would shrink is a drift, said, raised and cleared", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
out.reset()
smaller := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{smaller}})
if len(bus.updated) != 0 || fmt.Sprint(out.events) != "[drifted m log]" || fmt.Sprint(out.conds) != "[raised nats.drifted.m_log]" ||
!strings.Contains(out.bodies[0].Why, "max-bytes asked 16777216, held 1073741824") {
t.Fatalf("updated %v, said %v %v %+v: the drift said, nothing changed", bus.updated, out.events, out.conds, out.bodies)
}
if l := k.CheckAll(); len(l) != 1 || l[0].Drift == "" || len(out.events) != 1 {
t.Errorf("check answers %+v and said %v: the drift once", l, out.events)
}
k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{log}})
if fmt.Sprint(out.events) != "[drifted m log drift-cleared m log]" || out.conds[len(out.conds)-1] != "cleared nats.drifted.m_log" {
t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds)
}
}},
{"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
p, _ := PlanOf("m", drafts)
bus.held["KV_m_drafts"] = p.Objects[0].Settings
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" {
t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events)
}
}},
// CAT-N5: orphans (design 25 § 12.2-12.3): an empty manifest-accepted asks nothing now, so what was made is an
// orphan, said and raised, kept until a drop; asked again it is kept and its binding said again; a module missing
// from the first view (set aside to wait) keeps its key; one never made leaves nothing behind.
{"an empty verdict orphans what was made, and asking again keeps it", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
out.reset()
k.Heard(Accepted{Module: "m", Revision: 2})
a, held := k.Held()["m"]
if !held || !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 || fmt.Sprint(out.events) != "[orphaned m drafts]" ||
fmt.Sprint(out.conds) != "[raised nats.orphaned.m_drafts]" {
t.Fatalf("held %v %+v, deleted %v, said %v %v: orphaned, kept, raised", held, a, bus.deleted, out.events, out.conds)
}
k.Heard(Accepted{Module: "m", Revision: 3})
k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}})
if fmt.Sprint(out.events) != "[orphaned m drafts created m drafts]" || out.conds[len(out.conds)-1] != "cleared nats.orphaned.m_drafts" {
t.Errorf("said %v %v: orphaned once, then created again and cleared", out.events, out.conds)
}
}},
{"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil }
if _, err := k.AskFirstView(0); err != nil {
t.Fatal(err)
}
if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
t.Errorf("held %+v: kept as asked, never orphaned by a first view", a)
}
bus.refuse["KV_n_x"] = errors.New("no")
k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
k.Heard(Accepted{Module: "n", Revision: 2})
if _, held := k.Held()["n"]; held {
t.Errorf("n's key is still held: nothing of it was made, so its empty verdict ends it")
}
}},
} {
t.Run(row.name, func(t *testing.T) {
k, bus, _, out := newKeeper(FirstView{Revision: 1})
if _, err := k.AskFirstView(0); err != nil {
t.Fatal(err)
}
row.run(t, k, bus, out)
})
}
}