package keeper import ( "context" "encoding/json" "errors" "fmt" "sort" "strings" "sync" "testing" "time" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-sdk/go/catalogue" ) // fakeBus holds objects by stream, as a bus would, and refuses what a test tells it to. type fakeBus struct { mu sync.Mutex held map[string]Settings fill map[string]uint64 refuse map[string]error created []string updated []string deleted []string snaps [][]string } func newFakeBus() *fakeBus { return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}} } func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) { b.mu.Lock() defer b.mu.Unlock() s, ok := b.held[o.Stream()] return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil } func (b *fakeBus) Create(_ context.Context, o Object) error { b.mu.Lock() defer b.mu.Unlock() if err := b.refuse[o.Stream()]; err != nil { return err } b.held[o.Stream()] = o.Settings b.created = append(b.created, o.Stream()) return nil } func (b *fakeBus) Update(_ context.Context, o Object) error { b.mu.Lock() defer b.mu.Unlock() b.held[o.Stream()] = o.Settings b.updated = append(b.updated, o.Stream()) return nil } func (b *fakeBus) Delete(_ context.Context, o Object) error { b.mu.Lock() defer b.mu.Unlock() delete(b.held, o.Stream()) b.deleted = append(b.deleted, o.Stream()) return nil } func (b *fakeBus) Snapshot(_ context.Context, streams []string) (string, error) { b.mu.Lock() defer b.mu.Unlock() b.snaps = append(b.snaps, streams) if err := b.refuse["snapshot"]; err != nil { return "", err } return "/snapshots/" + strings.Join(streams, "+") + ".tar", nil } // memStore is a key-value state in memory. type memStore struct { // refuse refuses a Put of `/`. refuse map[string]error mu sync.Mutex kv map[string][]byte revs map[string]uint64 rev uint64 } func newMemStore() *memStore { return &memStore{kv: map[string][]byte{}, revs: map[string]uint64{}, refuse: map[string]error{}} } func (s *memStore) Get(state, key string) ([]byte, uint64, bool, error) { s.mu.Lock() defer s.mu.Unlock() v, ok := s.kv[state+"/"+key] return v, s.revs[state+"/"+key], ok, nil } func (s *memStore) Put(state, key string, value []byte) (uint64, error) { s.mu.Lock() defer s.mu.Unlock() if err := s.refuse[state+"/"+key]; err != nil { return 0, err } s.rev++ s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev return s.rev, nil } func (s *memStore) Create(state, key string, value []byte) (uint64, error) { s.mu.Lock() defer s.mu.Unlock() if _, ok := s.kv[state+"/"+key]; ok { return 0, ErrExists } s.rev++ s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev return s.rev, nil } func (s *memStore) Delete(state, key string) error { s.mu.Lock() defer s.mu.Unlock() delete(s.kv, state+"/"+key) return nil } func (s *memStore) Keys(state string) ([]string, error) { s.mu.Lock() defer s.mu.Unlock() var out []string for k := range s.kv { if st, key, _ := strings.Cut(k, "/"); st == state { out = append(out, key) } } sort.Strings(out) return out, nil } // said is what a keeper said: its events and its conditions, in order. type said struct { mu sync.Mutex events []string bodies []Event ids []string conds []string // down refuses every event and condition while set, as a bus that does not take them; partly takes an event on the // seat's subject and refuses its history. down, partly bool // left are the subjects each try was asked to say an event on (nil: every one). left [][]string } func (s *said) emit(e *Saying) error { s.mu.Lock() defer s.mu.Unlock() s.left = append(s.left, e.Left) if s.down { e.Left, e.Said = []string{"seat", "history"}, nil return errors.New("no responders") } if s.partly && e.Left == nil { e.Left, e.Said = []string{"history"}, []string{"seat"} s.ids = append(s.ids, e.ID) return errors.New("the history stream is full") } s.events = append(s.events, e.Event+" "+e.Body.Module+" "+e.Body.Local) s.bodies = append(s.bodies, e.Body) s.ids = append(s.ids, e.ID) e.Left = nil return nil } func (s *said) say(c oc.Condition) error { s.mu.Lock() defer s.mu.Unlock() if s.down { return errors.New("no responders") } s.conds = append(s.conds, c.State+" "+c.Key) return nil } func (s *said) reset() { s.mu.Lock() defer s.mu.Unlock() s.events, s.bodies, s.conds = nil, nil, nil } func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) { bus, store, out := newFakeBus(), newMemStore(), &said{} at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) k := &Keeper{Bus: bus, Store: store, Emit: out.emit, Say: out.say, View: func() (FirstView, error) { return view, nil }, Now: func() time.Time { return at }} return k, bus, store, out } func entry(name, storage, template string, options map[string]string) catalogue.BusStateEntry { e := catalogue.BusStateEntry{Name: name, Storage: storage, Template: template} if len(options) > 0 { e.Options = map[string]json.RawMessage{} for k, v := range options { e.Options[k] = json.RawMessage(v) } } return e } // What one entry is on the bus (design 25 § 12.1, CAT-N3): each template's objects, named `_`, its // binding, and the fixed settings no option changes; and the entries the keeper refuses itself, naming why. func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) { ledger := entry("records", catalogue.StorageLedger, catalogue.TemplateLedger, map[string]string{"max-mib": "16"}) ledger.Types = []catalogue.LedgerType{{Name: "decision", Indexes: []catalogue.LedgerIndex{{Name: "by-topic", On: []string{"topic"}}}}} bare := entry("raw", catalogue.StorageStream, "", nil) bare.Configuration = map[string]json.RawMessage{"when-full": json.RawMessage(`"drop-oldest"`), "max-mib": json.RawMessage("8"), "max-age-days": json.RawMessage("3")} for _, row := range []struct { name string entry catalogue.BusStateEntry objects string binding string refused string }{ {name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}), objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}", binding: "bucket mesh-knowledge_drafts"}, {name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil), objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}", binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"}, {name: "work queue", entry: entry("jobs", catalogue.StorageStream, catalogue.TemplateWorkQueue, map[string]string{"max-mib": "4", "duplicates": "120"}), objects: "stream mesh-knowledge_jobs [mesh.bus.mesh-knowledge.jobs.>] {4 MiB, age 0, history 0, new, size 0, work-queue, duplicates 120}", binding: "stream mesh-knowledge_jobs on mesh.bus.mesh-knowledge.jobs"}, {name: "log", entry: entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil), objects: "stream mesh-knowledge_log [mesh.bus.mesh-knowledge.log.>] {1024 MiB, age 0, history 0, new, size 262144}", binding: "stream mesh-knowledge_log on mesh.bus.mesh-knowledge.log"}, {name: "objects", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "32"}), objects: "object-store mesh-knowledge_files {32 MiB, age 0, history 0, new, size 0}", binding: "objects mesh-knowledge_files"}, {name: "ledger", entry: ledger, objects: "stream mesh-knowledge_records [mesh.ledger.mesh-knowledge.records.history.>] {16 MiB, age 0, history 0, new, size 262144}; " + "key-value mesh-knowledge_records_state {64 MiB, age 0, history 1, new, size 262144}; " + "key-value mesh-knowledge_records_index-by-topic {64 MiB, age 0, history 1, new, size 262144}; " + "stream mesh-knowledge_records_snapshot [mesh.ledger.mesh-knowledge.records.snapshot.>] {16 MiB, age 0, history 1, old, size 0}", binding: "history mesh-knowledge_records on mesh.ledger.mesh-knowledge.records.history, state mesh-knowledge_records_state, indexes map[by-topic:mesh-knowledge_records_index-by-topic], snapshot mesh.ledger.mesh-knowledge.records.snapshot"}, {name: "bare stream", entry: bare, objects: "stream mesh-knowledge_raw [mesh.bus.mesh-knowledge.raw.>] {8 MiB, age 259200, history 0, old, size 0}", binding: "stream mesh-knowledge_raw on mesh.bus.mesh-knowledge.raw"}, {name: "a local name holding the separator", entry: entry("my_drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil), refused: `the local name "my_drafts" is empty or holds "_"`}, {name: "objects with no cap", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, nil), refused: "gives no max-mib"}, {name: "a bare stream with no when-full", entry: entry("raw", catalogue.StorageStream, "", nil), refused: "must give when-full"}, {name: "a template on the wrong storage type", entry: entry("x", catalogue.StorageKeyValue, catalogue.TemplateLog, nil), refused: "is none the bus module makes"}, } { t.Run(row.name, func(t *testing.T) { p, err := PlanOf("mesh-knowledge", row.entry) if row.refused != "" { if err == nil || !strings.Contains(err.Error(), row.refused) { t.Fatalf("planned %+v (%v): refused, naming %q", p, err, row.refused) } return } if err != nil { t.Fatal(err) } var objs []string for _, o := range p.Objects { s := o.Settings w := fmt.Sprintf("%s %s", o.Kind, o.Name) if len(o.Subjects) > 0 { w += fmt.Sprintf(" %v", o.Subjects) } w += fmt.Sprintf(" {%d MiB, age %d, history %d, %s, size %d", s.MaxBytes/MiB, s.MaxAgeSeconds, s.History, s.Discard, s.MaxMsgSize) if s.WorkQueue { w += fmt.Sprintf(", work-queue, duplicates %d", s.DuplicatesSeconds) } objs = append(objs, w+"}") } if got := strings.Join(objs, "; "); got != row.objects { t.Errorf("objects\n got %s\nwant %s", got, row.objects) } b := p.Binding var binding string switch { case b.History != "": binding = fmt.Sprintf("history %s on %s, state %s, indexes %v, snapshot %s", b.History, b.Subject, b.State, b.Indexes, b.Snapshot) case b.Stream != "": binding = fmt.Sprintf("stream %s on %s", b.Stream, b.Subject) case b.Bucket != "": binding = "bucket " + b.Bucket case b.ObjectStore != "": binding = "objects " + b.ObjectStore } if binding != row.binding || b.Local != row.entry.Name || b.Storage != row.entry.Storage { t.Errorf("binding %s (%+v), want %s", binding, b, row.binding) } }) } } // The keeper makes what an accepted verdict asks for, once, and says it (design 25 § 12.2, ADR 0595 §4; CAT-N2, N3): // nothing before its first view; a replay not newer than what it holds is not applied; a refusal of its grant waits and // is made on the backoff, never said refused; a refusal of the entry is said and raised, and cleared by a later make. func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { drafts := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil) log := entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil) for _, row := range []struct { name string run func(t *testing.T, k *Keeper, bus *fakeBus, out *said) }{ {"created once, binding said, nothing before the first view", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.mu.Lock() k.viewed = false k.mu.Unlock() k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}}) if len(bus.created) != 0 { t.Fatalf("created %v before the first view", bus.created) } if _, err := k.AskFirstView(0); err != nil { t.Fatal(err) } k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}}) k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}}) if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want { t.Errorf("created %v, want %s", bus.created, want) } if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want { t.Errorf("said %v, want %s", out.events, want) } if b := out.bodies[1]; b.Binding.Stream != "m_log" || b.Binding.Subject != "mesh.bus.m.log" || b.Revision != 3 || b.Asked["m_log"].MaxBytes == 0 { t.Errorf("created carries %+v: the binding, the revision and the settings asked", b) } // In the field names of the seat mesh-broker's message said (mesh-contracts), beside binding and revision. raw, _ := json.Marshal(out.bodies[1]) for _, w := range []string{`"module":"m"`, `"local-name":"log"`, `"storage-type":"stream"`, `"template":"log"`, `"bus-name":"m_log"`, `"binding":{`, `"revision":3`} { if !strings.Contains(string(raw), w) { t.Errorf("the body %s lacks %s", raw, w) } } }}, {"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}) k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil}) if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned { t.Errorf("held %+v: the replay at 4 took back what 9 asked", a) } }}, {"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { bus.refuse["m_log"] = fmt.Errorf("%w: permissions violation", ErrRetry) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}}) if w := k.Waiting(); len(w) != 1 || len(out.events) != 0 { t.Fatalf("waiting %v, said %v: waits, says nothing", w, out.events) } delete(bus.refuse, "m_log") k.CheckAll() if len(k.Waiting()) != 0 || fmt.Sprint(out.events) != "[created m log]" { t.Errorf("waiting %v, said %v: made at the next check", k.Waiting(), out.events) } }}, {"a refused create is said and raised, cleared by a later create", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { bus.refuse["KV_m_drafts"] = errors.New("insufficient resources") k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) if fmt.Sprint(out.events) != "[refused m drafts]" || fmt.Sprint(out.conds) != "[raised nats.refused.m_drafts]" || !strings.Contains(out.bodies[0].Why, "insufficient resources") { t.Fatalf("said %v %v %+v", out.events, out.conds, out.bodies) } delete(bus.refuse, "KV_m_drafts") out.reset() k.CheckAll() if fmt.Sprint(out.events) != "[created m drafts]" || fmt.Sprint(out.conds) != "[cleared nats.refused.m_drafts]" { t.Errorf("said %v %v: created, the refusal cleared", out.events, out.conds) } }}, {"an entry the keeper cannot plan is refused, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{entry("a_b", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}}) if len(bus.created) != 0 || fmt.Sprint(out.events) != "[refused m a_b]" { t.Errorf("created %v, said %v", bus.created, out.events) } }}, // CAT-N4: check and drift (ADR 0595 §5, §6): a difference that only grows is applied, said by nothing; any other // is said and raised, never fixed quietly, and cleared once the bus holds what is asked again. {"a held entry that grows is applied without a person", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}}) out.reset() bigger := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "2048"}) k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{bigger}}) if fmt.Sprint(bus.updated) != "[m_log]" || bus.held["m_log"].MaxBytes != 2048*MiB || len(out.events)+len(out.conds) != 0 { t.Errorf("updated %v to %d, said %v %v: grown quietly", bus.updated, bus.held["m_log"].MaxBytes, out.events, out.conds) } }}, {"a held entry that would shrink is a drift, said, raised and cleared", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}}) out.reset() smaller := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"}) k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{smaller}}) if len(bus.updated) != 0 || fmt.Sprint(out.events) != "[drifted m log]" || fmt.Sprint(out.conds) != "[raised nats.drifted.m_log]" || !strings.Contains(out.bodies[0].Why, "max-bytes asked 16777216, held 1073741824") { t.Fatalf("updated %v, said %v %v %+v: the drift said, nothing changed", bus.updated, out.events, out.conds, out.bodies) } if l := k.CheckAll(); len(l) != 1 || l[0].Drift == "" || len(out.events) != 1 { t.Errorf("check answers %+v and said %v: the drift once", l, out.events) } k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{log}}) if fmt.Sprint(out.events) != "[drifted m log drift-cleared m log]" || out.conds[len(out.conds)-1] != "cleared nats.drifted.m_log" { t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds) } }}, {"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { p, _ := PlanOf("m", drafts) bus.held["KV_m_drafts"] = p.Objects[0].Settings k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" { t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events) } }}, // CAT-N5: orphans (design 25 § 12.2-12.3): an empty manifest-accepted asks nothing now, so what was made is an // orphan, said and raised, kept until a drop; asked again it is kept and its binding said again; a module missing // from the first view (set aside to wait) keeps its key; one never made leaves nothing behind. {"an empty verdict orphans what was made, and asking again keeps it", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) out.reset() k.Heard(Accepted{Module: "m", Revision: 2}) a, held := k.Held()["m"] if !held || !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 || fmt.Sprint(out.events) != "[orphaned m drafts]" || fmt.Sprint(out.conds) != "[raised nats.orphaned.m_drafts]" { t.Fatalf("held %v %+v, deleted %v, said %v %v: orphaned, kept, raised", held, a, bus.deleted, out.events, out.conds) } k.Heard(Accepted{Module: "m", Revision: 3}) k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}}) if fmt.Sprint(out.events) != "[orphaned m drafts created m drafts]" || out.conds[len(out.conds)-1] != "cleared nats.orphaned.m_drafts" { t.Errorf("said %v %v: orphaned once, then created again and cleared", out.events, out.conds) } }}, {"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil } if _, err := k.AskFirstView(0); err != nil { t.Fatal(err) } if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned { t.Errorf("held %+v: kept as asked, never orphaned by a first view", a) } bus.refuse["KV_n_x"] = errors.New("no") k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}}) k.Heard(Accepted{Module: "n", Revision: 2}) if _, held := k.Held()["n"]; held { t.Errorf("n's key is still held: nothing of it was made, so its empty verdict ends it") } }}, // CAT-N6: fill (ADR 0595 §6): an entry that refuses when full raises `full` at the raise mark and clears only // below the clear mark; one that drops its oldest is never read; the health names the pass. {"full at the raise mark, cleared only below the clear mark", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { events := entry("events", catalogue.StorageStream, catalogue.TemplateEvents, map[string]string{"max-mib": "1"}) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, events}}) out.reset() bus.fill["m_events"] = 1 * uint64(MiB) var steps []string for _, mib := range []uint64{40, 48, 46, 45, 44} { bus.fill["KV_m_drafts"] = mib * uint64(MiB) k.FillPass() steps = append(steps, fmt.Sprintf("%d:%d", mib, len(out.events))) } if fmt.Sprint(steps) != "[40:0 48:1 46:1 45:1 44:2]" || fmt.Sprint(out.events) != "[full m drafts full-cleared m drafts]" || fmt.Sprint(out.conds) != "[raised nats.full.m_drafts cleared nats.full.m_drafts]" { t.Errorf("said %v %v: full at 48 of 64 MiB (75%%), kept at 46 and 45 (71%%, 70%%), cleared at 44 (68%%)", out.events, out.conds) } h := HealthOf(k.Status(), k.Held(), 0, k.now()) if !h.Healthy || h.Pass == "none" { t.Errorf("health %+v: healthy, naming its pass", h) } }}, {"the health is unhealthy past its bound, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { at := k.now() k.FillPass() later := at.Add(4 * time.Minute) if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "last fill pass is 4m0s old") { t.Errorf("health %+v: unhealthy, its pass too old", h) } if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") { t.Errorf("health %+v: names the verdicts waiting", h) } k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") } k.mu.Lock() k.status.FirstView.Answered = time.Time{} k.mu.Unlock() if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second { t.Fatalf("asked again in %s (%v): on the backoff", next, err) } // The first view unanswered is unhealthy only past the bound since the start (review round 1: health.go:41). if h := HealthOf(k.Status(), nil, 0, at); !h.Healthy { t.Errorf("health %+v: healthy within the bound while the first view is asked", h) } k.FillPass() if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") { t.Errorf("health %+v: unhealthy past the bound, naming the verb", h) } // What waits on its consumer unread is never skipped silently (review round 1: health.go:47). if h := HealthOf(k.Status(), nil, -1, at); h.Healthy || !strings.Contains(h.Why, "cannot be read") { t.Errorf("health %+v: unhealthy when its consumer's backlog cannot be read", h) } }}, // CAT-N8: a person's requests: a drop removes an orphan alone, only on the operator's warrant binding exactly // that drop, after a snapshot, said dropped and recorded; a recreate rebuilds a drifted entry the same way. {"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}}) k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}}) var acts []string k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil } ask := oc.Ask{ID: "bus-1", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour))} var w *oc.Warrant k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil } put := func(r Request) Request { raw, _ := json.Marshal(r) _, _ = store.Put(StateRequests, r.ID, raw) k.ActOnRequests() raw, _, _, _ = store.Get(StateRequests, r.ID) var got Request _ = json.Unmarshal(raw, &got) return got } if got := put(Request{ID: "r0", Verb: VerbDrop, Module: "m", Local: "log", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "drop removes an orphan alone") { t.Errorf("dropping an asked entry: %+v", got) } if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !got.Done.IsZero() || len(bus.deleted) != 0 { t.Fatalf("acted with no warrant: %+v, deleted %v", got, bus.deleted) } binds := func(act occ.Act) string { d, _ := occ.ActDigest(act); return d } choose := func(act occ.Act) { ask.Options = []oc.Option{{ID: "go", Label: "Remove", Level: occ.Destroy, Binds: binds(act)}} w = &oc.Warrant{Ask: "bus-1", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator}, Option: "go", Label: "Remove", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)} } choose(occ.Act{"verb": "drop", "module": "m", "local": "log"}) if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "bound") || len(bus.deleted) != 0 { t.Fatalf("acted on a warrant for another act: %+v, deleted %v", got, bus.deleted) } // ADR 0626 §4: the keeper trusts nothing in the request; each of these is refused, said why, and deletes nothing. refused := func(r Request, why string) { t.Helper() if got := put(r); got.Done.IsZero() || !strings.Contains(got.Error, why) || len(bus.deleted) != 0 { t.Errorf("%s: ended %+v, deleted %v: refused for %q", r.ID, got, bus.deleted, why) } } refused(Request{ID: "r1a", Verb: VerbDrop, Module: "m", Local: "drafts", Caller: "person.jo"}, "names no ask") choose(occ.Act{"verb": "drop", "module": "m", "local": "drafts"}) w.Level, w.Label, ask.Options[0].Level = occ.Approve, "Keep", occ.Approve w.AskDigest = occ.Digest(ask) refused(Request{ID: "r1b", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "nothing is removed") choose(occ.Act{"verb": "drop", "module": "m", "local": "drafts"}) w.Asker = "mesh-grants" refused(Request{ID: "r1c", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "no warrant") refused(Request{ID: "r1d", Verb: VerbCreate, Module: "m", Local: "drafts", Caller: "person.jo"}, "no accepted manifest of m asks for drafts") choose(occ.Act{"verb": "drop", "module": "m", "local": "drafts"}) bus.refuse["snapshot"] = errors.New("no space") refused(Request{ID: "r1e", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "the snapshot before drop failed, so nothing was removed") delete(bus.refuse, "snapshot") bus.snaps = nil out.reset() got := put(Request{ID: "r2", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", Why: "unused"}) if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts]" || fmt.Sprint(out.events) != "[dropped m drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_drop: dropped drafts of m") || !strings.Contains(acts[0], "person.jo") { t.Errorf("ended %+v, snapshots %v, deleted %v, said %v, recorded %v", got, bus.snaps, bus.deleted, out.events, acts) } if _, made := k.Held()["m"].Made["drafts"]; made { t.Error("the dropped entry is still held") } }}, {"a recreate rebuilds a drifted entry after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}}) k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})}}) binds, _ := occ.ActDigest(occ.Act{"verb": "recreate", "module": "m", "local": "log"}) ask := oc.Ask{ID: "bus-2", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour)), Options: []oc.Option{{ID: "go", Label: "Rebuild", Level: occ.Destroy, Binds: binds}}} w := &oc.Warrant{Ask: "bus-2", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator}, Option: "go", Label: "Rebuild", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)} k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil } out.reset() raw, _ := json.Marshal(Request{ID: "r3", Verb: VerbRecreate, Module: "m", Local: "log", Ask: "bus-2"}) _, _ = store.Put(StateRequests, "r3", raw) k.ActOnRequests() if bus.held["m_log"].MaxBytes != 16*MiB || fmt.Sprint(out.events) != "[recreated m log]" || fmt.Sprint(out.conds) != "[cleared nats.drifted.m_log]" || k.Held()["m"].Made["log"].Drift != "" { t.Errorf("held %+v, said %v %v: rebuilt as asked, the drift cleared", bus.held["m_log"], out.events, out.conds) } }}, // Review round 1 (security F2; design 25 § 12.3): a warrant acts once, spent by compare-and-set before anything is // deleted; a second request naming the same ask is refused and deletes nothing. {"a warrant acts once", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}}) k.Heard(Accepted{Module: "m", Revision: 2}) binds, _ := occ.ActDigest(occ.Act{"verb": "drop", "module": "m", "local": "drafts"}) ask := oc.Ask{ID: "bus-5", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour)), Options: []oc.Option{{ID: "go", Label: "Remove", Level: occ.Destroy, Binds: binds}}} w := &oc.Warrant{Ask: "bus-5", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator}, Option: "go", Label: "Remove", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)} k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil } put := func(id string) Request { raw, _ := json.Marshal(Request{ID: id, Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-5", Caller: "person.jo"}) _, _ = store.Put(StateRequests, id, raw) k.ActOnRequests() raw, _, _, _ = store.Get(StateRequests, id) var got Request _ = json.Unmarshal(raw, &got) return got } if got := put("r5"); got.Error != "" || len(bus.deleted) != 1 { t.Fatalf("the first drop: %+v, deleted %v", got, bus.deleted) } // The entry is asked for and orphaned again, so a second drop would remove it: the spent warrant refuses it. k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}}) k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{log}}) if got := put("r6"); !strings.Contains(got.Error, "already acted (drop of m's drafts, request r5") || len(bus.deleted) != 1 { t.Errorf("the second request on the same warrant: %+v, deleted %v", got, bus.deleted) } if raw, _, found, _ := store.Get(StateSpent, "bus-5"); !found || !strings.Contains(string(raw), `"request":"r5"`) { t.Errorf("the spend record is %s (%v)", raw, found) } // The spend record is the process's own state, never in requests, which the tools write: every key of requests // gone, a third request on the same warrant is still refused (review round 2, security F2). keys, _ := store.Keys(StateRequests) for _, key := range keys { _ = store.Delete(StateRequests, key) } if got := put("r6b"); !strings.Contains(got.Error, "already acted") || len(bus.deleted) != 1 { t.Errorf("with requests wiped, the spent warrant: %+v, deleted %v", got, bus.deleted) } }}, // An ask that ended unanswered ends its request, said, instead of a request that waits until its key ages out; and a // hand act that cannot be recorded is said on the request's end, never only logged. {"an ask ended unanswered ends its request; an unrecorded hand act is said", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}}) k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}}) k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return nil, nil, nil } get := func(id string) Request { raw, _, _, _ := store.Get(StateRequests, id) var got Request _ = json.Unmarshal(raw, &got) return got } raw, _ := json.Marshal(Request{ID: "r7", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-7", Caller: "person.jo", At: k.now().Add(-time.Hour)}) _, _ = store.Put(StateRequests, "r7", raw) raw, _ = json.Marshal(Request{ID: "r8", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-8", Caller: "person.jo", At: k.now().Add(-AskLasts - time.Minute)}) _, _ = store.Put(StateRequests, "r8", raw) k.ActOnRequests() if got := get("r7"); !got.Done.IsZero() { t.Errorf("a request whose ask still runs ended: %+v", got) } if got := get("r8"); got.Done.IsZero() || !strings.Contains(got.Error, "ended at") || len(bus.deleted) != 0 { t.Errorf("a request whose ask ended unanswered: %+v, deleted %v", got, bus.deleted) } k.Act = func(string, string, string) error { return errors.New("the hand-act seat does not answer") } raw, _ = json.Marshal(Request{ID: "r9", Verb: VerbCreate, Module: "m", Local: "log", Caller: "person.jo"}) _, _ = store.Put(StateRequests, "r9", raw) k.ActOnRequests() if got := get("r9"); got.Done.IsZero() || !strings.Contains(got.Unrecorded, "does not answer") { t.Errorf("an unrecorded hand act: %+v", got) } }}, // Review round 1 (health F1): created is marked only once said, so the grants part always gets its binding; an // event or a condition the bus did not take is kept, named by the health, and said again at the next fill pass // with the same id. {"what the bus did not take is said again; created marked only once said", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { out.down = true k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, entry("raw", catalogue.StorageStream, "", nil)}}) if m := k.Held()["m"].Made["drafts"]; m.Created || !strings.Contains(m.Waiting, "its created could not be said yet") { t.Fatalf("made %+v: not created until said, waiting", m) } st := k.Status() if len(st.Unsaid) != 1 || st.Unsaid[0].Event != EventRefused || len(st.UnsaidConditions) != 1 { t.Fatalf("unsaid %+v %+v: the refused event and its condition kept", st.Unsaid, st.UnsaidConditions) } if h := HealthOf(k.Status(), nil, 0, k.now()); h.Healthy || !strings.Contains(h.Why, "2 word(s) the bus did not take") { t.Errorf("health %+v: unhealthy, naming what was not said", h) } id := st.Unsaid[0].ID out.down = false k.FillPass() k.CheckAll() if !strings.Contains(fmt.Sprint(out.events), "refused m raw") || !strings.Contains(fmt.Sprint(out.events), "created m drafts") || fmt.Sprint(out.conds) != "[raised nats.refused.m_raw]" || out.ids[0] != id { t.Errorf("said %v (ids %v) %v: the refused said again with its id %s, its condition raised, created said", out.events, out.ids, out.conds, id) } if st := k.Status(); len(st.Unsaid) != 0 || len(st.UnsaidConditions) != 0 || !k.Held()["m"].Made["drafts"].Created { t.Errorf("after the pass: unsaid %+v %+v, drafts %+v", st.Unsaid, st.UnsaidConditions, k.Held()["m"].Made["drafts"]) } }}, // Review round 2 (health note 3): created said on the seat's subject and refused by its history is created, and its // history is said again at the fill pass with the same id, on the history alone. {"created said on the seat and not its history is said again on what is left", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { out.partly = true k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) st := k.Status() if m := k.Held()["m"].Made["drafts"]; !m.Created || len(st.Unsaid) != 1 || fmt.Sprint(st.Unsaid[0].Left) != "[history]" { t.Fatalf("made %+v, unsaid %+v: created, its history kept to say", m, st.Unsaid) } id := st.Unsaid[0].ID out.partly = false k.FillPass() if last := out.left[len(out.left)-1]; fmt.Sprint(last) != "[history]" || out.ids[len(out.ids)-1] != id || len(k.Status().Unsaid) != 0 { t.Errorf("said again on %v with id %s (want [history], %s); unsaid %+v", last, out.ids[len(out.ids)-1], id, k.Status().Unsaid) } if n := strings.Count(fmt.Sprint(out.ids), id); n != 2 { t.Errorf("the id %s was used %d times: once on the seat, once on the history", id, n) } }}, // Review round 2 (seams note 3, health note 4): words unsaid with a status that cannot be written raise the keeper's // own condition, once, cleared when the status is written again; the health names what a restart would lose. {"a status that cannot be written with words unsaid is said", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) k.FillPass() store.refuse[StateAsked+"/"+KeyStatus] = errors.New("the bucket is full") k.Emit = func(e *Saying) error { e.Left = []string{"seat", "history"}; return errors.New("no responders") } k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, entry("raw", catalogue.StorageStream, "", nil)}}) k.FillPass() raisedN := strings.Count(fmt.Sprint(out.conds), "raised nats.status-unwritten.nats_keeper") if raisedN != 1 { t.Errorf("said %v: status-unwritten raised once", out.conds) } if h := HealthOf(k.Status(), nil, 0, k.now().Add(4*time.Minute)); !strings.Contains(h.Why, "may be lost at a restart") { t.Errorf("health %+v: names what a restart loses", h) } delete(store.refuse, StateAsked+"/"+KeyStatus) k.Emit = out.emit k.FillPass() if !strings.Contains(fmt.Sprint(out.conds), "cleared nats.status-unwritten.nats_keeper") { t.Errorf("said %v: cleared once the status is written", out.conds) } }}, // CAT-N2b: nats_first_view_again asks the first view at once, by hand, recorded as a hand act. {"the first view asked again by hand", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) var acts []string k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil } k.View = func() (FirstView, error) { return FirstView{Revision: 8, Modules: []Accepted{{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}, nil } raw, _ := json.Marshal(Request{ID: "r9", Verb: VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"}) _, _ = store.Put(StateRequests, "r9", raw) k.ActOnRequests() raw, _, _, _ = store.Get(StateRequests, "r9") var got Request _ = json.Unmarshal(raw, &got) if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked mesh-delivery-indexer.indexer_bus_state again by hand") { t.Errorf("ended %+v, created %v, recorded %v", got, bus.created, acts) } }}, } { t.Run(row.name, func(t *testing.T) { k, bus, _, out := newKeeper(FirstView{Revision: 1}) if _, err := k.AskFirstView(0); err != nil { t.Fatal(err) } row.run(t, k, bus, out) }) } }