jschoubben
f735392cb9
Review round 1 of the registry milestone (design 25 § 12.2, § 12.3, § 12.7): - the fill pass runs from the start, while the first view is asked; - recreate and drop wake on the operator channel's decided on the keeper's durable consumer, never a minute's tick; - a warrant is spent by compare-and-set (acted.<ask id> in requests) before anything is deleted; a second request on it is refused; a request whose ask ended unanswered is ended, said; the ask is raised before the request is handed; - created is marked only once said; an event or a condition the bus did not take is kept, named by the health and said again at the next fill pass with its one id, on the subjects left (the double publish); - the health judges the first view past its bound and never skips an unread consumer backlog; - an unrecordable hand act is said on the request; nats_requests reads what waits, what ended and each warrant spent, and drops a waiting request by hand; a tool waits for its request's end on one watch of requests; - the whole-module test reads the warrant from the router's record under a process connection. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
698 lines
36 KiB
Go
698 lines
36 KiB
Go
package keeper
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
|
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
)
|
|
|
|
// fakeBus holds objects by stream, as a bus would, and refuses what a test tells it to.
|
|
type fakeBus struct {
|
|
mu sync.Mutex
|
|
held map[string]Settings
|
|
fill map[string]uint64
|
|
refuse map[string]error
|
|
created []string
|
|
updated []string
|
|
deleted []string
|
|
snaps [][]string
|
|
}
|
|
|
|
func newFakeBus() *fakeBus {
|
|
return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}}
|
|
}
|
|
|
|
func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
s, ok := b.held[o.Stream()]
|
|
return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil
|
|
}
|
|
|
|
func (b *fakeBus) Create(_ context.Context, o Object) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
if err := b.refuse[o.Stream()]; err != nil {
|
|
return err
|
|
}
|
|
b.held[o.Stream()] = o.Settings
|
|
b.created = append(b.created, o.Stream())
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Update(_ context.Context, o Object) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.held[o.Stream()] = o.Settings
|
|
b.updated = append(b.updated, o.Stream())
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Delete(_ context.Context, o Object) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
delete(b.held, o.Stream())
|
|
b.deleted = append(b.deleted, o.Stream())
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Snapshot(_ context.Context, streams []string) (string, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.snaps = append(b.snaps, streams)
|
|
if err := b.refuse["snapshot"]; err != nil {
|
|
return "", err
|
|
}
|
|
return "/snapshots/" + strings.Join(streams, "+") + ".tar", nil
|
|
}
|
|
|
|
// memStore is a key-value state in memory.
|
|
type memStore struct {
|
|
mu sync.Mutex
|
|
kv map[string][]byte
|
|
revs map[string]uint64
|
|
rev uint64
|
|
}
|
|
|
|
func newMemStore() *memStore { return &memStore{kv: map[string][]byte{}, revs: map[string]uint64{}} }
|
|
|
|
func (s *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
v, ok := s.kv[state+"/"+key]
|
|
return v, s.revs[state+"/"+key], ok, nil
|
|
}
|
|
|
|
func (s *memStore) Put(state, key string, value []byte) (uint64, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.rev++
|
|
s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev
|
|
return s.rev, nil
|
|
}
|
|
|
|
func (s *memStore) Create(state, key string, value []byte) (uint64, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if _, ok := s.kv[state+"/"+key]; ok {
|
|
return 0, ErrExists
|
|
}
|
|
s.rev++
|
|
s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev
|
|
return s.rev, nil
|
|
}
|
|
|
|
func (s *memStore) Delete(state, key string) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
delete(s.kv, state+"/"+key)
|
|
return nil
|
|
}
|
|
|
|
func (s *memStore) Keys(state string) ([]string, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
var out []string
|
|
for k := range s.kv {
|
|
if st, key, _ := strings.Cut(k, "/"); st == state {
|
|
out = append(out, key)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out, nil
|
|
}
|
|
|
|
// said is what a keeper said: its events and its conditions, in order.
|
|
type said struct {
|
|
mu sync.Mutex
|
|
events []string
|
|
bodies []Event
|
|
ids []string
|
|
conds []string
|
|
// down refuses every event and condition while set, as a bus that does not take them.
|
|
down bool
|
|
}
|
|
|
|
func (s *said) emit(e *Saying) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.down {
|
|
e.Left = []string{"seat", "history"}
|
|
return errors.New("no responders")
|
|
}
|
|
s.events = append(s.events, e.Event+" "+e.Body.Module+" "+e.Body.Local)
|
|
s.bodies = append(s.bodies, e.Body)
|
|
s.ids = append(s.ids, e.ID)
|
|
e.Left = nil
|
|
return nil
|
|
}
|
|
|
|
func (s *said) say(c oc.Condition) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.down {
|
|
return errors.New("no responders")
|
|
}
|
|
s.conds = append(s.conds, c.State+" "+c.Key)
|
|
return nil
|
|
}
|
|
|
|
func (s *said) reset() {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.events, s.bodies, s.conds = nil, nil, nil
|
|
}
|
|
|
|
func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) {
|
|
bus, store, out := newFakeBus(), newMemStore(), &said{}
|
|
at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)
|
|
k := &Keeper{Bus: bus, Store: store, Emit: out.emit, Say: out.say,
|
|
View: func() (FirstView, error) { return view, nil },
|
|
Now: func() time.Time { return at }}
|
|
return k, bus, store, out
|
|
}
|
|
|
|
func entry(name, storage, template string, options map[string]string) catalogue.BusStateEntry {
|
|
e := catalogue.BusStateEntry{Name: name, Storage: storage, Template: template}
|
|
if len(options) > 0 {
|
|
e.Options = map[string]json.RawMessage{}
|
|
for k, v := range options {
|
|
e.Options[k] = json.RawMessage(v)
|
|
}
|
|
}
|
|
return e
|
|
}
|
|
|
|
// What one entry is on the bus (design 25 § 12.1, CAT-N3): each template's objects, named `<module>_<local>`, its
|
|
// binding, and the fixed settings no option changes; and the entries the keeper refuses itself, naming why.
|
|
func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) {
|
|
ledger := entry("records", catalogue.StorageLedger, catalogue.TemplateLedger, map[string]string{"max-mib": "16"})
|
|
ledger.Types = []catalogue.LedgerType{{Name: "decision", Indexes: []catalogue.LedgerIndex{{Name: "by-topic", On: []string{"topic"}}}}}
|
|
bare := entry("raw", catalogue.StorageStream, "", nil)
|
|
bare.Configuration = map[string]json.RawMessage{"when-full": json.RawMessage(`"drop-oldest"`), "max-mib": json.RawMessage("8"), "max-age-days": json.RawMessage("3")}
|
|
for _, row := range []struct {
|
|
name string
|
|
entry catalogue.BusStateEntry
|
|
objects string
|
|
binding string
|
|
refused string
|
|
}{
|
|
{name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}),
|
|
objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}",
|
|
binding: "bucket mesh-knowledge_drafts"},
|
|
{name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil),
|
|
objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}",
|
|
binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"},
|
|
{name: "work queue", entry: entry("jobs", catalogue.StorageStream, catalogue.TemplateWorkQueue, map[string]string{"max-mib": "4", "duplicates": "120"}),
|
|
objects: "stream mesh-knowledge_jobs [mesh.bus.mesh-knowledge.jobs.>] {4 MiB, age 0, history 0, new, size 0, work-queue, duplicates 120}",
|
|
binding: "stream mesh-knowledge_jobs on mesh.bus.mesh-knowledge.jobs"},
|
|
{name: "log", entry: entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil),
|
|
objects: "stream mesh-knowledge_log [mesh.bus.mesh-knowledge.log.>] {1024 MiB, age 0, history 0, new, size 262144}",
|
|
binding: "stream mesh-knowledge_log on mesh.bus.mesh-knowledge.log"},
|
|
{name: "objects", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "32"}),
|
|
objects: "object-store mesh-knowledge_files {32 MiB, age 0, history 0, new, size 0}",
|
|
binding: "objects mesh-knowledge_files"},
|
|
{name: "ledger", entry: ledger,
|
|
objects: "stream mesh-knowledge_records [mesh.ledger.mesh-knowledge.records.history.>] {16 MiB, age 0, history 0, new, size 262144}; " +
|
|
"key-value mesh-knowledge_records_state {64 MiB, age 0, history 1, new, size 262144}; " +
|
|
"key-value mesh-knowledge_records_index-by-topic {64 MiB, age 0, history 1, new, size 262144}; " +
|
|
"stream mesh-knowledge_records_snapshot [mesh.ledger.mesh-knowledge.records.snapshot.>] {16 MiB, age 0, history 1, old, size 0}",
|
|
binding: "history mesh-knowledge_records on mesh.ledger.mesh-knowledge.records.history, state mesh-knowledge_records_state, indexes map[by-topic:mesh-knowledge_records_index-by-topic], snapshot mesh.ledger.mesh-knowledge.records.snapshot"},
|
|
{name: "bare stream", entry: bare,
|
|
objects: "stream mesh-knowledge_raw [mesh.bus.mesh-knowledge.raw.>] {8 MiB, age 259200, history 0, old, size 0}",
|
|
binding: "stream mesh-knowledge_raw on mesh.bus.mesh-knowledge.raw"},
|
|
{name: "a local name holding the separator", entry: entry("my_drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil),
|
|
refused: `the local name "my_drafts" is empty or holds "_"`},
|
|
{name: "objects with no cap", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, nil),
|
|
refused: "gives no max-mib"},
|
|
{name: "a bare stream with no when-full", entry: entry("raw", catalogue.StorageStream, "", nil),
|
|
refused: "must give when-full"},
|
|
{name: "a template on the wrong storage type", entry: entry("x", catalogue.StorageKeyValue, catalogue.TemplateLog, nil),
|
|
refused: "is none the bus module makes"},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
p, err := PlanOf("mesh-knowledge", row.entry)
|
|
if row.refused != "" {
|
|
if err == nil || !strings.Contains(err.Error(), row.refused) {
|
|
t.Fatalf("planned %+v (%v): refused, naming %q", p, err, row.refused)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var objs []string
|
|
for _, o := range p.Objects {
|
|
s := o.Settings
|
|
w := fmt.Sprintf("%s %s", o.Kind, o.Name)
|
|
if len(o.Subjects) > 0 {
|
|
w += fmt.Sprintf(" %v", o.Subjects)
|
|
}
|
|
w += fmt.Sprintf(" {%d MiB, age %d, history %d, %s, size %d", s.MaxBytes/MiB, s.MaxAgeSeconds, s.History, s.Discard, s.MaxMsgSize)
|
|
if s.WorkQueue {
|
|
w += fmt.Sprintf(", work-queue, duplicates %d", s.DuplicatesSeconds)
|
|
}
|
|
objs = append(objs, w+"}")
|
|
}
|
|
if got := strings.Join(objs, "; "); got != row.objects {
|
|
t.Errorf("objects\n got %s\nwant %s", got, row.objects)
|
|
}
|
|
b := p.Binding
|
|
var binding string
|
|
switch {
|
|
case b.History != "":
|
|
binding = fmt.Sprintf("history %s on %s, state %s, indexes %v, snapshot %s", b.History, b.Subject, b.State, b.Indexes, b.Snapshot)
|
|
case b.Stream != "":
|
|
binding = fmt.Sprintf("stream %s on %s", b.Stream, b.Subject)
|
|
case b.Bucket != "":
|
|
binding = "bucket " + b.Bucket
|
|
case b.ObjectStore != "":
|
|
binding = "objects " + b.ObjectStore
|
|
}
|
|
if binding != row.binding || b.Local != row.entry.Name || b.Storage != row.entry.Storage {
|
|
t.Errorf("binding %s (%+v), want %s", binding, b, row.binding)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The keeper makes what an accepted verdict asks for, once, and says it (design 25 § 12.2, ADR 0595 §4; CAT-N2, N3):
|
|
// nothing before its first view; a replay not newer than what it holds is not applied; a refusal of its grant waits and
|
|
// is made on the backoff, never said refused; a refusal of the entry is said and raised, and cleared by a later make.
|
|
func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
|
|
drafts := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil)
|
|
log := entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil)
|
|
for _, row := range []struct {
|
|
name string
|
|
run func(t *testing.T, k *Keeper, bus *fakeBus, out *said)
|
|
}{
|
|
{"created once, binding said, nothing before the first view", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.mu.Lock()
|
|
k.viewed = false
|
|
k.mu.Unlock()
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if len(bus.created) != 0 {
|
|
t.Fatalf("created %v before the first view", bus.created)
|
|
}
|
|
if _, err := k.AskFirstView(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want {
|
|
t.Errorf("created %v, want %s", bus.created, want)
|
|
}
|
|
if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want {
|
|
t.Errorf("said %v, want %s", out.events, want)
|
|
}
|
|
if b := out.bodies[1]; b.Binding.Stream != "m_log" || b.Binding.Subject != "mesh.bus.m.log" || b.Revision != 3 || b.Asked["m_log"].MaxBytes == 0 {
|
|
t.Errorf("created carries %+v: the binding, the revision and the settings asked", b)
|
|
}
|
|
// In the field names of the seat mesh-broker's message said (mesh-contracts), beside binding and revision.
|
|
raw, _ := json.Marshal(out.bodies[1])
|
|
for _, w := range []string{`"module":"m"`, `"local-name":"log"`, `"storage-type":"stream"`, `"template":"log"`, `"bus-name":"m_log"`, `"binding":{`, `"revision":3`} {
|
|
if !strings.Contains(string(raw), w) {
|
|
t.Errorf("the body %s lacks %s", raw, w)
|
|
}
|
|
}
|
|
}},
|
|
{"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}})
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil})
|
|
if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
|
|
t.Errorf("held %+v: the replay at 4 took back what 9 asked", a)
|
|
}
|
|
}},
|
|
{"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
bus.refuse["m_log"] = fmt.Errorf("%w: permissions violation", ErrRetry)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
if w := k.Waiting(); len(w) != 1 || len(out.events) != 0 {
|
|
t.Fatalf("waiting %v, said %v: waits, says nothing", w, out.events)
|
|
}
|
|
delete(bus.refuse, "m_log")
|
|
k.CheckAll()
|
|
if len(k.Waiting()) != 0 || fmt.Sprint(out.events) != "[created m log]" {
|
|
t.Errorf("waiting %v, said %v: made at the next check", k.Waiting(), out.events)
|
|
}
|
|
}},
|
|
{"a refused create is said and raised, cleared by a later create", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
bus.refuse["KV_m_drafts"] = errors.New("insufficient resources")
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if fmt.Sprint(out.events) != "[refused m drafts]" || fmt.Sprint(out.conds) != "[raised nats.refused.m_drafts]" ||
|
|
!strings.Contains(out.bodies[0].Why, "insufficient resources") {
|
|
t.Fatalf("said %v %v %+v", out.events, out.conds, out.bodies)
|
|
}
|
|
delete(bus.refuse, "KV_m_drafts")
|
|
out.reset()
|
|
k.CheckAll()
|
|
if fmt.Sprint(out.events) != "[created m drafts]" || fmt.Sprint(out.conds) != "[cleared nats.refused.m_drafts]" {
|
|
t.Errorf("said %v %v: created, the refusal cleared", out.events, out.conds)
|
|
}
|
|
}},
|
|
{"an entry the keeper cannot plan is refused, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{entry("a_b", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
|
|
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[refused m a_b]" {
|
|
t.Errorf("created %v, said %v", bus.created, out.events)
|
|
}
|
|
}},
|
|
// CAT-N4: check and drift (ADR 0595 §5, §6): a difference that only grows is applied, said by nothing; any other
|
|
// is said and raised, never fixed quietly, and cleared once the bus holds what is asked again.
|
|
{"a held entry that grows is applied without a person", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
out.reset()
|
|
bigger := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "2048"})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{bigger}})
|
|
if fmt.Sprint(bus.updated) != "[m_log]" || bus.held["m_log"].MaxBytes != 2048*MiB || len(out.events)+len(out.conds) != 0 {
|
|
t.Errorf("updated %v to %d, said %v %v: grown quietly", bus.updated, bus.held["m_log"].MaxBytes, out.events, out.conds)
|
|
}
|
|
}},
|
|
{"a held entry that would shrink is a drift, said, raised and cleared", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
out.reset()
|
|
smaller := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{smaller}})
|
|
if len(bus.updated) != 0 || fmt.Sprint(out.events) != "[drifted m log]" || fmt.Sprint(out.conds) != "[raised nats.drifted.m_log]" ||
|
|
!strings.Contains(out.bodies[0].Why, "max-bytes asked 16777216, held 1073741824") {
|
|
t.Fatalf("updated %v, said %v %v %+v: the drift said, nothing changed", bus.updated, out.events, out.conds, out.bodies)
|
|
}
|
|
if l := k.CheckAll(); len(l) != 1 || l[0].Drift == "" || len(out.events) != 1 {
|
|
t.Errorf("check answers %+v and said %v: the drift once", l, out.events)
|
|
}
|
|
k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{log}})
|
|
if fmt.Sprint(out.events) != "[drifted m log drift-cleared m log]" || out.conds[len(out.conds)-1] != "cleared nats.drifted.m_log" {
|
|
t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds)
|
|
}
|
|
}},
|
|
{"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
p, _ := PlanOf("m", drafts)
|
|
bus.held["KV_m_drafts"] = p.Objects[0].Settings
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" {
|
|
t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events)
|
|
}
|
|
}},
|
|
// CAT-N5: orphans (design 25 § 12.2-12.3): an empty manifest-accepted asks nothing now, so what was made is an
|
|
// orphan, said and raised, kept until a drop; asked again it is kept and its binding said again; a module missing
|
|
// from the first view (set aside to wait) keeps its key; one never made leaves nothing behind.
|
|
{"an empty verdict orphans what was made, and asking again keeps it", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
out.reset()
|
|
k.Heard(Accepted{Module: "m", Revision: 2})
|
|
a, held := k.Held()["m"]
|
|
if !held || !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 || fmt.Sprint(out.events) != "[orphaned m drafts]" ||
|
|
fmt.Sprint(out.conds) != "[raised nats.orphaned.m_drafts]" {
|
|
t.Fatalf("held %v %+v, deleted %v, said %v %v: orphaned, kept, raised", held, a, bus.deleted, out.events, out.conds)
|
|
}
|
|
k.Heard(Accepted{Module: "m", Revision: 3})
|
|
k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if fmt.Sprint(out.events) != "[orphaned m drafts created m drafts]" || out.conds[len(out.conds)-1] != "cleared nats.orphaned.m_drafts" {
|
|
t.Errorf("said %v %v: orphaned once, then created again and cleared", out.events, out.conds)
|
|
}
|
|
}},
|
|
{"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil }
|
|
if _, err := k.AskFirstView(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
|
|
t.Errorf("held %+v: kept as asked, never orphaned by a first view", a)
|
|
}
|
|
bus.refuse["KV_n_x"] = errors.New("no")
|
|
k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
|
|
k.Heard(Accepted{Module: "n", Revision: 2})
|
|
if _, held := k.Held()["n"]; held {
|
|
t.Errorf("n's key is still held: nothing of it was made, so its empty verdict ends it")
|
|
}
|
|
}},
|
|
// CAT-N6: fill (ADR 0595 §6): an entry that refuses when full raises `full` at the raise mark and clears only
|
|
// below the clear mark; one that drops its oldest is never read; the health names the pass.
|
|
{"full at the raise mark, cleared only below the clear mark", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
events := entry("events", catalogue.StorageStream, catalogue.TemplateEvents, map[string]string{"max-mib": "1"})
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, events}})
|
|
out.reset()
|
|
bus.fill["m_events"] = 1 * uint64(MiB)
|
|
var steps []string
|
|
for _, mib := range []uint64{40, 48, 46, 45, 44} {
|
|
bus.fill["KV_m_drafts"] = mib * uint64(MiB)
|
|
k.FillPass()
|
|
steps = append(steps, fmt.Sprintf("%d:%d", mib, len(out.events)))
|
|
}
|
|
if fmt.Sprint(steps) != "[40:0 48:1 46:1 45:1 44:2]" || fmt.Sprint(out.events) != "[full m drafts full-cleared m drafts]" ||
|
|
fmt.Sprint(out.conds) != "[raised nats.full.m_drafts cleared nats.full.m_drafts]" {
|
|
t.Errorf("said %v %v: full at 48 of 64 MiB (75%%), kept at 46 and 45 (71%%, 70%%), cleared at 44 (68%%)", out.events, out.conds)
|
|
}
|
|
h := HealthOf(k.Status(), k.Held(), 0, k.now())
|
|
if !h.Healthy || h.Pass == "none" {
|
|
t.Errorf("health %+v: healthy, naming its pass", h)
|
|
}
|
|
}},
|
|
{"the health is unhealthy past its bound, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
at := k.now()
|
|
k.FillPass()
|
|
later := at.Add(4 * time.Minute)
|
|
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "last fill pass is 4m0s old") {
|
|
t.Errorf("health %+v: unhealthy, its pass too old", h)
|
|
}
|
|
if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") {
|
|
t.Errorf("health %+v: names the verdicts waiting", h)
|
|
}
|
|
k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") }
|
|
k.mu.Lock()
|
|
k.status.FirstView.Answered = time.Time{}
|
|
k.mu.Unlock()
|
|
if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second {
|
|
t.Fatalf("asked again in %s (%v): on the backoff", next, err)
|
|
}
|
|
// The first view unanswered is unhealthy only past the bound since the start (review round 1: health.go:41).
|
|
if h := HealthOf(k.Status(), nil, 0, at); !h.Healthy {
|
|
t.Errorf("health %+v: healthy within the bound while the first view is asked", h)
|
|
}
|
|
k.FillPass()
|
|
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") {
|
|
t.Errorf("health %+v: unhealthy past the bound, naming the verb", h)
|
|
}
|
|
// What waits on its consumer unread is never skipped silently (review round 1: health.go:47).
|
|
if h := HealthOf(k.Status(), nil, -1, at); h.Healthy || !strings.Contains(h.Why, "cannot be read") {
|
|
t.Errorf("health %+v: unhealthy when its consumer's backlog cannot be read", h)
|
|
}
|
|
}},
|
|
// CAT-N8: a person's requests: a drop removes an orphan alone, only on the operator's warrant binding exactly
|
|
// that drop, after a snapshot, said dropped and recorded; a recreate rebuilds a drifted entry the same way.
|
|
{"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}})
|
|
var acts []string
|
|
k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
|
|
ask := oc.Ask{ID: "bus-1", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour))}
|
|
var w *oc.Warrant
|
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil }
|
|
put := func(r Request) Request {
|
|
raw, _ := json.Marshal(r)
|
|
_, _ = store.Put(StateRequests, r.ID, raw)
|
|
k.ActOnRequests()
|
|
raw, _, _, _ = store.Get(StateRequests, r.ID)
|
|
var got Request
|
|
_ = json.Unmarshal(raw, &got)
|
|
return got
|
|
}
|
|
if got := put(Request{ID: "r0", Verb: VerbDrop, Module: "m", Local: "log", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "drop removes an orphan alone") {
|
|
t.Errorf("dropping an asked entry: %+v", got)
|
|
}
|
|
if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !got.Done.IsZero() || len(bus.deleted) != 0 {
|
|
t.Fatalf("acted with no warrant: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
binds := func(act occ.Act) string { d, _ := occ.ActDigest(act); return d }
|
|
choose := func(act occ.Act) {
|
|
ask.Options = []oc.Option{{ID: "go", Label: "Remove", Level: occ.Destroy, Binds: binds(act)}}
|
|
w = &oc.Warrant{Ask: "bus-1", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator},
|
|
Option: "go", Label: "Remove", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)}
|
|
}
|
|
choose(occ.Act{"verb": "drop", "module": "m", "local": "log"})
|
|
if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "bound") || len(bus.deleted) != 0 {
|
|
t.Fatalf("acted on a warrant for another act: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
// ADR 0626 §4: the keeper trusts nothing in the request; each of these is refused, said why, and deletes nothing.
|
|
refused := func(r Request, why string) {
|
|
t.Helper()
|
|
if got := put(r); got.Done.IsZero() || !strings.Contains(got.Error, why) || len(bus.deleted) != 0 {
|
|
t.Errorf("%s: ended %+v, deleted %v: refused for %q", r.ID, got, bus.deleted, why)
|
|
}
|
|
}
|
|
refused(Request{ID: "r1a", Verb: VerbDrop, Module: "m", Local: "drafts", Caller: "person.jo"}, "names no ask")
|
|
choose(occ.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
|
w.Level, w.Label, ask.Options[0].Level = occ.Approve, "Keep", occ.Approve
|
|
w.AskDigest = occ.Digest(ask)
|
|
refused(Request{ID: "r1b", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "nothing is removed")
|
|
choose(occ.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
|
w.Asker = "mesh-grants"
|
|
refused(Request{ID: "r1c", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "no warrant")
|
|
refused(Request{ID: "r1d", Verb: VerbCreate, Module: "m", Local: "drafts", Caller: "person.jo"}, "no accepted manifest of m asks for drafts")
|
|
choose(occ.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
|
bus.refuse["snapshot"] = errors.New("no space")
|
|
refused(Request{ID: "r1e", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "the snapshot before drop failed, so nothing was removed")
|
|
delete(bus.refuse, "snapshot")
|
|
bus.snaps = nil
|
|
out.reset()
|
|
got := put(Request{ID: "r2", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", Why: "unused"})
|
|
if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts]" ||
|
|
fmt.Sprint(out.events) != "[dropped m drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_drop: dropped drafts of m") ||
|
|
!strings.Contains(acts[0], "person.jo") {
|
|
t.Errorf("ended %+v, snapshots %v, deleted %v, said %v, recorded %v", got, bus.snaps, bus.deleted, out.events, acts)
|
|
}
|
|
if _, made := k.Held()["m"].Made["drafts"]; made {
|
|
t.Error("the dropped entry is still held")
|
|
}
|
|
}},
|
|
{"a recreate rebuilds a drifted entry after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})}})
|
|
binds, _ := occ.ActDigest(occ.Act{"verb": "recreate", "module": "m", "local": "log"})
|
|
ask := oc.Ask{ID: "bus-2", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour)), Options: []oc.Option{{ID: "go", Label: "Rebuild", Level: occ.Destroy, Binds: binds}}}
|
|
w := &oc.Warrant{Ask: "bus-2", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator},
|
|
Option: "go", Label: "Rebuild", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)}
|
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil }
|
|
out.reset()
|
|
raw, _ := json.Marshal(Request{ID: "r3", Verb: VerbRecreate, Module: "m", Local: "log", Ask: "bus-2"})
|
|
_, _ = store.Put(StateRequests, "r3", raw)
|
|
k.ActOnRequests()
|
|
if bus.held["m_log"].MaxBytes != 16*MiB || fmt.Sprint(out.events) != "[recreated m log]" ||
|
|
fmt.Sprint(out.conds) != "[cleared nats.drifted.m_log]" || k.Held()["m"].Made["log"].Drift != "" {
|
|
t.Errorf("held %+v, said %v %v: rebuilt as asked, the drift cleared", bus.held["m_log"], out.events, out.conds)
|
|
}
|
|
}},
|
|
// Review round 1 (security F2; design 25 § 12.3): a warrant acts once, spent by compare-and-set before anything is
|
|
// deleted; a second request naming the same ask is refused and deletes nothing.
|
|
{"a warrant acts once", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2})
|
|
binds, _ := occ.ActDigest(occ.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
|
ask := oc.Ask{ID: "bus-5", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour)), Options: []oc.Option{{ID: "go", Label: "Remove", Level: occ.Destroy, Binds: binds}}}
|
|
w := &oc.Warrant{Ask: "bus-5", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator},
|
|
Option: "go", Label: "Remove", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)}
|
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil }
|
|
put := func(id string) Request {
|
|
raw, _ := json.Marshal(Request{ID: id, Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-5", Caller: "person.jo"})
|
|
_, _ = store.Put(StateRequests, id, raw)
|
|
k.ActOnRequests()
|
|
raw, _, _, _ = store.Get(StateRequests, id)
|
|
var got Request
|
|
_ = json.Unmarshal(raw, &got)
|
|
return got
|
|
}
|
|
if got := put("r5"); got.Error != "" || len(bus.deleted) != 1 {
|
|
t.Fatalf("the first drop: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
// The entry is asked for and orphaned again, so a second drop would remove it: the spent warrant refuses it.
|
|
k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{log}})
|
|
if got := put("r6"); !strings.Contains(got.Error, "already acted (drop of m's drafts, request r5") || len(bus.deleted) != 1 {
|
|
t.Errorf("the second request on the same warrant: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
if raw, _, found, _ := store.Get(StateRequests, ActedKey("bus-5")); !found || !strings.Contains(string(raw), `"request":"r5"`) {
|
|
t.Errorf("the acted key is %s (%v)", raw, found)
|
|
}
|
|
}},
|
|
// An ask that ended unanswered ends its request, said, instead of a request that waits until its key ages out; and a
|
|
// hand act that cannot be recorded is said on the request's end, never only logged.
|
|
{"an ask ended unanswered ends its request; an unrecorded hand act is said", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}})
|
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return nil, nil, nil }
|
|
get := func(id string) Request {
|
|
raw, _, _, _ := store.Get(StateRequests, id)
|
|
var got Request
|
|
_ = json.Unmarshal(raw, &got)
|
|
return got
|
|
}
|
|
raw, _ := json.Marshal(Request{ID: "r7", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-7", Caller: "person.jo", At: k.now().Add(-time.Hour)})
|
|
_, _ = store.Put(StateRequests, "r7", raw)
|
|
raw, _ = json.Marshal(Request{ID: "r8", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-8", Caller: "person.jo", At: k.now().Add(-AskLasts - time.Minute)})
|
|
_, _ = store.Put(StateRequests, "r8", raw)
|
|
k.ActOnRequests()
|
|
if got := get("r7"); !got.Done.IsZero() {
|
|
t.Errorf("a request whose ask still runs ended: %+v", got)
|
|
}
|
|
if got := get("r8"); got.Done.IsZero() || !strings.Contains(got.Error, "ended at") || len(bus.deleted) != 0 {
|
|
t.Errorf("a request whose ask ended unanswered: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
k.Act = func(string, string, string) error { return errors.New("the hand-act seat does not answer") }
|
|
raw, _ = json.Marshal(Request{ID: "r9", Verb: VerbCreate, Module: "m", Local: "log", Caller: "person.jo"})
|
|
_, _ = store.Put(StateRequests, "r9", raw)
|
|
k.ActOnRequests()
|
|
if got := get("r9"); got.Done.IsZero() || !strings.Contains(got.Unrecorded, "does not answer") {
|
|
t.Errorf("an unrecorded hand act: %+v", got)
|
|
}
|
|
}},
|
|
// Review round 1 (health F1): created is marked only once said, so the grants part always gets its binding; an
|
|
// event or a condition the bus did not take is kept, named by the health, and said again at the next fill pass
|
|
// with the same id.
|
|
{"what the bus did not take is said again; created marked only once said", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
out.down = true
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, entry("raw", catalogue.StorageStream, "", nil)}})
|
|
if m := k.Held()["m"].Made["drafts"]; m.Created || !strings.Contains(m.Waiting, "its created could not be said yet") {
|
|
t.Fatalf("made %+v: not created until said, waiting", m)
|
|
}
|
|
st := k.Status()
|
|
if len(st.Unsaid) != 1 || st.Unsaid[0].Event != EventRefused || len(st.UnsaidConditions) != 1 {
|
|
t.Fatalf("unsaid %+v %+v: the refused event and its condition kept", st.Unsaid, st.UnsaidConditions)
|
|
}
|
|
if h := HealthOf(k.Status(), nil, 0, k.now()); h.Healthy || !strings.Contains(h.Why, "2 word(s) the bus did not take") {
|
|
t.Errorf("health %+v: unhealthy, naming what was not said", h)
|
|
}
|
|
id := st.Unsaid[0].ID
|
|
out.down = false
|
|
k.FillPass()
|
|
k.CheckAll()
|
|
if !strings.Contains(fmt.Sprint(out.events), "refused m raw") || !strings.Contains(fmt.Sprint(out.events), "created m drafts") ||
|
|
fmt.Sprint(out.conds) != "[raised nats.refused.m_raw]" || out.ids[0] != id {
|
|
t.Errorf("said %v (ids %v) %v: the refused said again with its id %s, its condition raised, created said", out.events, out.ids, out.conds, id)
|
|
}
|
|
if st := k.Status(); len(st.Unsaid) != 0 || len(st.UnsaidConditions) != 0 || !k.Held()["m"].Made["drafts"].Created {
|
|
t.Errorf("after the pass: unsaid %+v %+v, drafts %+v", st.Unsaid, st.UnsaidConditions, k.Held()["m"].Made["drafts"])
|
|
}
|
|
}},
|
|
// CAT-N2b: nats_first_view_again asks the first view at once, by hand, recorded as a hand act.
|
|
{"the first view asked again by hand", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
var acts []string
|
|
k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
|
|
k.View = func() (FirstView, error) {
|
|
return FirstView{Revision: 8, Modules: []Accepted{{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}, nil
|
|
}
|
|
raw, _ := json.Marshal(Request{ID: "r9", Verb: VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"})
|
|
_, _ = store.Put(StateRequests, "r9", raw)
|
|
k.ActOnRequests()
|
|
raw, _, _, _ = store.Get(StateRequests, "r9")
|
|
var got Request
|
|
_ = json.Unmarshal(raw, &got)
|
|
if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts]" ||
|
|
len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked mesh-delivery-indexer.indexer_bus_state again by hand") {
|
|
t.Errorf("ended %+v, created %v, recorded %v", got, bus.created, acts)
|
|
}
|
|
}},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
k, bus, _, out := newKeeper(FirstView{Revision: 1})
|
|
if _, err := k.AskFirstView(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
row.run(t, k, bus, out)
|
|
})
|
|
}
|
|
}
|