jschoubben
3b61a05532
The bus module's tools hold no grant of the seat's acting verbs (design 25 § 12.3): nats_create, nats_apply, nats_recreate and nats_drop write a request into the keeper's own state requests, stamped with stdio.CallerOf (a name in the arguments is never the caller), and read its end back. recreate and drop raise the operator's ask first, its option binding exactly that act; the keeper acts only on the router's warrant for it (Warrant.For, Option.Performs), after a snapshot of the entry's streams (bussnap, moved from the snapshot command with a streams filter), says recreated or dropped, and records every act as a hand act through mesh-hand-acts. A drop removes an orphan alone. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
523 lines
25 KiB
Go
523 lines
25 KiB
Go
package keeper
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
)
|
|
|
|
// fakeBus holds objects by stream, as a bus would, and refuses what a test tells it to.
|
|
type fakeBus struct {
|
|
mu sync.Mutex
|
|
held map[string]Settings
|
|
fill map[string]uint64
|
|
refuse map[string]error
|
|
created []string
|
|
updated []string
|
|
deleted []string
|
|
snaps [][]string
|
|
}
|
|
|
|
func newFakeBus() *fakeBus {
|
|
return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}}
|
|
}
|
|
|
|
func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
s, ok := b.held[o.Stream()]
|
|
return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil
|
|
}
|
|
|
|
func (b *fakeBus) Create(_ context.Context, o Object) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
if err := b.refuse[o.Stream()]; err != nil {
|
|
return err
|
|
}
|
|
b.held[o.Stream()] = o.Settings
|
|
b.created = append(b.created, o.Stream())
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Update(_ context.Context, o Object) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.held[o.Stream()] = o.Settings
|
|
b.updated = append(b.updated, o.Stream())
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Delete(_ context.Context, o Object) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
delete(b.held, o.Stream())
|
|
b.deleted = append(b.deleted, o.Stream())
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Snapshot(_ context.Context, streams []string) (string, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.snaps = append(b.snaps, streams)
|
|
return "/snapshots/" + strings.Join(streams, "+") + ".tar", nil
|
|
}
|
|
|
|
// memStore is a key-value state in memory.
|
|
type memStore struct {
|
|
mu sync.Mutex
|
|
kv map[string][]byte
|
|
revs map[string]uint64
|
|
rev uint64
|
|
}
|
|
|
|
func newMemStore() *memStore { return &memStore{kv: map[string][]byte{}, revs: map[string]uint64{}} }
|
|
|
|
func (s *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
v, ok := s.kv[state+"/"+key]
|
|
return v, s.revs[state+"/"+key], ok, nil
|
|
}
|
|
|
|
func (s *memStore) Put(state, key string, value []byte) (uint64, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.rev++
|
|
s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev
|
|
return s.rev, nil
|
|
}
|
|
|
|
func (s *memStore) Delete(state, key string) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
delete(s.kv, state+"/"+key)
|
|
return nil
|
|
}
|
|
|
|
func (s *memStore) Keys(state string) ([]string, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
var out []string
|
|
for k := range s.kv {
|
|
if st, key, _ := strings.Cut(k, "/"); st == state {
|
|
out = append(out, key)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out, nil
|
|
}
|
|
|
|
// said is what a keeper said: its events and its conditions, in order.
|
|
type said struct {
|
|
mu sync.Mutex
|
|
events []string
|
|
bodies []Event
|
|
conds []string
|
|
}
|
|
|
|
func (s *said) emit(event string, body Event) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.events = append(s.events, event+" "+body.Module+" "+body.Local)
|
|
s.bodies = append(s.bodies, body)
|
|
return nil
|
|
}
|
|
|
|
func (s *said) say(c asks.Condition) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.conds = append(s.conds, c.State+" "+c.Key)
|
|
return nil
|
|
}
|
|
|
|
func (s *said) reset() {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.events, s.bodies, s.conds = nil, nil, nil
|
|
}
|
|
|
|
func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) {
|
|
bus, store, out := newFakeBus(), newMemStore(), &said{}
|
|
at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)
|
|
k := &Keeper{Bus: bus, Store: store, Emit: out.emit, Say: out.say,
|
|
View: func() (FirstView, error) { return view, nil },
|
|
Now: func() time.Time { return at }}
|
|
return k, bus, store, out
|
|
}
|
|
|
|
func entry(name, storage, template string, options map[string]string) catalogue.BusStateEntry {
|
|
e := catalogue.BusStateEntry{Name: name, Storage: storage, Template: template}
|
|
if len(options) > 0 {
|
|
e.Options = map[string]json.RawMessage{}
|
|
for k, v := range options {
|
|
e.Options[k] = json.RawMessage(v)
|
|
}
|
|
}
|
|
return e
|
|
}
|
|
|
|
// What one entry is on the bus (design 25 § 12.1, CAT-N3): each template's objects, named `<module>_<local>`, its
|
|
// binding, and the fixed settings no option changes; and the entries the keeper refuses itself, naming why.
|
|
func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) {
|
|
ledger := entry("records", catalogue.StorageLedger, catalogue.TemplateLedger, map[string]string{"max-mib": "16"})
|
|
ledger.Types = []catalogue.LedgerType{{Name: "decision", Indexes: []catalogue.LedgerIndex{{Name: "by-topic", On: []string{"topic"}}}}}
|
|
bare := entry("raw", catalogue.StorageStream, "", nil)
|
|
bare.Configuration = map[string]json.RawMessage{"when-full": json.RawMessage(`"drop-oldest"`), "max-mib": json.RawMessage("8"), "max-age-days": json.RawMessage("3")}
|
|
for _, row := range []struct {
|
|
name string
|
|
entry catalogue.BusStateEntry
|
|
objects string
|
|
binding string
|
|
refused string
|
|
}{
|
|
{name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}),
|
|
objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}",
|
|
binding: "bucket mesh-knowledge_drafts"},
|
|
{name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil),
|
|
objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}",
|
|
binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"},
|
|
{name: "work queue", entry: entry("jobs", catalogue.StorageStream, catalogue.TemplateWorkQueue, map[string]string{"max-mib": "4", "duplicates": "120"}),
|
|
objects: "stream mesh-knowledge_jobs [mesh.bus.mesh-knowledge.jobs.>] {4 MiB, age 0, history 0, new, size 0, work-queue, duplicates 120}",
|
|
binding: "stream mesh-knowledge_jobs on mesh.bus.mesh-knowledge.jobs"},
|
|
{name: "log", entry: entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil),
|
|
objects: "stream mesh-knowledge_log [mesh.bus.mesh-knowledge.log.>] {1024 MiB, age 0, history 0, new, size 262144}",
|
|
binding: "stream mesh-knowledge_log on mesh.bus.mesh-knowledge.log"},
|
|
{name: "objects", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "32"}),
|
|
objects: "object-store mesh-knowledge_files {32 MiB, age 0, history 0, new, size 0}",
|
|
binding: "objects mesh-knowledge_files"},
|
|
{name: "ledger", entry: ledger,
|
|
objects: "stream mesh-knowledge_records [mesh.ledger.mesh-knowledge.records.history.>] {16 MiB, age 0, history 0, new, size 262144}; " +
|
|
"key-value mesh-knowledge_records_state {64 MiB, age 0, history 1, new, size 262144}; " +
|
|
"key-value mesh-knowledge_records_index-by-topic {64 MiB, age 0, history 1, new, size 262144}; " +
|
|
"stream mesh-knowledge_records_snapshot [mesh.ledger.mesh-knowledge.records.snapshot.>] {16 MiB, age 0, history 1, old, size 0}",
|
|
binding: "history mesh-knowledge_records on mesh.ledger.mesh-knowledge.records.history, state mesh-knowledge_records_state, indexes map[by-topic:mesh-knowledge_records_index-by-topic], snapshot mesh.ledger.mesh-knowledge.records.snapshot"},
|
|
{name: "bare stream", entry: bare,
|
|
objects: "stream mesh-knowledge_raw [mesh.bus.mesh-knowledge.raw.>] {8 MiB, age 259200, history 0, old, size 0}",
|
|
binding: "stream mesh-knowledge_raw on mesh.bus.mesh-knowledge.raw"},
|
|
{name: "a local name holding the separator", entry: entry("my_drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil),
|
|
refused: `the local name "my_drafts" is empty or holds "_"`},
|
|
{name: "objects with no cap", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, nil),
|
|
refused: "gives no max-mib"},
|
|
{name: "a bare stream with no when-full", entry: entry("raw", catalogue.StorageStream, "", nil),
|
|
refused: "must give when-full"},
|
|
{name: "a template on the wrong storage type", entry: entry("x", catalogue.StorageKeyValue, catalogue.TemplateLog, nil),
|
|
refused: "is none the bus module makes"},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
p, err := PlanOf("mesh-knowledge", row.entry)
|
|
if row.refused != "" {
|
|
if err == nil || !strings.Contains(err.Error(), row.refused) {
|
|
t.Fatalf("planned %+v (%v): refused, naming %q", p, err, row.refused)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var objs []string
|
|
for _, o := range p.Objects {
|
|
s := o.Settings
|
|
w := fmt.Sprintf("%s %s", o.Kind, o.Name)
|
|
if len(o.Subjects) > 0 {
|
|
w += fmt.Sprintf(" %v", o.Subjects)
|
|
}
|
|
w += fmt.Sprintf(" {%d MiB, age %d, history %d, %s, size %d", s.MaxBytes/MiB, s.MaxAgeSeconds, s.History, s.Discard, s.MaxMsgSize)
|
|
if s.WorkQueue {
|
|
w += fmt.Sprintf(", work-queue, duplicates %d", s.DuplicatesSeconds)
|
|
}
|
|
objs = append(objs, w+"}")
|
|
}
|
|
if got := strings.Join(objs, "; "); got != row.objects {
|
|
t.Errorf("objects\n got %s\nwant %s", got, row.objects)
|
|
}
|
|
b := p.Binding
|
|
var binding string
|
|
switch {
|
|
case b.History != "":
|
|
binding = fmt.Sprintf("history %s on %s, state %s, indexes %v, snapshot %s", b.History, b.Subject, b.State, b.Indexes, b.Snapshot)
|
|
case b.Stream != "":
|
|
binding = fmt.Sprintf("stream %s on %s", b.Stream, b.Subject)
|
|
case b.Bucket != "":
|
|
binding = "bucket " + b.Bucket
|
|
case b.ObjectStore != "":
|
|
binding = "objects " + b.ObjectStore
|
|
}
|
|
if binding != row.binding || b.Local != row.entry.Name || b.Storage != row.entry.Storage {
|
|
t.Errorf("binding %s (%+v), want %s", binding, b, row.binding)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The keeper makes what an accepted verdict asks for, once, and says it (design 25 § 12.2, ADR 0595 §4; CAT-N2, N3):
|
|
// nothing before its first view; a replay not newer than what it holds is not applied; a refusal of its grant waits and
|
|
// is made on the backoff, never said refused; a refusal of the entry is said and raised, and cleared by a later make.
|
|
func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
|
|
drafts := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil)
|
|
log := entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil)
|
|
for _, row := range []struct {
|
|
name string
|
|
run func(t *testing.T, k *Keeper, bus *fakeBus, out *said)
|
|
}{
|
|
{"created once, binding said, nothing before the first view", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.mu.Lock()
|
|
k.viewed = false
|
|
k.mu.Unlock()
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if len(bus.created) != 0 {
|
|
t.Fatalf("created %v before the first view", bus.created)
|
|
}
|
|
if _, err := k.AskFirstView(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want {
|
|
t.Errorf("created %v, want %s", bus.created, want)
|
|
}
|
|
if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want {
|
|
t.Errorf("said %v, want %s", out.events, want)
|
|
}
|
|
if b := out.bodies[1]; b.Binding.Stream != "m_log" || b.Binding.Subject != "mesh.bus.m.log" || b.Revision != 3 || b.Asked["m_log"].MaxBytes == 0 {
|
|
t.Errorf("created carries %+v: the binding, the revision and the settings asked", b)
|
|
}
|
|
}},
|
|
{"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}})
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil})
|
|
if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
|
|
t.Errorf("held %+v: the replay at 4 took back what 9 asked", a)
|
|
}
|
|
}},
|
|
{"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
bus.refuse["m_log"] = fmt.Errorf("%w: permissions violation", ErrRetry)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
if w := k.Waiting(); len(w) != 1 || len(out.events) != 0 {
|
|
t.Fatalf("waiting %v, said %v: waits, says nothing", w, out.events)
|
|
}
|
|
delete(bus.refuse, "m_log")
|
|
k.CheckAll()
|
|
if len(k.Waiting()) != 0 || fmt.Sprint(out.events) != "[created m log]" {
|
|
t.Errorf("waiting %v, said %v: made at the next check", k.Waiting(), out.events)
|
|
}
|
|
}},
|
|
{"a refused create is said and raised, cleared by a later create", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
bus.refuse["KV_m_drafts"] = errors.New("insufficient resources")
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if fmt.Sprint(out.events) != "[refused m drafts]" || fmt.Sprint(out.conds) != "[raised nats.refused.m_drafts]" ||
|
|
!strings.Contains(out.bodies[0].Why, "insufficient resources") {
|
|
t.Fatalf("said %v %v %+v", out.events, out.conds, out.bodies)
|
|
}
|
|
delete(bus.refuse, "KV_m_drafts")
|
|
out.reset()
|
|
k.CheckAll()
|
|
if fmt.Sprint(out.events) != "[created m drafts]" || fmt.Sprint(out.conds) != "[cleared nats.refused.m_drafts]" {
|
|
t.Errorf("said %v %v: created, the refusal cleared", out.events, out.conds)
|
|
}
|
|
}},
|
|
{"an entry the keeper cannot plan is refused, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{entry("a_b", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
|
|
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[refused m a_b]" {
|
|
t.Errorf("created %v, said %v", bus.created, out.events)
|
|
}
|
|
}},
|
|
// CAT-N4: check and drift (ADR 0595 §5, §6): a difference that only grows is applied, said by nothing; any other
|
|
// is said and raised, never fixed quietly, and cleared once the bus holds what is asked again.
|
|
{"a held entry that grows is applied without a person", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
out.reset()
|
|
bigger := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "2048"})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{bigger}})
|
|
if fmt.Sprint(bus.updated) != "[m_log]" || bus.held["m_log"].MaxBytes != 2048*MiB || len(out.events)+len(out.conds) != 0 {
|
|
t.Errorf("updated %v to %d, said %v %v: grown quietly", bus.updated, bus.held["m_log"].MaxBytes, out.events, out.conds)
|
|
}
|
|
}},
|
|
{"a held entry that would shrink is a drift, said, raised and cleared", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
out.reset()
|
|
smaller := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{smaller}})
|
|
if len(bus.updated) != 0 || fmt.Sprint(out.events) != "[drifted m log]" || fmt.Sprint(out.conds) != "[raised nats.drifted.m_log]" ||
|
|
!strings.Contains(out.bodies[0].Why, "max-bytes asked 16777216, held 1073741824") {
|
|
t.Fatalf("updated %v, said %v %v %+v: the drift said, nothing changed", bus.updated, out.events, out.conds, out.bodies)
|
|
}
|
|
if l := k.CheckAll(); len(l) != 1 || l[0].Drift == "" || len(out.events) != 1 {
|
|
t.Errorf("check answers %+v and said %v: the drift once", l, out.events)
|
|
}
|
|
k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{log}})
|
|
if fmt.Sprint(out.events) != "[drifted m log drift-cleared m log]" || out.conds[len(out.conds)-1] != "cleared nats.drifted.m_log" {
|
|
t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds)
|
|
}
|
|
}},
|
|
{"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
p, _ := PlanOf("m", drafts)
|
|
bus.held["KV_m_drafts"] = p.Objects[0].Settings
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" {
|
|
t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events)
|
|
}
|
|
}},
|
|
// CAT-N5: orphans (design 25 § 12.2-12.3): an empty manifest-accepted asks nothing now, so what was made is an
|
|
// orphan, said and raised, kept until a drop; asked again it is kept and its binding said again; a module missing
|
|
// from the first view (set aside to wait) keeps its key; one never made leaves nothing behind.
|
|
{"an empty verdict orphans what was made, and asking again keeps it", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
out.reset()
|
|
k.Heard(Accepted{Module: "m", Revision: 2})
|
|
a, held := k.Held()["m"]
|
|
if !held || !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 || fmt.Sprint(out.events) != "[orphaned m drafts]" ||
|
|
fmt.Sprint(out.conds) != "[raised nats.orphaned.m_drafts]" {
|
|
t.Fatalf("held %v %+v, deleted %v, said %v %v: orphaned, kept, raised", held, a, bus.deleted, out.events, out.conds)
|
|
}
|
|
k.Heard(Accepted{Module: "m", Revision: 3})
|
|
k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if fmt.Sprint(out.events) != "[orphaned m drafts created m drafts]" || out.conds[len(out.conds)-1] != "cleared nats.orphaned.m_drafts" {
|
|
t.Errorf("said %v %v: orphaned once, then created again and cleared", out.events, out.conds)
|
|
}
|
|
}},
|
|
{"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil }
|
|
if _, err := k.AskFirstView(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
|
|
t.Errorf("held %+v: kept as asked, never orphaned by a first view", a)
|
|
}
|
|
bus.refuse["KV_n_x"] = errors.New("no")
|
|
k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
|
|
k.Heard(Accepted{Module: "n", Revision: 2})
|
|
if _, held := k.Held()["n"]; held {
|
|
t.Errorf("n's key is still held: nothing of it was made, so its empty verdict ends it")
|
|
}
|
|
}},
|
|
// CAT-N6: fill (ADR 0595 §6): an entry that refuses when full raises `full` at the raise mark and clears only
|
|
// below the clear mark; one that drops its oldest is never read; the health names the pass.
|
|
{"full at the raise mark, cleared only below the clear mark", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
events := entry("events", catalogue.StorageStream, catalogue.TemplateEvents, map[string]string{"max-mib": "1"})
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, events}})
|
|
out.reset()
|
|
bus.fill["m_events"] = 1 * uint64(MiB)
|
|
var steps []string
|
|
for _, mib := range []uint64{40, 48, 46, 45, 44} {
|
|
bus.fill["KV_m_drafts"] = mib * uint64(MiB)
|
|
k.FillPass()
|
|
steps = append(steps, fmt.Sprintf("%d:%d", mib, len(out.events)))
|
|
}
|
|
if fmt.Sprint(steps) != "[40:0 48:1 46:1 45:1 44:2]" || fmt.Sprint(out.events) != "[full m drafts full-cleared m drafts]" ||
|
|
fmt.Sprint(out.conds) != "[raised nats.full.m_drafts cleared nats.full.m_drafts]" {
|
|
t.Errorf("said %v %v: full at 48 of 64 MiB (75%%), kept at 46 and 45 (71%%, 70%%), cleared at 44 (68%%)", out.events, out.conds)
|
|
}
|
|
h := HealthOf(k.Status(), k.Held(), 0, k.now())
|
|
if !h.Healthy || h.Pass == "none" {
|
|
t.Errorf("health %+v: healthy, naming its pass", h)
|
|
}
|
|
}},
|
|
{"the health is unhealthy past its bound, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
at := k.now()
|
|
k.FillPass()
|
|
later := at.Add(4 * time.Minute)
|
|
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "last fill pass is 4m0s old") {
|
|
t.Errorf("health %+v: unhealthy, its pass too old", h)
|
|
}
|
|
if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") {
|
|
t.Errorf("health %+v: names the verdicts waiting", h)
|
|
}
|
|
k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") }
|
|
k.mu.Lock()
|
|
k.status.FirstView.Answered = time.Time{}
|
|
k.mu.Unlock()
|
|
if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second {
|
|
t.Fatalf("asked again in %s (%v): on the backoff", next, err)
|
|
}
|
|
if h := HealthOf(k.Status(), nil, 0, at); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") {
|
|
t.Errorf("health %+v: unhealthy, naming the verb", h)
|
|
}
|
|
}},
|
|
// CAT-N8: a person's requests: a drop removes an orphan alone, only on the operator's warrant binding exactly
|
|
// that drop, after a snapshot, said dropped and recorded; a recreate rebuilds a drifted entry the same way.
|
|
{"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}})
|
|
var acts []string
|
|
k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
|
|
ask := asks.Ask{ID: "bus-1", Who: asks.Operator, Expires: k.now().Add(time.Hour)}
|
|
var w *asks.Warrant
|
|
k.Warrant = func(string) (*asks.Ask, *asks.Warrant, error) { return &ask, w, nil }
|
|
put := func(r Request) Request {
|
|
raw, _ := json.Marshal(r)
|
|
_, _ = store.Put(StateRequests, r.ID, raw)
|
|
k.ActOnRequests()
|
|
raw, _, _, _ = store.Get(StateRequests, r.ID)
|
|
var got Request
|
|
_ = json.Unmarshal(raw, &got)
|
|
return got
|
|
}
|
|
if got := put(Request{ID: "r0", Verb: VerbDrop, Module: "m", Local: "log", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "drop removes an orphan alone") {
|
|
t.Errorf("dropping an asked entry: %+v", got)
|
|
}
|
|
if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !got.Done.IsZero() || len(bus.deleted) != 0 {
|
|
t.Fatalf("acted with no warrant: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
binds := func(act asks.Act) string { d, _ := asks.ActDigest(act); return d }
|
|
choose := func(act asks.Act) {
|
|
ask.Options = []asks.Option{{ID: "go", Label: "Remove", Level: asks.Destroy, Binds: binds(act)}}
|
|
w = &asks.Warrant{Ask: "bus-1", Asker: Module, Outcome: asks.OutcomeChosen, By: &asks.Person{Who: asks.Operator},
|
|
Option: "go", Label: "Remove", Level: asks.Destroy, At: k.now(), AskDigest: ask.Digest()}
|
|
}
|
|
choose(asks.Act{"verb": "drop", "module": "m", "local": "log"})
|
|
if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "bound") || len(bus.deleted) != 0 {
|
|
t.Fatalf("acted on a warrant for another act: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
choose(asks.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
|
out.reset()
|
|
got := put(Request{ID: "r2", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", Why: "unused"})
|
|
if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts]" ||
|
|
fmt.Sprint(out.events) != "[dropped m drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_drop: dropped drafts of m") ||
|
|
!strings.Contains(acts[0], "person.jo") {
|
|
t.Errorf("ended %+v, snapshots %v, deleted %v, said %v, recorded %v", got, bus.snaps, bus.deleted, out.events, acts)
|
|
}
|
|
if _, made := k.Held()["m"].Made["drafts"]; made {
|
|
t.Error("the dropped entry is still held")
|
|
}
|
|
}},
|
|
{"a recreate rebuilds a drifted entry after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})}})
|
|
binds, _ := asks.ActDigest(asks.Act{"verb": "recreate", "module": "m", "local": "log"})
|
|
ask := asks.Ask{ID: "bus-2", Who: asks.Operator, Expires: k.now().Add(time.Hour), Options: []asks.Option{{ID: "go", Label: "Rebuild", Level: asks.Destroy, Binds: binds}}}
|
|
w := &asks.Warrant{Ask: "bus-2", Asker: Module, Outcome: asks.OutcomeChosen, By: &asks.Person{Who: asks.Operator},
|
|
Option: "go", Label: "Rebuild", Level: asks.Destroy, At: k.now(), AskDigest: ask.Digest()}
|
|
k.Warrant = func(string) (*asks.Ask, *asks.Warrant, error) { return &ask, w, nil }
|
|
out.reset()
|
|
raw, _ := json.Marshal(Request{ID: "r3", Verb: VerbRecreate, Module: "m", Local: "log", Ask: "bus-2"})
|
|
_, _ = store.Put(StateRequests, "r3", raw)
|
|
k.ActOnRequests()
|
|
if bus.held["m_log"].MaxBytes != 16*MiB || fmt.Sprint(out.events) != "[recreated m log]" ||
|
|
fmt.Sprint(out.conds) != "[cleared nats.drifted.m_log]" || k.Held()["m"].Made["log"].Drift != "" {
|
|
t.Errorf("held %+v, said %v %v: rebuilt as asked, the drift cleared", bus.held["m_log"], out.events, out.conds)
|
|
}
|
|
}},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
k, bus, _, out := newKeeper(FirstView{Revision: 1})
|
|
if _, err := k.AskFirstView(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
row.run(t, k, bus, out)
|
|
})
|
|
}
|
|
}
|