jschoubben
0d6d7acaae
Review round 2 of the registry milestone: - security F2: the spend record leaves requests, which the tools write as the tool runner's account, for state spent, named in the keeper's bus.state and declared read-only (the lead's 18:1x ruling: the tools read it, never write); nats_requests lists the warrants spent from it; - running F1: the fill pass also runs while the manifest-accepted consumer is bound again; - health note 3: an event said on the seat's subject and refused by its history is kept with its one id and said again on the history alone, never again with a new id; - seams note 3, health note 4: words unsaid with a status that cannot be written raise the keeper's own condition status-unwritten, cleared when it is written; the health names what a restart would lose. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
765 lines
39 KiB
Go
765 lines
39 KiB
Go
package keeper
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
|
|
oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
)
|
|
|
|
// fakeBus holds objects by stream, as a bus would, and refuses what a test tells it to.
|
|
type fakeBus struct {
|
|
mu sync.Mutex
|
|
held map[string]Settings
|
|
fill map[string]uint64
|
|
refuse map[string]error
|
|
created []string
|
|
updated []string
|
|
deleted []string
|
|
snaps [][]string
|
|
}
|
|
|
|
func newFakeBus() *fakeBus {
|
|
return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}}
|
|
}
|
|
|
|
func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
s, ok := b.held[o.Stream()]
|
|
return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil
|
|
}
|
|
|
|
func (b *fakeBus) Create(_ context.Context, o Object) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
if err := b.refuse[o.Stream()]; err != nil {
|
|
return err
|
|
}
|
|
b.held[o.Stream()] = o.Settings
|
|
b.created = append(b.created, o.Stream())
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Update(_ context.Context, o Object) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.held[o.Stream()] = o.Settings
|
|
b.updated = append(b.updated, o.Stream())
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Delete(_ context.Context, o Object) error {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
delete(b.held, o.Stream())
|
|
b.deleted = append(b.deleted, o.Stream())
|
|
return nil
|
|
}
|
|
|
|
func (b *fakeBus) Snapshot(_ context.Context, streams []string) (string, error) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
b.snaps = append(b.snaps, streams)
|
|
if err := b.refuse["snapshot"]; err != nil {
|
|
return "", err
|
|
}
|
|
return "/snapshots/" + strings.Join(streams, "+") + ".tar", nil
|
|
}
|
|
|
|
// memStore is a key-value state in memory.
|
|
type memStore struct {
|
|
// refuse refuses a Put of `<state>/<key>`.
|
|
refuse map[string]error
|
|
mu sync.Mutex
|
|
kv map[string][]byte
|
|
revs map[string]uint64
|
|
rev uint64
|
|
}
|
|
|
|
func newMemStore() *memStore {
|
|
return &memStore{kv: map[string][]byte{}, revs: map[string]uint64{}, refuse: map[string]error{}}
|
|
}
|
|
|
|
func (s *memStore) Get(state, key string) ([]byte, uint64, bool, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
v, ok := s.kv[state+"/"+key]
|
|
return v, s.revs[state+"/"+key], ok, nil
|
|
}
|
|
|
|
func (s *memStore) Put(state, key string, value []byte) (uint64, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if err := s.refuse[state+"/"+key]; err != nil {
|
|
return 0, err
|
|
}
|
|
s.rev++
|
|
s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev
|
|
return s.rev, nil
|
|
}
|
|
|
|
func (s *memStore) Create(state, key string, value []byte) (uint64, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if _, ok := s.kv[state+"/"+key]; ok {
|
|
return 0, ErrExists
|
|
}
|
|
s.rev++
|
|
s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev
|
|
return s.rev, nil
|
|
}
|
|
|
|
func (s *memStore) Delete(state, key string) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
delete(s.kv, state+"/"+key)
|
|
return nil
|
|
}
|
|
|
|
func (s *memStore) Keys(state string) ([]string, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
var out []string
|
|
for k := range s.kv {
|
|
if st, key, _ := strings.Cut(k, "/"); st == state {
|
|
out = append(out, key)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out, nil
|
|
}
|
|
|
|
// said is what a keeper said: its events and its conditions, in order.
|
|
type said struct {
|
|
mu sync.Mutex
|
|
events []string
|
|
bodies []Event
|
|
ids []string
|
|
conds []string
|
|
// down refuses every event and condition while set, as a bus that does not take them; partly takes an event on the
|
|
// seat's subject and refuses its history.
|
|
down, partly bool
|
|
// left are the subjects each try was asked to say an event on (nil: every one).
|
|
left [][]string
|
|
}
|
|
|
|
func (s *said) emit(e *Saying) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.left = append(s.left, e.Left)
|
|
if s.down {
|
|
e.Left, e.Said = []string{"seat", "history"}, nil
|
|
return errors.New("no responders")
|
|
}
|
|
if s.partly && e.Left == nil {
|
|
e.Left, e.Said = []string{"history"}, []string{"seat"}
|
|
s.ids = append(s.ids, e.ID)
|
|
return errors.New("the history stream is full")
|
|
}
|
|
s.events = append(s.events, e.Event+" "+e.Body.Module+" "+e.Body.Local)
|
|
s.bodies = append(s.bodies, e.Body)
|
|
s.ids = append(s.ids, e.ID)
|
|
e.Left = nil
|
|
return nil
|
|
}
|
|
|
|
func (s *said) say(c oc.Condition) error {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.down {
|
|
return errors.New("no responders")
|
|
}
|
|
s.conds = append(s.conds, c.State+" "+c.Key)
|
|
return nil
|
|
}
|
|
|
|
func (s *said) reset() {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.events, s.bodies, s.conds = nil, nil, nil
|
|
}
|
|
|
|
func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) {
|
|
bus, store, out := newFakeBus(), newMemStore(), &said{}
|
|
at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)
|
|
k := &Keeper{Bus: bus, Store: store, Emit: out.emit, Say: out.say,
|
|
View: func() (FirstView, error) { return view, nil },
|
|
Now: func() time.Time { return at }}
|
|
return k, bus, store, out
|
|
}
|
|
|
|
func entry(name, storage, template string, options map[string]string) catalogue.BusStateEntry {
|
|
e := catalogue.BusStateEntry{Name: name, Storage: storage, Template: template}
|
|
if len(options) > 0 {
|
|
e.Options = map[string]json.RawMessage{}
|
|
for k, v := range options {
|
|
e.Options[k] = json.RawMessage(v)
|
|
}
|
|
}
|
|
return e
|
|
}
|
|
|
|
// What one entry is on the bus (design 25 § 12.1, CAT-N3): each template's objects, named `<module>_<local>`, its
|
|
// binding, and the fixed settings no option changes; and the entries the keeper refuses itself, naming why.
|
|
func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) {
|
|
ledger := entry("records", catalogue.StorageLedger, catalogue.TemplateLedger, map[string]string{"max-mib": "16"})
|
|
ledger.Types = []catalogue.LedgerType{{Name: "decision", Indexes: []catalogue.LedgerIndex{{Name: "by-topic", On: []string{"topic"}}}}}
|
|
bare := entry("raw", catalogue.StorageStream, "", nil)
|
|
bare.Configuration = map[string]json.RawMessage{"when-full": json.RawMessage(`"drop-oldest"`), "max-mib": json.RawMessage("8"), "max-age-days": json.RawMessage("3")}
|
|
for _, row := range []struct {
|
|
name string
|
|
entry catalogue.BusStateEntry
|
|
objects string
|
|
binding string
|
|
refused string
|
|
}{
|
|
{name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}),
|
|
objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}",
|
|
binding: "bucket mesh-knowledge_drafts"},
|
|
{name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil),
|
|
objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}",
|
|
binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"},
|
|
{name: "work queue", entry: entry("jobs", catalogue.StorageStream, catalogue.TemplateWorkQueue, map[string]string{"max-mib": "4", "duplicates": "120"}),
|
|
objects: "stream mesh-knowledge_jobs [mesh.bus.mesh-knowledge.jobs.>] {4 MiB, age 0, history 0, new, size 0, work-queue, duplicates 120}",
|
|
binding: "stream mesh-knowledge_jobs on mesh.bus.mesh-knowledge.jobs"},
|
|
{name: "log", entry: entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil),
|
|
objects: "stream mesh-knowledge_log [mesh.bus.mesh-knowledge.log.>] {1024 MiB, age 0, history 0, new, size 262144}",
|
|
binding: "stream mesh-knowledge_log on mesh.bus.mesh-knowledge.log"},
|
|
{name: "objects", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "32"}),
|
|
objects: "object-store mesh-knowledge_files {32 MiB, age 0, history 0, new, size 0}",
|
|
binding: "objects mesh-knowledge_files"},
|
|
{name: "ledger", entry: ledger,
|
|
objects: "stream mesh-knowledge_records [mesh.ledger.mesh-knowledge.records.history.>] {16 MiB, age 0, history 0, new, size 262144}; " +
|
|
"key-value mesh-knowledge_records_state {64 MiB, age 0, history 1, new, size 262144}; " +
|
|
"key-value mesh-knowledge_records_index-by-topic {64 MiB, age 0, history 1, new, size 262144}; " +
|
|
"stream mesh-knowledge_records_snapshot [mesh.ledger.mesh-knowledge.records.snapshot.>] {16 MiB, age 0, history 1, old, size 0}",
|
|
binding: "history mesh-knowledge_records on mesh.ledger.mesh-knowledge.records.history, state mesh-knowledge_records_state, indexes map[by-topic:mesh-knowledge_records_index-by-topic], snapshot mesh.ledger.mesh-knowledge.records.snapshot"},
|
|
{name: "bare stream", entry: bare,
|
|
objects: "stream mesh-knowledge_raw [mesh.bus.mesh-knowledge.raw.>] {8 MiB, age 259200, history 0, old, size 0}",
|
|
binding: "stream mesh-knowledge_raw on mesh.bus.mesh-knowledge.raw"},
|
|
{name: "a local name holding the separator", entry: entry("my_drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil),
|
|
refused: `the local name "my_drafts" is empty or holds "_"`},
|
|
{name: "objects with no cap", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, nil),
|
|
refused: "gives no max-mib"},
|
|
{name: "a bare stream with no when-full", entry: entry("raw", catalogue.StorageStream, "", nil),
|
|
refused: "must give when-full"},
|
|
{name: "a template on the wrong storage type", entry: entry("x", catalogue.StorageKeyValue, catalogue.TemplateLog, nil),
|
|
refused: "is none the bus module makes"},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
p, err := PlanOf("mesh-knowledge", row.entry)
|
|
if row.refused != "" {
|
|
if err == nil || !strings.Contains(err.Error(), row.refused) {
|
|
t.Fatalf("planned %+v (%v): refused, naming %q", p, err, row.refused)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var objs []string
|
|
for _, o := range p.Objects {
|
|
s := o.Settings
|
|
w := fmt.Sprintf("%s %s", o.Kind, o.Name)
|
|
if len(o.Subjects) > 0 {
|
|
w += fmt.Sprintf(" %v", o.Subjects)
|
|
}
|
|
w += fmt.Sprintf(" {%d MiB, age %d, history %d, %s, size %d", s.MaxBytes/MiB, s.MaxAgeSeconds, s.History, s.Discard, s.MaxMsgSize)
|
|
if s.WorkQueue {
|
|
w += fmt.Sprintf(", work-queue, duplicates %d", s.DuplicatesSeconds)
|
|
}
|
|
objs = append(objs, w+"}")
|
|
}
|
|
if got := strings.Join(objs, "; "); got != row.objects {
|
|
t.Errorf("objects\n got %s\nwant %s", got, row.objects)
|
|
}
|
|
b := p.Binding
|
|
var binding string
|
|
switch {
|
|
case b.History != "":
|
|
binding = fmt.Sprintf("history %s on %s, state %s, indexes %v, snapshot %s", b.History, b.Subject, b.State, b.Indexes, b.Snapshot)
|
|
case b.Stream != "":
|
|
binding = fmt.Sprintf("stream %s on %s", b.Stream, b.Subject)
|
|
case b.Bucket != "":
|
|
binding = "bucket " + b.Bucket
|
|
case b.ObjectStore != "":
|
|
binding = "objects " + b.ObjectStore
|
|
}
|
|
if binding != row.binding || b.Local != row.entry.Name || b.Storage != row.entry.Storage {
|
|
t.Errorf("binding %s (%+v), want %s", binding, b, row.binding)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The keeper makes what an accepted verdict asks for, once, and says it (design 25 § 12.2, ADR 0595 §4; CAT-N2, N3):
|
|
// nothing before its first view; a replay not newer than what it holds is not applied; a refusal of its grant waits and
|
|
// is made on the backoff, never said refused; a refusal of the entry is said and raised, and cleared by a later make.
|
|
func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
|
|
drafts := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil)
|
|
log := entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil)
|
|
for _, row := range []struct {
|
|
name string
|
|
run func(t *testing.T, k *Keeper, bus *fakeBus, out *said)
|
|
}{
|
|
{"created once, binding said, nothing before the first view", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.mu.Lock()
|
|
k.viewed = false
|
|
k.mu.Unlock()
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if len(bus.created) != 0 {
|
|
t.Fatalf("created %v before the first view", bus.created)
|
|
}
|
|
if _, err := k.AskFirstView(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want {
|
|
t.Errorf("created %v, want %s", bus.created, want)
|
|
}
|
|
if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want {
|
|
t.Errorf("said %v, want %s", out.events, want)
|
|
}
|
|
if b := out.bodies[1]; b.Binding.Stream != "m_log" || b.Binding.Subject != "mesh.bus.m.log" || b.Revision != 3 || b.Asked["m_log"].MaxBytes == 0 {
|
|
t.Errorf("created carries %+v: the binding, the revision and the settings asked", b)
|
|
}
|
|
// In the field names of the seat mesh-broker's message said (mesh-contracts), beside binding and revision.
|
|
raw, _ := json.Marshal(out.bodies[1])
|
|
for _, w := range []string{`"module":"m"`, `"local-name":"log"`, `"storage-type":"stream"`, `"template":"log"`, `"bus-name":"m_log"`, `"binding":{`, `"revision":3`} {
|
|
if !strings.Contains(string(raw), w) {
|
|
t.Errorf("the body %s lacks %s", raw, w)
|
|
}
|
|
}
|
|
}},
|
|
{"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}})
|
|
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil})
|
|
if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
|
|
t.Errorf("held %+v: the replay at 4 took back what 9 asked", a)
|
|
}
|
|
}},
|
|
{"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
bus.refuse["m_log"] = fmt.Errorf("%w: permissions violation", ErrRetry)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
if w := k.Waiting(); len(w) != 1 || len(out.events) != 0 {
|
|
t.Fatalf("waiting %v, said %v: waits, says nothing", w, out.events)
|
|
}
|
|
delete(bus.refuse, "m_log")
|
|
k.CheckAll()
|
|
if len(k.Waiting()) != 0 || fmt.Sprint(out.events) != "[created m log]" {
|
|
t.Errorf("waiting %v, said %v: made at the next check", k.Waiting(), out.events)
|
|
}
|
|
}},
|
|
{"a refused create is said and raised, cleared by a later create", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
bus.refuse["KV_m_drafts"] = errors.New("insufficient resources")
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if fmt.Sprint(out.events) != "[refused m drafts]" || fmt.Sprint(out.conds) != "[raised nats.refused.m_drafts]" ||
|
|
!strings.Contains(out.bodies[0].Why, "insufficient resources") {
|
|
t.Fatalf("said %v %v %+v", out.events, out.conds, out.bodies)
|
|
}
|
|
delete(bus.refuse, "KV_m_drafts")
|
|
out.reset()
|
|
k.CheckAll()
|
|
if fmt.Sprint(out.events) != "[created m drafts]" || fmt.Sprint(out.conds) != "[cleared nats.refused.m_drafts]" {
|
|
t.Errorf("said %v %v: created, the refusal cleared", out.events, out.conds)
|
|
}
|
|
}},
|
|
{"an entry the keeper cannot plan is refused, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{entry("a_b", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
|
|
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[refused m a_b]" {
|
|
t.Errorf("created %v, said %v", bus.created, out.events)
|
|
}
|
|
}},
|
|
// CAT-N4: check and drift (ADR 0595 §5, §6): a difference that only grows is applied, said by nothing; any other
|
|
// is said and raised, never fixed quietly, and cleared once the bus holds what is asked again.
|
|
{"a held entry that grows is applied without a person", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
out.reset()
|
|
bigger := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "2048"})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{bigger}})
|
|
if fmt.Sprint(bus.updated) != "[m_log]" || bus.held["m_log"].MaxBytes != 2048*MiB || len(out.events)+len(out.conds) != 0 {
|
|
t.Errorf("updated %v to %d, said %v %v: grown quietly", bus.updated, bus.held["m_log"].MaxBytes, out.events, out.conds)
|
|
}
|
|
}},
|
|
{"a held entry that would shrink is a drift, said, raised and cleared", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
out.reset()
|
|
smaller := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{smaller}})
|
|
if len(bus.updated) != 0 || fmt.Sprint(out.events) != "[drifted m log]" || fmt.Sprint(out.conds) != "[raised nats.drifted.m_log]" ||
|
|
!strings.Contains(out.bodies[0].Why, "max-bytes asked 16777216, held 1073741824") {
|
|
t.Fatalf("updated %v, said %v %v %+v: the drift said, nothing changed", bus.updated, out.events, out.conds, out.bodies)
|
|
}
|
|
if l := k.CheckAll(); len(l) != 1 || l[0].Drift == "" || len(out.events) != 1 {
|
|
t.Errorf("check answers %+v and said %v: the drift once", l, out.events)
|
|
}
|
|
k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{log}})
|
|
if fmt.Sprint(out.events) != "[drifted m log drift-cleared m log]" || out.conds[len(out.conds)-1] != "cleared nats.drifted.m_log" {
|
|
t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds)
|
|
}
|
|
}},
|
|
{"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
p, _ := PlanOf("m", drafts)
|
|
bus.held["KV_m_drafts"] = p.Objects[0].Settings
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" {
|
|
t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events)
|
|
}
|
|
}},
|
|
// CAT-N5: orphans (design 25 § 12.2-12.3): an empty manifest-accepted asks nothing now, so what was made is an
|
|
// orphan, said and raised, kept until a drop; asked again it is kept and its binding said again; a module missing
|
|
// from the first view (set aside to wait) keeps its key; one never made leaves nothing behind.
|
|
{"an empty verdict orphans what was made, and asking again keeps it", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
out.reset()
|
|
k.Heard(Accepted{Module: "m", Revision: 2})
|
|
a, held := k.Held()["m"]
|
|
if !held || !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 || fmt.Sprint(out.events) != "[orphaned m drafts]" ||
|
|
fmt.Sprint(out.conds) != "[raised nats.orphaned.m_drafts]" {
|
|
t.Fatalf("held %v %+v, deleted %v, said %v %v: orphaned, kept, raised", held, a, bus.deleted, out.events, out.conds)
|
|
}
|
|
k.Heard(Accepted{Module: "m", Revision: 3})
|
|
k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}})
|
|
if fmt.Sprint(out.events) != "[orphaned m drafts created m drafts]" || out.conds[len(out.conds)-1] != "cleared nats.orphaned.m_drafts" {
|
|
t.Errorf("said %v %v: orphaned once, then created again and cleared", out.events, out.conds)
|
|
}
|
|
}},
|
|
{"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil }
|
|
if _, err := k.AskFirstView(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
|
|
t.Errorf("held %+v: kept as asked, never orphaned by a first view", a)
|
|
}
|
|
bus.refuse["KV_n_x"] = errors.New("no")
|
|
k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
|
|
k.Heard(Accepted{Module: "n", Revision: 2})
|
|
if _, held := k.Held()["n"]; held {
|
|
t.Errorf("n's key is still held: nothing of it was made, so its empty verdict ends it")
|
|
}
|
|
}},
|
|
// CAT-N6: fill (ADR 0595 §6): an entry that refuses when full raises `full` at the raise mark and clears only
|
|
// below the clear mark; one that drops its oldest is never read; the health names the pass.
|
|
{"full at the raise mark, cleared only below the clear mark", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
events := entry("events", catalogue.StorageStream, catalogue.TemplateEvents, map[string]string{"max-mib": "1"})
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, events}})
|
|
out.reset()
|
|
bus.fill["m_events"] = 1 * uint64(MiB)
|
|
var steps []string
|
|
for _, mib := range []uint64{40, 48, 46, 45, 44} {
|
|
bus.fill["KV_m_drafts"] = mib * uint64(MiB)
|
|
k.FillPass()
|
|
steps = append(steps, fmt.Sprintf("%d:%d", mib, len(out.events)))
|
|
}
|
|
if fmt.Sprint(steps) != "[40:0 48:1 46:1 45:1 44:2]" || fmt.Sprint(out.events) != "[full m drafts full-cleared m drafts]" ||
|
|
fmt.Sprint(out.conds) != "[raised nats.full.m_drafts cleared nats.full.m_drafts]" {
|
|
t.Errorf("said %v %v: full at 48 of 64 MiB (75%%), kept at 46 and 45 (71%%, 70%%), cleared at 44 (68%%)", out.events, out.conds)
|
|
}
|
|
h := HealthOf(k.Status(), k.Held(), 0, k.now())
|
|
if !h.Healthy || h.Pass == "none" {
|
|
t.Errorf("health %+v: healthy, naming its pass", h)
|
|
}
|
|
}},
|
|
{"the health is unhealthy past its bound, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
at := k.now()
|
|
k.FillPass()
|
|
later := at.Add(4 * time.Minute)
|
|
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "last fill pass is 4m0s old") {
|
|
t.Errorf("health %+v: unhealthy, its pass too old", h)
|
|
}
|
|
if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") {
|
|
t.Errorf("health %+v: names the verdicts waiting", h)
|
|
}
|
|
k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") }
|
|
k.mu.Lock()
|
|
k.status.FirstView.Answered = time.Time{}
|
|
k.mu.Unlock()
|
|
if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second {
|
|
t.Fatalf("asked again in %s (%v): on the backoff", next, err)
|
|
}
|
|
// The first view unanswered is unhealthy only past the bound since the start (review round 1: health.go:41).
|
|
if h := HealthOf(k.Status(), nil, 0, at); !h.Healthy {
|
|
t.Errorf("health %+v: healthy within the bound while the first view is asked", h)
|
|
}
|
|
k.FillPass()
|
|
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") {
|
|
t.Errorf("health %+v: unhealthy past the bound, naming the verb", h)
|
|
}
|
|
// What waits on its consumer unread is never skipped silently (review round 1: health.go:47).
|
|
if h := HealthOf(k.Status(), nil, -1, at); h.Healthy || !strings.Contains(h.Why, "cannot be read") {
|
|
t.Errorf("health %+v: unhealthy when its consumer's backlog cannot be read", h)
|
|
}
|
|
}},
|
|
// CAT-N8: a person's requests: a drop removes an orphan alone, only on the operator's warrant binding exactly
|
|
// that drop, after a snapshot, said dropped and recorded; a recreate rebuilds a drifted entry the same way.
|
|
{"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}})
|
|
var acts []string
|
|
k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
|
|
ask := oc.Ask{ID: "bus-1", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour))}
|
|
var w *oc.Warrant
|
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil }
|
|
put := func(r Request) Request {
|
|
raw, _ := json.Marshal(r)
|
|
_, _ = store.Put(StateRequests, r.ID, raw)
|
|
k.ActOnRequests()
|
|
raw, _, _, _ = store.Get(StateRequests, r.ID)
|
|
var got Request
|
|
_ = json.Unmarshal(raw, &got)
|
|
return got
|
|
}
|
|
if got := put(Request{ID: "r0", Verb: VerbDrop, Module: "m", Local: "log", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "drop removes an orphan alone") {
|
|
t.Errorf("dropping an asked entry: %+v", got)
|
|
}
|
|
if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !got.Done.IsZero() || len(bus.deleted) != 0 {
|
|
t.Fatalf("acted with no warrant: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
binds := func(act occ.Act) string { d, _ := occ.ActDigest(act); return d }
|
|
choose := func(act occ.Act) {
|
|
ask.Options = []oc.Option{{ID: "go", Label: "Remove", Level: occ.Destroy, Binds: binds(act)}}
|
|
w = &oc.Warrant{Ask: "bus-1", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator},
|
|
Option: "go", Label: "Remove", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)}
|
|
}
|
|
choose(occ.Act{"verb": "drop", "module": "m", "local": "log"})
|
|
if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "bound") || len(bus.deleted) != 0 {
|
|
t.Fatalf("acted on a warrant for another act: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
// ADR 0626 §4: the keeper trusts nothing in the request; each of these is refused, said why, and deletes nothing.
|
|
refused := func(r Request, why string) {
|
|
t.Helper()
|
|
if got := put(r); got.Done.IsZero() || !strings.Contains(got.Error, why) || len(bus.deleted) != 0 {
|
|
t.Errorf("%s: ended %+v, deleted %v: refused for %q", r.ID, got, bus.deleted, why)
|
|
}
|
|
}
|
|
refused(Request{ID: "r1a", Verb: VerbDrop, Module: "m", Local: "drafts", Caller: "person.jo"}, "names no ask")
|
|
choose(occ.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
|
w.Level, w.Label, ask.Options[0].Level = occ.Approve, "Keep", occ.Approve
|
|
w.AskDigest = occ.Digest(ask)
|
|
refused(Request{ID: "r1b", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "nothing is removed")
|
|
choose(occ.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
|
w.Asker = "mesh-grants"
|
|
refused(Request{ID: "r1c", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "no warrant")
|
|
refused(Request{ID: "r1d", Verb: VerbCreate, Module: "m", Local: "drafts", Caller: "person.jo"}, "no accepted manifest of m asks for drafts")
|
|
choose(occ.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
|
bus.refuse["snapshot"] = errors.New("no space")
|
|
refused(Request{ID: "r1e", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}, "the snapshot before drop failed, so nothing was removed")
|
|
delete(bus.refuse, "snapshot")
|
|
bus.snaps = nil
|
|
out.reset()
|
|
got := put(Request{ID: "r2", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", Why: "unused"})
|
|
if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts]" ||
|
|
fmt.Sprint(out.events) != "[dropped m drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_drop: dropped drafts of m") ||
|
|
!strings.Contains(acts[0], "person.jo") {
|
|
t.Errorf("ended %+v, snapshots %v, deleted %v, said %v, recorded %v", got, bus.snaps, bus.deleted, out.events, acts)
|
|
}
|
|
if _, made := k.Held()["m"].Made["drafts"]; made {
|
|
t.Error("the dropped entry is still held")
|
|
}
|
|
}},
|
|
{"a recreate rebuilds a drifted entry after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})}})
|
|
binds, _ := occ.ActDigest(occ.Act{"verb": "recreate", "module": "m", "local": "log"})
|
|
ask := oc.Ask{ID: "bus-2", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour)), Options: []oc.Option{{ID: "go", Label: "Rebuild", Level: occ.Destroy, Binds: binds}}}
|
|
w := &oc.Warrant{Ask: "bus-2", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator},
|
|
Option: "go", Label: "Rebuild", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)}
|
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil }
|
|
out.reset()
|
|
raw, _ := json.Marshal(Request{ID: "r3", Verb: VerbRecreate, Module: "m", Local: "log", Ask: "bus-2"})
|
|
_, _ = store.Put(StateRequests, "r3", raw)
|
|
k.ActOnRequests()
|
|
if bus.held["m_log"].MaxBytes != 16*MiB || fmt.Sprint(out.events) != "[recreated m log]" ||
|
|
fmt.Sprint(out.conds) != "[cleared nats.drifted.m_log]" || k.Held()["m"].Made["log"].Drift != "" {
|
|
t.Errorf("held %+v, said %v %v: rebuilt as asked, the drift cleared", bus.held["m_log"], out.events, out.conds)
|
|
}
|
|
}},
|
|
// Review round 1 (security F2; design 25 § 12.3): a warrant acts once, spent by compare-and-set before anything is
|
|
// deleted; a second request naming the same ask is refused and deletes nothing.
|
|
{"a warrant acts once", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2})
|
|
binds, _ := occ.ActDigest(occ.Act{"verb": "drop", "module": "m", "local": "drafts"})
|
|
ask := oc.Ask{ID: "bus-5", Who: occ.Operator, Expires: occ.FormatTime(k.now().Add(time.Hour)), Options: []oc.Option{{ID: "go", Label: "Remove", Level: occ.Destroy, Binds: binds}}}
|
|
w := &oc.Warrant{Ask: "bus-5", Asker: Module, Outcome: occ.OutcomeChosen, By: &oc.Person{Who: occ.Operator},
|
|
Option: "go", Label: "Remove", Level: occ.Destroy, At: occ.FormatTime(k.now()), AskDigest: occ.Digest(ask)}
|
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return &ask, w, nil }
|
|
put := func(id string) Request {
|
|
raw, _ := json.Marshal(Request{ID: id, Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-5", Caller: "person.jo"})
|
|
_, _ = store.Put(StateRequests, id, raw)
|
|
k.ActOnRequests()
|
|
raw, _, _, _ = store.Get(StateRequests, id)
|
|
var got Request
|
|
_ = json.Unmarshal(raw, &got)
|
|
return got
|
|
}
|
|
if got := put("r5"); got.Error != "" || len(bus.deleted) != 1 {
|
|
t.Fatalf("the first drop: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
// The entry is asked for and orphaned again, so a second drop would remove it: the spent warrant refuses it.
|
|
k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{log}})
|
|
if got := put("r6"); !strings.Contains(got.Error, "already acted (drop of m's drafts, request r5") || len(bus.deleted) != 1 {
|
|
t.Errorf("the second request on the same warrant: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
if raw, _, found, _ := store.Get(StateSpent, "bus-5"); !found || !strings.Contains(string(raw), `"request":"r5"`) {
|
|
t.Errorf("the spend record is %s (%v)", raw, found)
|
|
}
|
|
// The spend record is the process's own state, never in requests, which the tools write: every key of requests
|
|
// gone, a third request on the same warrant is still refused (review round 2, security F2).
|
|
keys, _ := store.Keys(StateRequests)
|
|
for _, key := range keys {
|
|
_ = store.Delete(StateRequests, key)
|
|
}
|
|
if got := put("r6b"); !strings.Contains(got.Error, "already acted") || len(bus.deleted) != 1 {
|
|
t.Errorf("with requests wiped, the spent warrant: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
}},
|
|
// An ask that ended unanswered ends its request, said, instead of a request that waits until its key ages out; and a
|
|
// hand act that cannot be recorded is said on the request's end, never only logged.
|
|
{"an ask ended unanswered ends its request; an unrecorded hand act is said", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
|
|
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}})
|
|
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return nil, nil, nil }
|
|
get := func(id string) Request {
|
|
raw, _, _, _ := store.Get(StateRequests, id)
|
|
var got Request
|
|
_ = json.Unmarshal(raw, &got)
|
|
return got
|
|
}
|
|
raw, _ := json.Marshal(Request{ID: "r7", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-7", Caller: "person.jo", At: k.now().Add(-time.Hour)})
|
|
_, _ = store.Put(StateRequests, "r7", raw)
|
|
raw, _ = json.Marshal(Request{ID: "r8", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-8", Caller: "person.jo", At: k.now().Add(-AskLasts - time.Minute)})
|
|
_, _ = store.Put(StateRequests, "r8", raw)
|
|
k.ActOnRequests()
|
|
if got := get("r7"); !got.Done.IsZero() {
|
|
t.Errorf("a request whose ask still runs ended: %+v", got)
|
|
}
|
|
if got := get("r8"); got.Done.IsZero() || !strings.Contains(got.Error, "ended at") || len(bus.deleted) != 0 {
|
|
t.Errorf("a request whose ask ended unanswered: %+v, deleted %v", got, bus.deleted)
|
|
}
|
|
k.Act = func(string, string, string) error { return errors.New("the hand-act seat does not answer") }
|
|
raw, _ = json.Marshal(Request{ID: "r9", Verb: VerbCreate, Module: "m", Local: "log", Caller: "person.jo"})
|
|
_, _ = store.Put(StateRequests, "r9", raw)
|
|
k.ActOnRequests()
|
|
if got := get("r9"); got.Done.IsZero() || !strings.Contains(got.Unrecorded, "does not answer") {
|
|
t.Errorf("an unrecorded hand act: %+v", got)
|
|
}
|
|
}},
|
|
// Review round 1 (health F1): created is marked only once said, so the grants part always gets its binding; an
|
|
// event or a condition the bus did not take is kept, named by the health, and said again at the next fill pass
|
|
// with the same id.
|
|
{"what the bus did not take is said again; created marked only once said", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
out.down = true
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, entry("raw", catalogue.StorageStream, "", nil)}})
|
|
if m := k.Held()["m"].Made["drafts"]; m.Created || !strings.Contains(m.Waiting, "its created could not be said yet") {
|
|
t.Fatalf("made %+v: not created until said, waiting", m)
|
|
}
|
|
st := k.Status()
|
|
if len(st.Unsaid) != 1 || st.Unsaid[0].Event != EventRefused || len(st.UnsaidConditions) != 1 {
|
|
t.Fatalf("unsaid %+v %+v: the refused event and its condition kept", st.Unsaid, st.UnsaidConditions)
|
|
}
|
|
if h := HealthOf(k.Status(), nil, 0, k.now()); h.Healthy || !strings.Contains(h.Why, "2 word(s) the bus did not take") {
|
|
t.Errorf("health %+v: unhealthy, naming what was not said", h)
|
|
}
|
|
id := st.Unsaid[0].ID
|
|
out.down = false
|
|
k.FillPass()
|
|
k.CheckAll()
|
|
if !strings.Contains(fmt.Sprint(out.events), "refused m raw") || !strings.Contains(fmt.Sprint(out.events), "created m drafts") ||
|
|
fmt.Sprint(out.conds) != "[raised nats.refused.m_raw]" || out.ids[0] != id {
|
|
t.Errorf("said %v (ids %v) %v: the refused said again with its id %s, its condition raised, created said", out.events, out.ids, out.conds, id)
|
|
}
|
|
if st := k.Status(); len(st.Unsaid) != 0 || len(st.UnsaidConditions) != 0 || !k.Held()["m"].Made["drafts"].Created {
|
|
t.Errorf("after the pass: unsaid %+v %+v, drafts %+v", st.Unsaid, st.UnsaidConditions, k.Held()["m"].Made["drafts"])
|
|
}
|
|
}},
|
|
// Review round 2 (health note 3): created said on the seat's subject and refused by its history is created, and its
|
|
// history is said again at the fill pass with the same id, on the history alone.
|
|
{"created said on the seat and not its history is said again on what is left", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
out.partly = true
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
|
|
st := k.Status()
|
|
if m := k.Held()["m"].Made["drafts"]; !m.Created || len(st.Unsaid) != 1 || fmt.Sprint(st.Unsaid[0].Left) != "[history]" {
|
|
t.Fatalf("made %+v, unsaid %+v: created, its history kept to say", m, st.Unsaid)
|
|
}
|
|
id := st.Unsaid[0].ID
|
|
out.partly = false
|
|
k.FillPass()
|
|
if last := out.left[len(out.left)-1]; fmt.Sprint(last) != "[history]" || out.ids[len(out.ids)-1] != id || len(k.Status().Unsaid) != 0 {
|
|
t.Errorf("said again on %v with id %s (want [history], %s); unsaid %+v", last, out.ids[len(out.ids)-1], id, k.Status().Unsaid)
|
|
}
|
|
if n := strings.Count(fmt.Sprint(out.ids), id); n != 2 {
|
|
t.Errorf("the id %s was used %d times: once on the seat, once on the history", id, n)
|
|
}
|
|
}},
|
|
// Review round 2 (seams note 3, health note 4): words unsaid with a status that cannot be written raise the keeper's
|
|
// own condition, once, cleared when the status is written again; the health names what a restart would lose.
|
|
{"a status that cannot be written with words unsaid is said", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
k.FillPass()
|
|
store.refuse[StateAsked+"/"+KeyStatus] = errors.New("the bucket is full")
|
|
k.Emit = func(e *Saying) error { e.Left = []string{"seat", "history"}; return errors.New("no responders") }
|
|
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, entry("raw", catalogue.StorageStream, "", nil)}})
|
|
k.FillPass()
|
|
raisedN := strings.Count(fmt.Sprint(out.conds), "raised nats.status-unwritten.nats_keeper")
|
|
if raisedN != 1 {
|
|
t.Errorf("said %v: status-unwritten raised once", out.conds)
|
|
}
|
|
if h := HealthOf(k.Status(), nil, 0, k.now().Add(4*time.Minute)); !strings.Contains(h.Why, "may be lost at a restart") {
|
|
t.Errorf("health %+v: names what a restart loses", h)
|
|
}
|
|
delete(store.refuse, StateAsked+"/"+KeyStatus)
|
|
k.Emit = out.emit
|
|
k.FillPass()
|
|
if !strings.Contains(fmt.Sprint(out.conds), "cleared nats.status-unwritten.nats_keeper") {
|
|
t.Errorf("said %v: cleared once the status is written", out.conds)
|
|
}
|
|
}},
|
|
// CAT-N2b: nats_first_view_again asks the first view at once, by hand, recorded as a hand act.
|
|
{"the first view asked again by hand", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
|
|
store := k.Store.(*memStore)
|
|
var acts []string
|
|
k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
|
|
k.View = func() (FirstView, error) {
|
|
return FirstView{Revision: 8, Modules: []Accepted{{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}, nil
|
|
}
|
|
raw, _ := json.Marshal(Request{ID: "r9", Verb: VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"})
|
|
_, _ = store.Put(StateRequests, "r9", raw)
|
|
k.ActOnRequests()
|
|
raw, _, _, _ = store.Get(StateRequests, "r9")
|
|
var got Request
|
|
_ = json.Unmarshal(raw, &got)
|
|
if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts]" ||
|
|
len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked mesh-delivery-indexer.indexer_bus_state again by hand") {
|
|
t.Errorf("ended %+v, created %v, recorded %v", got, bus.created, acts)
|
|
}
|
|
}},
|
|
} {
|
|
t.Run(row.name, func(t *testing.T) {
|
|
k, bus, _, out := newKeeper(FirstView{Revision: 1})
|
|
if _, err := k.AskFirstView(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
row.run(t, k, bus, out)
|
|
})
|
|
}
|
|
}
|