package keeper import ( "context" "encoding/json" "errors" "fmt" "sort" "strings" "sync" "testing" "time" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" ) // fakeBus holds objects by stream, as a bus would, and refuses what a test tells it to. type fakeBus struct { mu sync.Mutex held map[string]Settings fill map[string]uint64 refuse map[string]error created []string updated []string deleted []string snaps [][]string } func newFakeBus() *fakeBus { return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}} } func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) { b.mu.Lock() defer b.mu.Unlock() s, ok := b.held[o.Stream()] return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil } func (b *fakeBus) Create(_ context.Context, o Object) error { b.mu.Lock() defer b.mu.Unlock() if err := b.refuse[o.Stream()]; err != nil { return err } b.held[o.Stream()] = o.Settings b.created = append(b.created, o.Stream()) return nil } func (b *fakeBus) Update(_ context.Context, o Object) error { b.mu.Lock() defer b.mu.Unlock() b.held[o.Stream()] = o.Settings b.updated = append(b.updated, o.Stream()) return nil } func (b *fakeBus) Delete(_ context.Context, o Object) error { b.mu.Lock() defer b.mu.Unlock() delete(b.held, o.Stream()) b.deleted = append(b.deleted, o.Stream()) return nil } func (b *fakeBus) Snapshot(_ context.Context, streams []string) (string, error) { b.mu.Lock() defer b.mu.Unlock() b.snaps = append(b.snaps, streams) return "/snapshots/" + strings.Join(streams, "+") + ".tar", nil } // memStore is a key-value state in memory. type memStore struct { mu sync.Mutex kv map[string][]byte revs map[string]uint64 rev uint64 } func newMemStore() *memStore { return &memStore{kv: map[string][]byte{}, revs: map[string]uint64{}} } func (s *memStore) Get(state, key string) ([]byte, uint64, bool, error) { s.mu.Lock() defer s.mu.Unlock() v, ok := s.kv[state+"/"+key] return v, s.revs[state+"/"+key], ok, nil } func (s *memStore) Put(state, key string, value []byte) (uint64, error) { s.mu.Lock() defer s.mu.Unlock() s.rev++ s.kv[state+"/"+key], s.revs[state+"/"+key] = value, s.rev return s.rev, nil } func (s *memStore) Delete(state, key string) error { s.mu.Lock() defer s.mu.Unlock() delete(s.kv, state+"/"+key) return nil } func (s *memStore) Keys(state string) ([]string, error) { s.mu.Lock() defer s.mu.Unlock() var out []string for k := range s.kv { if st, key, _ := strings.Cut(k, "/"); st == state { out = append(out, key) } } sort.Strings(out) return out, nil } // said is what a keeper said: its events and its conditions, in order. type said struct { mu sync.Mutex events []string bodies []Event conds []string } func (s *said) emit(event string, body Event) error { s.mu.Lock() defer s.mu.Unlock() s.events = append(s.events, event+" "+body.Module+" "+body.Local) s.bodies = append(s.bodies, body) return nil } func (s *said) say(c asks.Condition) error { s.mu.Lock() defer s.mu.Unlock() s.conds = append(s.conds, c.State+" "+c.Key) return nil } func (s *said) reset() { s.mu.Lock() defer s.mu.Unlock() s.events, s.bodies, s.conds = nil, nil, nil } func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) { bus, store, out := newFakeBus(), newMemStore(), &said{} at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) k := &Keeper{Bus: bus, Store: store, Emit: out.emit, Say: out.say, View: func() (FirstView, error) { return view, nil }, Now: func() time.Time { return at }} return k, bus, store, out } func entry(name, storage, template string, options map[string]string) catalogue.BusStateEntry { e := catalogue.BusStateEntry{Name: name, Storage: storage, Template: template} if len(options) > 0 { e.Options = map[string]json.RawMessage{} for k, v := range options { e.Options[k] = json.RawMessage(v) } } return e } // What one entry is on the bus (design 25 § 12.1, CAT-N3): each template's objects, named `_`, its // binding, and the fixed settings no option changes; and the entries the keeper refuses itself, naming why. func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) { ledger := entry("records", catalogue.StorageLedger, catalogue.TemplateLedger, map[string]string{"max-mib": "16"}) ledger.Types = []catalogue.LedgerType{{Name: "decision", Indexes: []catalogue.LedgerIndex{{Name: "by-topic", On: []string{"topic"}}}}} bare := entry("raw", catalogue.StorageStream, "", nil) bare.Configuration = map[string]json.RawMessage{"when-full": json.RawMessage(`"drop-oldest"`), "max-mib": json.RawMessage("8"), "max-age-days": json.RawMessage("3")} for _, row := range []struct { name string entry catalogue.BusStateEntry objects string binding string refused string }{ {name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}), objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}", binding: "bucket mesh-knowledge_drafts"}, {name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil), objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}", binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"}, {name: "work queue", entry: entry("jobs", catalogue.StorageStream, catalogue.TemplateWorkQueue, map[string]string{"max-mib": "4", "duplicates": "120"}), objects: "stream mesh-knowledge_jobs [mesh.bus.mesh-knowledge.jobs.>] {4 MiB, age 0, history 0, new, size 0, work-queue, duplicates 120}", binding: "stream mesh-knowledge_jobs on mesh.bus.mesh-knowledge.jobs"}, {name: "log", entry: entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil), objects: "stream mesh-knowledge_log [mesh.bus.mesh-knowledge.log.>] {1024 MiB, age 0, history 0, new, size 262144}", binding: "stream mesh-knowledge_log on mesh.bus.mesh-knowledge.log"}, {name: "objects", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "32"}), objects: "object-store mesh-knowledge_files {32 MiB, age 0, history 0, new, size 0}", binding: "objects mesh-knowledge_files"}, {name: "ledger", entry: ledger, objects: "stream mesh-knowledge_records [mesh.ledger.mesh-knowledge.records.history.>] {16 MiB, age 0, history 0, new, size 262144}; " + "key-value mesh-knowledge_records_state {64 MiB, age 0, history 1, new, size 262144}; " + "key-value mesh-knowledge_records_index-by-topic {64 MiB, age 0, history 1, new, size 262144}; " + "stream mesh-knowledge_records_snapshot [mesh.ledger.mesh-knowledge.records.snapshot.>] {16 MiB, age 0, history 1, old, size 0}", binding: "history mesh-knowledge_records on mesh.ledger.mesh-knowledge.records.history, state mesh-knowledge_records_state, indexes map[by-topic:mesh-knowledge_records_index-by-topic], snapshot mesh.ledger.mesh-knowledge.records.snapshot"}, {name: "bare stream", entry: bare, objects: "stream mesh-knowledge_raw [mesh.bus.mesh-knowledge.raw.>] {8 MiB, age 259200, history 0, old, size 0}", binding: "stream mesh-knowledge_raw on mesh.bus.mesh-knowledge.raw"}, {name: "a local name holding the separator", entry: entry("my_drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil), refused: `the local name "my_drafts" is empty or holds "_"`}, {name: "objects with no cap", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, nil), refused: "gives no max-mib"}, {name: "a bare stream with no when-full", entry: entry("raw", catalogue.StorageStream, "", nil), refused: "must give when-full"}, {name: "a template on the wrong storage type", entry: entry("x", catalogue.StorageKeyValue, catalogue.TemplateLog, nil), refused: "is none the bus module makes"}, } { t.Run(row.name, func(t *testing.T) { p, err := PlanOf("mesh-knowledge", row.entry) if row.refused != "" { if err == nil || !strings.Contains(err.Error(), row.refused) { t.Fatalf("planned %+v (%v): refused, naming %q", p, err, row.refused) } return } if err != nil { t.Fatal(err) } var objs []string for _, o := range p.Objects { s := o.Settings w := fmt.Sprintf("%s %s", o.Kind, o.Name) if len(o.Subjects) > 0 { w += fmt.Sprintf(" %v", o.Subjects) } w += fmt.Sprintf(" {%d MiB, age %d, history %d, %s, size %d", s.MaxBytes/MiB, s.MaxAgeSeconds, s.History, s.Discard, s.MaxMsgSize) if s.WorkQueue { w += fmt.Sprintf(", work-queue, duplicates %d", s.DuplicatesSeconds) } objs = append(objs, w+"}") } if got := strings.Join(objs, "; "); got != row.objects { t.Errorf("objects\n got %s\nwant %s", got, row.objects) } b := p.Binding var binding string switch { case b.History != "": binding = fmt.Sprintf("history %s on %s, state %s, indexes %v, snapshot %s", b.History, b.Subject, b.State, b.Indexes, b.Snapshot) case b.Stream != "": binding = fmt.Sprintf("stream %s on %s", b.Stream, b.Subject) case b.Bucket != "": binding = "bucket " + b.Bucket case b.ObjectStore != "": binding = "objects " + b.ObjectStore } if binding != row.binding || b.Local != row.entry.Name || b.Storage != row.entry.Storage { t.Errorf("binding %s (%+v), want %s", binding, b, row.binding) } }) } } // The keeper makes what an accepted verdict asks for, once, and says it (design 25 § 12.2, ADR 0595 §4; CAT-N2, N3): // nothing before its first view; a replay not newer than what it holds is not applied; a refusal of its grant waits and // is made on the backoff, never said refused; a refusal of the entry is said and raised, and cleared by a later make. func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { drafts := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil) log := entry("log", catalogue.StorageStream, catalogue.TemplateLog, nil) for _, row := range []struct { name string run func(t *testing.T, k *Keeper, bus *fakeBus, out *said) }{ {"created once, binding said, nothing before the first view", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.mu.Lock() k.viewed = false k.mu.Unlock() k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}}) if len(bus.created) != 0 { t.Fatalf("created %v before the first view", bus.created) } if _, err := k.AskFirstView(0); err != nil { t.Fatal(err) } k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}}) k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}}) if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want { t.Errorf("created %v, want %s", bus.created, want) } if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want { t.Errorf("said %v, want %s", out.events, want) } if b := out.bodies[1]; b.Binding.Stream != "m_log" || b.Binding.Subject != "mesh.bus.m.log" || b.Revision != 3 || b.Asked["m_log"].MaxBytes == 0 { t.Errorf("created carries %+v: the binding, the revision and the settings asked", b) } }}, {"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}) k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil}) if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned { t.Errorf("held %+v: the replay at 4 took back what 9 asked", a) } }}, {"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { bus.refuse["m_log"] = fmt.Errorf("%w: permissions violation", ErrRetry) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}}) if w := k.Waiting(); len(w) != 1 || len(out.events) != 0 { t.Fatalf("waiting %v, said %v: waits, says nothing", w, out.events) } delete(bus.refuse, "m_log") k.CheckAll() if len(k.Waiting()) != 0 || fmt.Sprint(out.events) != "[created m log]" { t.Errorf("waiting %v, said %v: made at the next check", k.Waiting(), out.events) } }}, {"a refused create is said and raised, cleared by a later create", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { bus.refuse["KV_m_drafts"] = errors.New("insufficient resources") k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) if fmt.Sprint(out.events) != "[refused m drafts]" || fmt.Sprint(out.conds) != "[raised nats.refused.m_drafts]" || !strings.Contains(out.bodies[0].Why, "insufficient resources") { t.Fatalf("said %v %v %+v", out.events, out.conds, out.bodies) } delete(bus.refuse, "KV_m_drafts") out.reset() k.CheckAll() if fmt.Sprint(out.events) != "[created m drafts]" || fmt.Sprint(out.conds) != "[cleared nats.refused.m_drafts]" { t.Errorf("said %v %v: created, the refusal cleared", out.events, out.conds) } }}, {"an entry the keeper cannot plan is refused, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{entry("a_b", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}}) if len(bus.created) != 0 || fmt.Sprint(out.events) != "[refused m a_b]" { t.Errorf("created %v, said %v", bus.created, out.events) } }}, // CAT-N4: check and drift (ADR 0595 §5, §6): a difference that only grows is applied, said by nothing; any other // is said and raised, never fixed quietly, and cleared once the bus holds what is asked again. {"a held entry that grows is applied without a person", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}}) out.reset() bigger := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "2048"}) k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{bigger}}) if fmt.Sprint(bus.updated) != "[m_log]" || bus.held["m_log"].MaxBytes != 2048*MiB || len(out.events)+len(out.conds) != 0 { t.Errorf("updated %v to %d, said %v %v: grown quietly", bus.updated, bus.held["m_log"].MaxBytes, out.events, out.conds) } }}, {"a held entry that would shrink is a drift, said, raised and cleared", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}}) out.reset() smaller := entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"}) k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{smaller}}) if len(bus.updated) != 0 || fmt.Sprint(out.events) != "[drifted m log]" || fmt.Sprint(out.conds) != "[raised nats.drifted.m_log]" || !strings.Contains(out.bodies[0].Why, "max-bytes asked 16777216, held 1073741824") { t.Fatalf("updated %v, said %v %v %+v: the drift said, nothing changed", bus.updated, out.events, out.conds, out.bodies) } if l := k.CheckAll(); len(l) != 1 || l[0].Drift == "" || len(out.events) != 1 { t.Errorf("check answers %+v and said %v: the drift once", l, out.events) } k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{log}}) if fmt.Sprint(out.events) != "[drifted m log drift-cleared m log]" || out.conds[len(out.conds)-1] != "cleared nats.drifted.m_log" { t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds) } }}, {"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { p, _ := PlanOf("m", drafts) bus.held["KV_m_drafts"] = p.Objects[0].Settings k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" { t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events) } }}, // CAT-N5: orphans (design 25 § 12.2-12.3): an empty manifest-accepted asks nothing now, so what was made is an // orphan, said and raised, kept until a drop; asked again it is kept and its binding said again; a module missing // from the first view (set aside to wait) keeps its key; one never made leaves nothing behind. {"an empty verdict orphans what was made, and asking again keeps it", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) out.reset() k.Heard(Accepted{Module: "m", Revision: 2}) a, held := k.Held()["m"] if !held || !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 || fmt.Sprint(out.events) != "[orphaned m drafts]" || fmt.Sprint(out.conds) != "[raised nats.orphaned.m_drafts]" { t.Fatalf("held %v %+v, deleted %v, said %v %v: orphaned, kept, raised", held, a, bus.deleted, out.events, out.conds) } k.Heard(Accepted{Module: "m", Revision: 3}) k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}}) if fmt.Sprint(out.events) != "[orphaned m drafts created m drafts]" || out.conds[len(out.conds)-1] != "cleared nats.orphaned.m_drafts" { t.Errorf("said %v %v: orphaned once, then created again and cleared", out.events, out.conds) } }}, {"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil } if _, err := k.AskFirstView(0); err != nil { t.Fatal(err) } if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned { t.Errorf("held %+v: kept as asked, never orphaned by a first view", a) } bus.refuse["KV_n_x"] = errors.New("no") k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}}) k.Heard(Accepted{Module: "n", Revision: 2}) if _, held := k.Held()["n"]; held { t.Errorf("n's key is still held: nothing of it was made, so its empty verdict ends it") } }}, // CAT-N6: fill (ADR 0595 §6): an entry that refuses when full raises `full` at the raise mark and clears only // below the clear mark; one that drops its oldest is never read; the health names the pass. {"full at the raise mark, cleared only below the clear mark", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { events := entry("events", catalogue.StorageStream, catalogue.TemplateEvents, map[string]string{"max-mib": "1"}) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, events}}) out.reset() bus.fill["m_events"] = 1 * uint64(MiB) var steps []string for _, mib := range []uint64{40, 48, 46, 45, 44} { bus.fill["KV_m_drafts"] = mib * uint64(MiB) k.FillPass() steps = append(steps, fmt.Sprintf("%d:%d", mib, len(out.events))) } if fmt.Sprint(steps) != "[40:0 48:1 46:1 45:1 44:2]" || fmt.Sprint(out.events) != "[full m drafts full-cleared m drafts]" || fmt.Sprint(out.conds) != "[raised nats.full.m_drafts cleared nats.full.m_drafts]" { t.Errorf("said %v %v: full at 48 of 64 MiB (75%%), kept at 46 and 45 (71%%, 70%%), cleared at 44 (68%%)", out.events, out.conds) } h := HealthOf(k.Status(), k.Held(), 0, k.now()) if !h.Healthy || h.Pass == "none" { t.Errorf("health %+v: healthy, naming its pass", h) } }}, {"the health is unhealthy past its bound, naming why", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { at := k.now() k.FillPass() later := at.Add(4 * time.Minute) if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "last fill pass is 4m0s old") { t.Errorf("health %+v: unhealthy, its pass too old", h) } if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") { t.Errorf("health %+v: names the verdicts waiting", h) } k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") } k.mu.Lock() k.status.FirstView.Answered = time.Time{} k.mu.Unlock() if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second { t.Fatalf("asked again in %s (%v): on the backoff", next, err) } if h := HealthOf(k.Status(), nil, 0, at); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") { t.Errorf("health %+v: unhealthy, naming the verb", h) } }}, // CAT-N8: a person's requests: a drop removes an orphan alone, only on the operator's warrant binding exactly // that drop, after a snapshot, said dropped and recorded; a recreate rebuilds a drifted entry the same way. {"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}}) k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{log}}) var acts []string k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil } ask := asks.Ask{ID: "bus-1", Who: asks.Operator, Expires: k.now().Add(time.Hour)} var w *asks.Warrant k.Warrant = func(string) (*asks.Ask, *asks.Warrant, error) { return &ask, w, nil } put := func(r Request) Request { raw, _ := json.Marshal(r) _, _ = store.Put(StateRequests, r.ID, raw) k.ActOnRequests() raw, _, _, _ = store.Get(StateRequests, r.ID) var got Request _ = json.Unmarshal(raw, &got) return got } if got := put(Request{ID: "r0", Verb: VerbDrop, Module: "m", Local: "log", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "drop removes an orphan alone") { t.Errorf("dropping an asked entry: %+v", got) } if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !got.Done.IsZero() || len(bus.deleted) != 0 { t.Fatalf("acted with no warrant: %+v, deleted %v", got, bus.deleted) } binds := func(act asks.Act) string { d, _ := asks.ActDigest(act); return d } choose := func(act asks.Act) { ask.Options = []asks.Option{{ID: "go", Label: "Remove", Level: asks.Destroy, Binds: binds(act)}} w = &asks.Warrant{Ask: "bus-1", Asker: Module, Outcome: asks.OutcomeChosen, By: &asks.Person{Who: asks.Operator}, Option: "go", Label: "Remove", Level: asks.Destroy, At: k.now(), AskDigest: ask.Digest()} } choose(asks.Act{"verb": "drop", "module": "m", "local": "log"}) if got := put(Request{ID: "r1", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo"}); !strings.Contains(got.Error, "bound") || len(bus.deleted) != 0 { t.Fatalf("acted on a warrant for another act: %+v, deleted %v", got, bus.deleted) } choose(asks.Act{"verb": "drop", "module": "m", "local": "drafts"}) out.reset() got := put(Request{ID: "r2", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", Why: "unused"}) if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts]" || fmt.Sprint(out.events) != "[dropped m drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_drop: dropped drafts of m") || !strings.Contains(acts[0], "person.jo") { t.Errorf("ended %+v, snapshots %v, deleted %v, said %v, recorded %v", got, bus.snaps, bus.deleted, out.events, acts) } if _, made := k.Held()["m"].Made["drafts"]; made { t.Error("the dropped entry is still held") } }}, {"a recreate rebuilds a drifted entry after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{log}}) k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{entry("log", catalogue.StorageStream, catalogue.TemplateLog, map[string]string{"max-mib": "16"})}}) binds, _ := asks.ActDigest(asks.Act{"verb": "recreate", "module": "m", "local": "log"}) ask := asks.Ask{ID: "bus-2", Who: asks.Operator, Expires: k.now().Add(time.Hour), Options: []asks.Option{{ID: "go", Label: "Rebuild", Level: asks.Destroy, Binds: binds}}} w := &asks.Warrant{Ask: "bus-2", Asker: Module, Outcome: asks.OutcomeChosen, By: &asks.Person{Who: asks.Operator}, Option: "go", Label: "Rebuild", Level: asks.Destroy, At: k.now(), AskDigest: ask.Digest()} k.Warrant = func(string) (*asks.Ask, *asks.Warrant, error) { return &ask, w, nil } out.reset() raw, _ := json.Marshal(Request{ID: "r3", Verb: VerbRecreate, Module: "m", Local: "log", Ask: "bus-2"}) _, _ = store.Put(StateRequests, "r3", raw) k.ActOnRequests() if bus.held["m_log"].MaxBytes != 16*MiB || fmt.Sprint(out.events) != "[recreated m log]" || fmt.Sprint(out.conds) != "[cleared nats.drifted.m_log]" || k.Held()["m"].Made["log"].Drift != "" { t.Errorf("held %+v, said %v %v: rebuilt as asked, the drift cleared", bus.held["m_log"], out.events, out.conds) } }}, } { t.Run(row.name, func(t *testing.T) { k, bus, _, out := newKeeper(FirstView{Revision: 1}) if _, err := k.AskFirstView(0); err != nil { t.Fatal(err) } row.run(t, k, bus, out) }) } }