From 0699741fab45093460df0664a4ebfae26905f595 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 04:55:47 +0100 Subject: [PATCH 01/15] NATS1: the bus module plans each state's -objects store (16 MiB, objects-max-mib) and says its binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §58, §91, §513, §591, §607, §549 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/nats/cmd/nats-tools/broker_test.go | 5 ++- modules/nats/internal/keeper/keeper.go | 6 ++- modules/nats/internal/keeper/keeper_test.go | 41 +++++++++++++++------ modules/nats/internal/keeper/plan.go | 19 +++++++++- 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/modules/nats/cmd/nats-tools/broker_test.go b/modules/nats/cmd/nats-tools/broker_test.go index 33a6cd298..6042b3af3 100644 --- a/modules/nats/cmd/nats-tools/broker_test.go +++ b/modules/nats/cmd/nats-tools/broker_test.go @@ -37,6 +37,7 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) { }}} jsz := `{"account_details":[{"stream_detail":[ {"name":"KV_m_drafts","config":{"name":"KV_m_drafts","retention":"limits","discard":"new","max_bytes":67108864,"max_msgs_per_subject":1,"max_msg_size":262144},"state":{"messages":3,"bytes":1048576}}, + {"name":"OBJ_m_drafts-objects","config":{"name":"OBJ_m_drafts-objects","retention":"limits","discard":"new","max_bytes":16777216},"state":{"messages":0,"bytes":0}}, {"name":"m_log","config":{"name":"m_log","retention":"limits","discard":"new","max_bytes":33554432,"max_msg_size":262144},"state":{"messages":9,"bytes":9000}}]}]}` tools := map[string]stdio.Tool{} for _, tl := range brokerTools(func() (map[string]keeper.Asked, error) { return held, nil }, @@ -54,9 +55,9 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) { }{ // Each answers the seat's contract (mesh-contracts meshbroker ListAnswer, Held, CheckAnswer, OrphansAnswer). {"mesh-broker.list", nil, []string{`{"entries":[`, `"bus-name":"m_drafts"`, `"local-name":"drafts"`, `"storage-type":"key-value"`, - `"KV_m_drafts":{"max-bytes":67108864`, `"fill":{"bytes":1048576,"cap":67108864,"count":3}`, `"options":{"max-mib":16}`, + `"KV_m_drafts":{"max-bytes":67108864`, `"fill":{"bytes":1048576,"cap":83886080,"count":3}`, `"options":{"max-mib":16}`, // The binding of an entry the bus holds is the one its created carries (design 25 bindings-first-view §2). - `"binding":{"bucket":"m_drafts","local":"drafts","storage":"key-value","template":"state"}`}, ""}, + `"binding":{"bucket":"m_drafts","local":"drafts","objects":"m_drafts-objects","storage":"key-value","template":"state"}`}, ""}, {"mesh-broker.show", map[string]any{"module": "m", "local-name": "log"}, []string{`"bus-name":"m_log"`, `"template":"log"`, `"m_log":{"max-bytes":33554432`}, ""}, {"mesh-broker.show", map[string]any{"module": "m", "local-name": "later"}, []string{`"local-name":"later"`}, ""}, diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index 4139151fd..a3e6c4a20 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -488,6 +488,9 @@ func (k *Keeper) ensure(held *Asked, m *Made) { defer cancel() asked, holds := map[string]Settings{}, map[string]Settings{} var drifts []string + // madeLater is an object newly made for an entry that was made whole before (a state's `-objects`, design 47 G + // §58): its binding gained a name, so `created` is said again. + madeLater := false for _, o := range m.Plan.Objects { asked[o.Name] = o.Settings s, _, found, err := k.Bus.Held(ctx, o) @@ -509,6 +512,7 @@ func (k *Keeper) ensure(held *Asked, m *Made) { return } holds[o.Name] = o.Settings + madeLater = madeLater || m.Created continue } holds[o.Name] = s @@ -530,7 +534,7 @@ func (k *Keeper) ensure(held *Asked, m *Made) { if wasRefused { k.clear(conditionRefused, held.Module, m.Plan.Local) } - if !m.Created { + if !m.Created || madeLater { // Marked created only once `created` is said whole: the grants part binds on it, so one the bus did not take is // said again on the backoff (Waiting), never marked made and forgotten. if err := k.sayNow(EventCreated, m.Plan, held.Revision, "", asked, holds); err != nil { diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 360c6f5d8..298f57322 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -328,8 +328,13 @@ func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) { refused string }{ {name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}), - objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}", + objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}; object-store mesh-knowledge_drafts-objects {16 MiB, age 0, history 0, new, size 0}", binding: "bucket mesh-knowledge_drafts"}, + {name: "state naming objects-max-mib (§591)", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"objects-max-mib": "512"}), + objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 1, new, size 262144}; object-store mesh-knowledge_drafts-objects {512 MiB, age 0, history 0, new, size 0}", + binding: "bucket mesh-knowledge_drafts"}, + {name: "state naming objects-max-mib 0", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"objects-max-mib": "0"}), + refused: "not a size above 0"}, {name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil), objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}", binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"}, @@ -434,7 +439,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { } k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}}) k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}}) - if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want { + if want := "[KV_m_drafts OBJ_m_drafts-objects m_log]"; fmt.Sprint(bus.created) != want { t.Errorf("created %v, want %s", bus.created, want) } if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want { @@ -455,7 +460,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { k.Sends("m") k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}}) k.Sends("m") - if want := "[OBJ_m_sent-objects KV_m_drafts]"; fmt.Sprint(bus.created) != want { + if want := "[OBJ_m_sent-objects KV_m_drafts OBJ_m_drafts-objects]"; fmt.Sprint(bus.created) != want { t.Fatalf("created %v, want %s", bus.created, want) } if s := bus.held["OBJ_m_sent-objects"]; s.MaxBytes != 256*MiB || s.MaxAgeSeconds != 7*24*3600 || s.Discard != DiscardNew { @@ -543,10 +548,22 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds) } }}, - {"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + {"a state made before its objects store existed gets the store and its binding said again (§58)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { p, _ := PlanOf("m", drafts) bus.held["KV_m_drafts"] = p.Objects[0].Settings k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) + out.reset() + bus.created = nil + delete(bus.held, "OBJ_m_drafts-objects") + k.CheckAll() + if fmt.Sprint(bus.created) != "[OBJ_m_drafts-objects]" || fmt.Sprint(out.events) != "[created m drafts]" || out.bodies[0].Binding.Objects != "m_drafts-objects" { + t.Errorf("created %v, said %v: the missing store made, its binding said again", bus.created, out.events) + } + }}, + {"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + p, _ := PlanOf("m", drafts) + bus.held["KV_m_drafts"], bus.held["OBJ_m_drafts-objects"] = p.Objects[0].Settings, p.Objects[1].Settings + k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" { t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events) } @@ -696,7 +713,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { bus.snaps = nil out.reset() got := put(Request{ID: "r2", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", Why: "unused"}) - if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts]" || + if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts OBJ_m_drafts-objects]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts OBJ_m_drafts-objects]" || fmt.Sprint(out.events) != "[dropped m drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_drop: dropped drafts of m") || !strings.Contains(acts[0], "person.jo") { t.Errorf("ended %+v, snapshots %v, deleted %v, said %v, recorded %v", got, bus.snaps, bus.deleted, out.events, acts) @@ -733,7 +750,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { out.reset() k.ActOnRequests() got := get("r10") - if got.Path != "/snapshots/KV_m_drafts.tar" || got.Size != 100 || !strings.HasPrefix(got.Digest, "sha256:") || !got.Protected || got.Error != "" { + if got.Path != "/snapshots/KV_m_drafts+OBJ_m_drafts-objects.tar" || got.Size != 100 || !strings.HasPrefix(got.Digest, "sha256:") || !got.Protected || got.Error != "" { t.Fatalf("the drop's record: %+v", got) } findings := k.Finding.(*findingLog) @@ -753,7 +770,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { k.Now = func() time.Time { return base.AddDate(0, 0, 91) } findings.calls = nil k.ExpireSnapshots() - if _, _, found, _ := store.Get(StateRequests, "r10"); found || fmt.Sprint(files.removed) != "[/snapshots/m_log.tar /snapshots/KV_m_drafts.tar]" || + if _, _, found, _ := store.Get(StateRequests, "r10"); found || fmt.Sprint(files.removed) != "[/snapshots/m_log.tar /snapshots/KV_m_drafts+OBJ_m_drafts-objects.tar]" || len(journal) != 1 || !strings.Contains(journal[0], "past its 90 days") || fmt.Sprint(findings.calls) != "[clear module.nats.snapshots.snapshots-filling]" { t.Errorf("after 91 days: removed %v, journal %v, finding calls %v", files.removed, journal, findings.calls) } @@ -767,7 +784,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { {"a snapshot is restored in place, verified, refused while its names are live, cancelled, and its failure removes the partial stream", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store, files := k.Store.(*memStore), k.Files.(*memFiles) out.sig = make(chan struct{}, 8) - path := "/snapshots/KV_m_drafts.tar" + path := "/snapshots/KV_m_drafts+OBJ_m_drafts-objects.tar" bus.snapStreams = map[string][]string{path: {"KV_m_drafts"}} files.missing = map[string]bool{"/snapshots/gone.tar": true} digest := "sha256:" + hex.EncodeToString([]byte(path)) @@ -942,13 +959,13 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { _ = json.Unmarshal(raw, &got) return got } - if got := put("r5"); got.Error != "" || len(bus.deleted) != 1 { + if got := put("r5"); got.Error != "" || len(bus.deleted) != 2 { t.Fatalf("the first drop: %+v, deleted %v", got, bus.deleted) } // The entry is asked for and orphaned again, so a second drop would remove it: the spent warrant refuses it. k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}}) k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{log}}) - if got := put("r6"); !strings.Contains(got.Error, "already acted (drop of m's drafts, request r5") || len(bus.deleted) != 1 { + if got := put("r6"); !strings.Contains(got.Error, "already acted (drop of m's drafts, request r5") || len(bus.deleted) != 2 { t.Errorf("the second request on the same warrant: %+v, deleted %v", got, bus.deleted) } if raw, _, found, _ := store.Get(StateSpent, "bus-5"); !found || !strings.Contains(string(raw), `"request":"r5"`) { @@ -960,7 +977,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { for _, key := range keys { _ = store.Delete(StateRequests, key) } - if got := put("r6b"); !strings.Contains(got.Error, "already acted") || len(bus.deleted) != 1 { + if got := put("r6b"); !strings.Contains(got.Error, "already acted") || len(bus.deleted) != 2 { t.Errorf("with requests wiped, the spent warrant: %+v, deleted %v", got, bus.deleted) } }}, @@ -1080,7 +1097,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { raw, _, _, _ = store.Get(StateRequests, "r9") var got Request _ = json.Unmarshal(raw, &got) - if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts]" || + if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts OBJ_m_drafts-objects]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked mesh-delivery-indexer.indexer_bus_state again by hand") { t.Errorf("ended %+v, created %v, recorded %v", got, bus.created, acts) } diff --git a/modules/nats/internal/keeper/plan.go b/modules/nats/internal/keeper/plan.go index ad89ae4e9..baa6a3e88 100644 --- a/modules/nats/internal/keeper/plan.go +++ b/modules/nats/internal/keeper/plan.go @@ -49,6 +49,10 @@ const ( SentAgeDays = 7 ) +// StateObjectsCapMiB is the cap of a state's object store `-objects` unless the state names `objects-max-mib` +// (design 47 G §591, §547). +const StateObjectsCapMiB = 16 + // Sends says whether a module sends (design 47 G §344): it claims a seat whose definition has a verb or an event, or it // declares an event of its own (emitsOwn). claims are the seats its catalog graph entry claims, defs the seat list's // definitions by seat; a claimed seat the list does not hold counts for nothing. A definition that includes parts takes @@ -156,8 +160,19 @@ func PlanOf(module string, e catalogue.BusStateEntry) (Plan, error) { if err != nil { return Plan{}, err } - p.Binding.Bucket = name - p.Objects = []Object{{Kind: KindBucket, Name: name, Settings: s}} + // Beside each state its object store `-objects`, which the grants part grants to the state's owner alone (design + // 47 G §58, §591): 16 MiB unless the state's requires names `objects-max-mib`. + objCap, err := opts.int(catalogue.OptionObjectsCapMiB, StateObjectsCapMiB) + if err != nil { + return Plan{}, err + } + if objCap <= 0 { + return Plan{}, fmt.Errorf("the option %s of %q is %d, not a size above 0", catalogue.OptionObjectsCapMiB, e.Name, objCap) + } + objects := BusName(module, catalogue.StateObjectsName(e.Name)) + p.Binding.Bucket, p.Binding.Objects = name, objects + p.Objects = []Object{{Kind: KindBucket, Name: name, Settings: s}, + {Kind: KindObjects, Name: objects, Settings: Settings{MaxBytes: objCap * MiB, Discard: DiscardNew}}} case e.Storage == catalogue.StorageStream && e.Template == catalogue.TemplateEvents: age, err := opts.int(catalogue.OptionAgeDays, EventsDays) if err != nil { From 24778d24b63985026a32fe003fe4e5f2542c74ed Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 04:56:56 +0100 Subject: [PATCH 02/15] NATS8: the bus module's ask lasts its level's bound and a refused ask ends its request with the router's words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §570 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/nats/cmd/nats-tools/requests_test.go | 18 ++++++++++++++++++ modules/nats/internal/keeper/keeper_test.go | 18 +++++++++++++++++- modules/nats/internal/keeper/requests.go | 15 +++++++++++---- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/modules/nats/cmd/nats-tools/requests_test.go b/modules/nats/cmd/nats-tools/requests_test.go index 63148231e..474f06fbb 100644 --- a/modules/nats/cmd/nats-tools/requests_test.go +++ b/modules/nats/cmd/nats-tools/requests_test.go @@ -145,3 +145,21 @@ func TestAPersonsRequestIsHandedToTheKeeperAsTheProvedCaller(t *testing.T) { t.Errorf("an end not written: %+v on %d watch(es)", r, watches) } } + +// The ask for a recreate or a drop lasts what its highest option level allows (design 47 G §570), so the asker client +// takes it: an expiry at the bound passes occ.Check, and the fixed 12 hours it was is refused. +func TestTheAskLastsWhatItsLevelAllows(t *testing.T) { + now := time.Now() + r := keeper.Request{ID: "r1", Verb: keeper.VerbDrop, Module: "m", Local: "drafts", Why: "unused", Caller: "person.jo"} + a, err := askFor(r, now) + if err != nil { + t.Fatal(err) + } + if err := occ.Check(a, now); err != nil { + t.Errorf("the ask is refused by the asker client: %v", err) + } + a.Expires = occ.FormatTime(now.Add(12 * time.Hour)) + if err := occ.Check(a, now); err == nil || !strings.Contains(err.Error(), "at most") { + t.Errorf("12 hours is held to the level's bound: %v", err) + } +} diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 298f57322..31a5105b5 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -656,6 +656,22 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { }}, // CAT-N8: a person's requests: a drop removes an orphan alone, only on the operator's warrant binding exactly // that drop, after a snapshot, said dropped and recorded; a recreate rebuilds a drifted entry the same way. + {"an ask the router refused ends its request with the router's words (§570)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + store := k.Store.(*memStore) + k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) + k.Heard(Accepted{Module: "m", Revision: 2, BusState: nil}) + w := &oc.Warrant{Ask: "bus-1", Asker: Module, Outcome: oc.WarrantOutcomeRefused, Words: "an ask that destroys lasts at most 10m0s"} + k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return nil, w, nil } + raw, _ := json.Marshal(Request{ID: "rr", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", At: k.now()}) + _, _ = store.Put(StateRequests, "rr", raw) + k.ActOnRequests() + raw, _, _, _ = store.Get(StateRequests, "rr") + var got Request + _ = json.Unmarshal(raw, &got) + if got.Done.IsZero() || got.Waiting != "" || !strings.Contains(got.Error, "refused") || !strings.Contains(got.Error, w.Words) || len(bus.deleted) != 0 { + t.Errorf("ended %+v, deleted %v: the router's refusal and its words, nothing removed", got, bus.deleted) + } + }}, {"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { store := k.Store.(*memStore) k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}}) @@ -994,7 +1010,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { _ = json.Unmarshal(raw, &got) return got } - raw, _ := json.Marshal(Request{ID: "r7", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-7", Caller: "person.jo", At: k.now().Add(-time.Hour)}) + raw, _ := json.Marshal(Request{ID: "r7", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-7", Caller: "person.jo", At: k.now().Add(-AskLasts / 2)}) _, _ = store.Put(StateRequests, "r7", raw) raw, _ = json.Marshal(Request{ID: "r8", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-8", Caller: "person.jo", At: k.now().Add(-AskLasts - time.Minute)}) _, _ = store.Put(StateRequests, "r8", raw) diff --git a/modules/nats/internal/keeper/requests.go b/modules/nats/internal/keeper/requests.go index ef07a9742..b9b87803e 100644 --- a/modules/nats/internal/keeper/requests.go +++ b/modules/nats/internal/keeper/requests.go @@ -9,6 +9,7 @@ import ( "time" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" ) // A person's request (design 25 § 12.3; CAT-N8): the bus module's tools hold no grant of the seat's acting verbs, so @@ -67,10 +68,11 @@ type Request struct { Unrecorded string `json:"unrecorded,omitempty"` } -// AskLasts is how long the operator's ask for a recreate or a drop waits for an answer: shorter than the day the state -// requests keeps a key, so a request whose ask ended unanswered is ended by the keeper, saying so, before its key ages -// out, and the warrant's acted key (ActedKey) outlives the ask it spends. -const AskLasts = 12 * time.Hour +// AskLasts is how long the operator's ask for a recreate or a drop waits for an answer: the bound of its highest option +// level, destroy (design 47 G §570: no asker asks for longer than its level allows; the asker client refuses more). It is +// shorter than the day the state requests keeps a key, so a request whose ask ended unanswered is ended by the keeper, +// saying so, before its key ages out, and the warrant's acted key (ActedKey) outlives the ask it spends. +const AskLasts = occ.DestroyLasts // ActedPrefix started the spend record's key while it was kept in StateRequests (`acted.`); a key of that form // still met there is no request and is skipped. @@ -288,6 +290,11 @@ func (k *Keeper) warranted(r *Request) error { if err != nil { return fmt.Errorf("%w: the router's record of the ask %s cannot be read now: %v", ErrNoWarrant, r.Ask, err) } + // An ask the router refused at take-in ends its request with the router's words, never "not answered yet" (design 47 G + // §570). + if w != nil && w.Outcome == oc.WarrantOutcomeRefused { + return fmt.Errorf("the router refused the operator's ask %s, so nothing is removed: %s", r.Ask, w.Words) + } // An ask that ended unanswered ends its request, said: never a request that waits until its key ages out. And a // warrant is not acted on once its ask ended, so a request handed after its acted key aged out acts on nothing. var expires time.Time From 78b12be7528c964af6d17438403d39f5bfbcae53 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 04:59:19 +0100 Subject: [PATCH 03/15] NATS5: the bus module reads the bus's storage limits, refuses a plan whose caps pass the account's and shows the sum in list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §547, §495, §661 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/nats/cmd/nats-tools/broker.go | 18 +++++- modules/nats/cmd/nats-tools/broker_test.go | 18 +++++- modules/nats/cmd/nats-tools/keep.go | 2 +- modules/nats/cmd/nats-tools/main.go | 2 +- modules/nats/cmd/nats-tools/monitor.go | 21 +++++++ modules/nats/internal/keeper/fill.go | 5 ++ modules/nats/internal/keeper/jetstream.go | 21 +++++++ modules/nats/internal/keeper/keeper.go | 68 +++++++++++++++++++++ modules/nats/internal/keeper/keeper_test.go | 27 ++++++++ 9 files changed, 177 insertions(+), 5 deletions(-) diff --git a/modules/nats/cmd/nats-tools/broker.go b/modules/nats/cmd/nats-tools/broker.go index 6bddf5e5d..9647be55b 100644 --- a/modules/nats/cmd/nats-tools/broker.go +++ b/modules/nats/cmd/nats-tools/broker.go @@ -48,6 +48,16 @@ func HeldInState() (map[string]keeper.Asked, error) { return out, nil } +// capsOf is the sum of the bus's caps against its limits (design 47 G §547, §661), in MiB; a limit the keeper has not read, +// or that is none, is absent. The sum is over every module, whatever list is narrowed to. +func capsOf(all map[string]keeper.Asked, limits func() (keeper.Limits, error)) *meshbroker.ListAnswerCaps { + c := &meshbroker.ListAnswerCaps{SumMiB: keeper.CapSum(all, "", "") / keeper.MiB} + if l, err := limits(); err == nil { + c.AccountLimitMiB, c.ServerLimitMiB = l.AccountBytes/keeper.MiB, l.ServerBytes/keeper.MiB + } + return c +} + // busStream is one stream as the monitoring endpoint answers it. type busStream struct { Config jetstream.StreamConfig `json:"config"` @@ -157,7 +167,7 @@ func anyOf(m map[string]json.RawMessage) map[string]any { } // brokerTools are the four reading verbs and nats_history. -func brokerTools(read heldEntries, jsz func() ([]byte, error), history func(module, local string, limit int) ([]keeper.Event, error)) []stdio.Tool { +func brokerTools(read heldEntries, limits func() (keeper.Limits, error), jsz func() ([]byte, error), history func(module, local string, limit int) ([]keeper.Event, error)) []stdio.Tool { both := func() (map[string]keeper.Asked, map[string]busStream, error) { entries, err := read() if err != nil { @@ -194,7 +204,11 @@ func brokerTools(read heldEntries, jsz func() ([]byte, error), history func(modu if m := arg(a, "module"); m != "" { entries = map[string]keeper.Asked{m: entries[m]} } - return meshbroker.ListAnswer{Entries: each(entries, held, func(keeper.Made) bool { return true })}, nil + all, err := read() + if err != nil { + return nil, err + } + return meshbroker.ListAnswer{Caps: capsOf(all, limits), Entries: each(entries, held, func(keeper.Made) bool { return true })}, nil }}, {Name: "mesh-broker.show", Description: "One entry of a module's kept data on the bus: its bus name, template and options as asked, the settings the bus holds of each of its objects, and its fill against its cap. Whether it is refused, waiting, drifted, full or orphaned is what nats_history says of it.", diff --git a/modules/nats/cmd/nats-tools/broker_test.go b/modules/nats/cmd/nats-tools/broker_test.go index 6042b3af3..b3fb30360 100644 --- a/modules/nats/cmd/nats-tools/broker_test.go +++ b/modules/nats/cmd/nats-tools/broker_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "reflect" @@ -41,6 +42,9 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) { {"name":"m_log","config":{"name":"m_log","retention":"limits","discard":"new","max_bytes":33554432,"max_msg_size":262144},"state":{"messages":9,"bytes":9000}}]}]}` tools := map[string]stdio.Tool{} for _, tl := range brokerTools(func() (map[string]keeper.Asked, error) { return held, nil }, + func() (keeper.Limits, error) { + return keeper.Limits{AccountBytes: 2048 << 20, ServerBytes: 4096 << 20}, nil + }, func() ([]byte, error) { return []byte(jsz), nil }, func(module, local string, limit int) ([]keeper.Event, error) { return []keeper.Event{{Module: module, Local: local, Why: "first"}, {Module: module, Local: local, Why: "second"}}[:limit], nil @@ -54,10 +58,12 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) { err string }{ // Each answers the seat's contract (mesh-contracts meshbroker ListAnswer, Held, CheckAnswer, OrphansAnswer). - {"mesh-broker.list", nil, []string{`{"entries":[`, `"bus-name":"m_drafts"`, `"local-name":"drafts"`, `"storage-type":"key-value"`, + {"mesh-broker.list", nil, []string{`"entries":[`, `"bus-name":"m_drafts"`, `"local-name":"drafts"`, `"storage-type":"key-value"`, `"KV_m_drafts":{"max-bytes":67108864`, `"fill":{"bytes":1048576,"cap":83886080,"count":3}`, `"options":{"max-mib":16}`, // The binding of an entry the bus holds is the one its created carries (design 25 bindings-first-view §2). `"binding":{"bucket":"m_drafts","local":"drafts","objects":"m_drafts-objects","storage":"key-value","template":"state"}`}, ""}, + // The sum of the caps against both limits (design 47 G §547): drafts 64+16, log 16, old 1024 (orphaned, still held); later is not made. + {"mesh-broker.list", nil, []string{`"caps":{"account-limit-mib":2048,"server-limit-mib":4096,"sum-mib":1120`}, ""}, {"mesh-broker.show", map[string]any{"module": "m", "local-name": "log"}, []string{`"bus-name":"m_log"`, `"template":"log"`, `"m_log":{"max-bytes":33554432`}, ""}, {"mesh-broker.show", map[string]any{"module": "m", "local-name": "later"}, []string{`"local-name":"later"`}, ""}, @@ -115,3 +121,13 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) { }) } } + +// The server's storage limit is read from the monitoring document's config (design 47 G §547); none is 0. +func TestTheServersStorageLimitIsReadFromItsMonitor(t *testing.T) { + for doc, want := range map[string]int64{`{"config":{"max_storage":5368709120}}`: 5368709120, `{"config":{"max_storage":-1}}`: 0, `{}`: 0} { + got, err := serverStorageLimit(func(context.Context, string) ([]byte, error) { return []byte(doc), nil })(context.Background()) + if err != nil || got != want { + t.Errorf("%s: %d (%v), want %d", doc, got, err, want) + } + } +} diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index 2547bf5b1..fa2f82cbd 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -267,7 +267,7 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer defer writer.Close() k := &keeper.Keeper{ Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR")), NC: nc, - Dir: getenv("MESH_NATS_SNAPSHOTS_DIR"), Log: logger.Printf}, + Dir: getenv("MESH_NATS_SNAPSHOTS_DIR"), Log: logger.Printf, Server: serverStorageLimit(LiveMonitor())}, Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn), Act: func(what, why, cause string) error { _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause}) diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index b20eefcba..c915c3f55 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -192,7 +192,7 @@ func allTools() []stdio.Tool { } out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState)...) out = append(out, expiryTools(LiveBus(), SaidByTheRuntime, RecordedBySeat)...) - out = append(out, brokerTools(HeldInState, jsz, StreamHistory(LiveBus()))...) + out = append(out, brokerTools(HeldInState, func() (keeper.Limits, error) { s, err := StatusInState(); return s.Limits, err }, jsz, StreamHistory(LiveBus()))...) out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz)) out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...) rs := RequestsInState() diff --git a/modules/nats/cmd/nats-tools/monitor.go b/modules/nats/cmd/nats-tools/monitor.go index 191e7b16c..961fcdbf7 100644 --- a/modules/nats/cmd/nats-tools/monitor.go +++ b/modules/nats/cmd/nats-tools/monitor.go @@ -11,6 +11,7 @@ package main import ( "context" + "encoding/json" "fmt" "io" "net/http" @@ -86,3 +87,23 @@ func ParseWritten(s string) (time.Time, error) { } return t, nil } + +// serverStorageLimit reads the server's own JetStream storage limit in bytes from its monitoring endpoint (`config`'s +// max_storage; 0 or less is none), shown beside the account's (design 47 G §547). +func serverStorageLimit(monitor Monitor) func(ctx context.Context) (int64, error) { + return func(ctx context.Context) (int64, error) { + raw, err := monitor(ctx, "/jsz?config=1") + if err != nil { + return 0, err + } + var doc struct { + Config struct { + MaxStorage int64 `json:"max_storage"` + } `json:"config"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return 0, err + } + return max(doc.Config.MaxStorage, 0), nil + } +} diff --git a/modules/nats/internal/keeper/fill.go b/modules/nats/internal/keeper/fill.go index ae062a478..87a9de1e7 100644 --- a/modules/nats/internal/keeper/fill.go +++ b/modules/nats/internal/keeper/fill.go @@ -21,6 +21,11 @@ func (k *Keeper) FillPass() { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() + if lim, err := k.Bus.Limits(ctx); err == nil { + k.setStatus(func(s *Status) { s.Limits = lim }) + } else { + k.logf("the bus's storage limits cannot be read: %v", err) + } for _, module := range keys { held, _, found := k.asked(module) if !found { diff --git a/modules/nats/internal/keeper/jetstream.go b/modules/nats/internal/keeper/jetstream.go index c3e20079f..c5bd4b628 100644 --- a/modules/nats/internal/keeper/jetstream.go +++ b/modules/nats/internal/keeper/jetstream.go @@ -23,6 +23,27 @@ type JetStream struct { NC *nats.Conn Dir string Log func(string, ...any) + // Server reads the server's own storage limit in bytes (0 none), from its monitoring endpoint. + Server func(ctx context.Context) (int64, error) +} + +// Limits reads the account's storage limit (-1 is none) and the server's (design 47 G §547). +func (b JetStream) Limits(ctx context.Context) (Limits, error) { + info, err := b.JS.AccountInfo(ctx) + if err != nil { + return Limits{}, err + } + var l Limits + if info.Limits.MaxStore > 0 { + l.AccountBytes = info.Limits.MaxStore + } + if b.Server != nil { + // The server's limit is only shown, never held against (§547): one that cannot be read is shown as none. + if n, err := b.Server(ctx); err == nil { + l.ServerBytes = n + } + } + return l, nil } // Held reads the object's stream. diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index a3e6c4a20..a25ba9e08 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -127,6 +127,8 @@ type Status struct { Started time.Time `json:"started"` // Pass is when the last fill pass ended. Pass time.Time `json:"pass"` + // Limits are the bus's storage limits as last read (design 47 G §547): what `list` shows against the sum of the caps. + Limits Limits `json:"limits,omitempty"` // FirstView is the indexer's first view: when asked, when answered and at which revision, or why not. FirstView FirstViewStatus `json:"first-view"` // Heard is when the last manifest-accepted was heard, and Unanswered since when one has waited unhandled. @@ -193,6 +195,54 @@ type Bus interface { // keeper's own partial stream of a restore that did not end). Restore(ctx context.Context, path string) error DeleteStream(ctx context.Context, name string) error + // Limits reads the bus's storage limits: the keeper's account's and the server's (design 47 G §547). + Limits(ctx context.Context) (Limits, error) +} + +// Limits are the bus's storage limits in bytes; 0 is no limit (an unlimited account, or a server that sets none). +type Limits struct { + AccountBytes int64 `json:"account-bytes,omitempty"` + ServerBytes int64 `json:"server-bytes,omitempty"` +} + +// CapSum is the sum, in bytes, of the caps of every object the entries made hold, but for the entry of skipModule and +// skipLocal (design 47 G §547): what the bus's limits are read against. An entry never made, or refused, holds none. +func CapSum(held map[string]Asked, skipModule, skipLocal string) int64 { + var sum int64 + for module, a := range held { + for local, m := range a.Made { + if (module == skipModule && local == skipLocal) || !(m.Created || m.Orphaned) { + continue + } + for _, o := range m.Plan.Objects { + sum += o.Settings.MaxBytes + } + } + } + return sum +} + +// capRoom refuses a plan that adds or raises a cap when the sum of the bus's caps would pass the account's limit, +// naming both (design 47 G §547); caps already made stay as they are. With no account limit nothing is refused. +func (k *Keeper) capRoom(ctx context.Context, held *Asked, m *Made) error { + lim, err := k.Bus.Limits(ctx) + if err != nil { + return fmt.Errorf("%w: the bus's storage limits cannot be read: %v", ErrRetry, err) + } + k.setStatus(func(s *Status) { s.Limits = lim }) + if lim.AccountBytes <= 0 { + return nil + } + all := k.Held() + all[held.Module] = *held + sum := CapSum(all, held.Module, m.Plan.Local) + for _, o := range m.Plan.Objects { + sum += o.Settings.MaxBytes + } + if sum > lim.AccountBytes { + return fmt.Errorf("the plan takes the sum of the bus's caps to %d MiB, past the account's storage limit of %d MiB", sum/MiB, lim.AccountBytes/MiB) + } + return nil } // Fill is how much one object holds: its bytes and messages. @@ -499,6 +549,18 @@ func (k *Keeper) ensure(held *Asked, m *Made) { return } if !found { + if err := k.capRoom(ctx, held, m); err != nil { + if errors.Is(err, ErrRetry) { + m.Waiting = err.Error() + return + } + m.Refused, m.Waiting = fmt.Sprintf("%s: %v", o.Stream(), err), "" + k.emit(EventRefused, m.Plan, held.Revision, m.Refused, asked, nil) + k.raise(conditionRefused, held.Module, m.Plan.Local, occ.SeverityWarning, "The bus module refused to create kept data", + "a smaller cap, or more storage for the bus's account", + fmt.Sprintf("%s of %s was not created: %s. The module gets no binding for it and fails at its start, naming it", m.Plan.Local, held.Module, m.Refused)) + return + } if err := k.Bus.Create(ctx, o); err != nil { if errors.Is(err, ErrRetry) { m.Waiting = fmt.Sprintf("%s cannot be created yet: %v", o.Stream(), err) @@ -520,6 +582,12 @@ func (k *Keeper) ensure(held *Asked, m *Made) { continue } if Grows(o.Settings, s) { + if o.Settings.MaxBytes > s.MaxBytes { + if err := k.capRoom(ctx, held, m); err != nil { + m.Waiting = err.Error() + return + } + } if err := k.Bus.Update(ctx, o); err != nil { m.Waiting = fmt.Sprintf("%s's larger settings cannot be applied yet: %v", o.Stream(), err) return diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 31a5105b5..30c07cb47 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -38,6 +38,8 @@ type fakeBus struct { restored []string gate chan struct{} deletedStreams []string + // limits are the bus's storage limits the fake answers. + limits Limits } func (b *fakeBus) SnapshotStreams(path string) ([]string, error) { @@ -73,6 +75,12 @@ func (b *fakeBus) DeleteStream(_ context.Context, name string) error { return nil } +func (b *fakeBus) Limits(context.Context) (Limits, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.limits, nil +} + func newFakeBus() *fakeBus { return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}} } @@ -548,6 +556,25 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds) } }}, + {"a plan whose caps pass the account's storage limit is refused, naming the sum and the limit, and the caps made stay (§547)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + // drafts: 64 MiB state + 16 MiB objects = 80 MiB; log: 1024 MiB. + bus.limits = Limits{AccountBytes: 100 * MiB} + k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) + k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{drafts, log}}) + m := k.Held()["m"].Made + if fmt.Sprint(bus.created) != "[KV_m_drafts OBJ_m_drafts-objects]" || !m["drafts"].Created || m["log"].Created || + !strings.Contains(m["log"].Refused, "1104 MiB, past the account's storage limit of 100 MiB") || fmt.Sprint(out.events) != "[created m drafts refused m log]" { + t.Errorf("created %v, said %v, log %+v: the caps made stay, the plan that passes the limit refused naming both", bus.created, out.events, m["log"]) + } + if CapSum(k.Held(), "", "") != 80*MiB { + t.Errorf("the sum is %d, want 80 MiB: a refused entry holds none", CapSum(k.Held(), "", "")) + } + bus.limits = Limits{} + k.CheckAll() + if !k.Held()["m"].Made["log"].Created { + t.Errorf("with no account limit nothing is refused: %+v", k.Held()["m"].Made["log"]) + } + }}, {"a state made before its objects store existed gets the store and its binding said again (§58)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { p, _ := PlanOf("m", drafts) bus.held["KV_m_drafts"] = p.Objects[0].Settings From 62770674657a5618eaa9747121e6f6d59d274a76 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:00:34 +0100 Subject: [PATCH 04/15] NATS6: the bus module keeps its plans and plans again only when a module's entries differ; nats_plans reads them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §538 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/nats/cmd/nats-tools/broker.go | 54 ++++++++++++++++++++ modules/nats/cmd/nats-tools/broker_test.go | 16 ++++++ modules/nats/cmd/nats-tools/main.go | 1 + modules/nats/cmd/nats-tools/manifest_test.go | 8 +-- modules/nats/internal/keeper/keeper.go | 52 +++++++++++++++++++ modules/nats/internal/keeper/keeper_test.go | 26 ++++++++++ modules/nats/module.json | 9 +++- 7 files changed, 161 insertions(+), 5 deletions(-) diff --git a/modules/nats/cmd/nats-tools/broker.go b/modules/nats/cmd/nats-tools/broker.go index 9647be55b..a78b45d97 100644 --- a/modules/nats/cmd/nats-tools/broker.go +++ b/modules/nats/cmd/nats-tools/broker.go @@ -331,3 +331,57 @@ func sortedNames[V any](m map[string]V) []string { sort.Strings(out) return out } + +// PlansInState reads the keeper's state plans (design 47 G §538): each module's entries as last planned. +func PlansInState() (map[string]keeper.Planned, error) { + s := stdio.State(keeper.StatePlans) + keys, err := s.Keys() + if err != nil { + return nil, fmt.Errorf("the keeper's own %s cannot be read: %w", keeper.StatePlans, err) + } + out := map[string]keeper.Planned{} + for _, key := range keys { + e, err := s.Get(key) + if err != nil || e == nil { + continue + } + var p keeper.Planned + if json.Unmarshal(e.Value, &p) == nil { + out[key] = p + } + } + return out, nil +} + +// plansTool is nats_plans: what the bus module planned for each module, from its own state plans (design 47 G §538). +func plansTool(read func() (map[string]keeper.Planned, error)) stdio.Tool { + return stdio.Tool{Name: "nats_plans", + Description: "What the bus module planned for each module: the bus-state entries it last planned, their digest (a verdict whose entries digest the same plans nothing), the revision of the verdict and when. Narrowed by a module.", + Input: map[string]any{"module": str("one module")}, + Run: func(a map[string]any) (any, error) { + plans, err := read() + if err != nil { + return nil, err + } + type line struct { + Module string `json:"module"` + Digest string `json:"digest"` + Revision uint64 `json:"revision"` + Entries []string `json:"entries"` + At string `json:"at"` + } + out := []line{} + for _, module := range sortedNames(plans) { + if m := arg(a, "module"); m != "" && m != module { + continue + } + p := plans[module] + names := []string{} + for _, e := range p.Entries { + names = append(names, e.Name) + } + out = append(out, line{Module: module, Digest: p.Digest, Revision: p.Revision, Entries: names, At: p.At.UTC().Format(time.RFC3339)}) + } + return map[string]any{"plans": out}, nil + }} +} diff --git a/modules/nats/cmd/nats-tools/broker_test.go b/modules/nats/cmd/nats-tools/broker_test.go index b3fb30360..a30b5407b 100644 --- a/modules/nats/cmd/nats-tools/broker_test.go +++ b/modules/nats/cmd/nats-tools/broker_test.go @@ -131,3 +131,19 @@ func TestTheServersStorageLimitIsReadFromItsMonitor(t *testing.T) { } } } + +// nats_plans answers each module's entries as last planned, narrowed by a module (design 47 G §538). +func TestNatsPlansReadsTheKeepersPlans(t *testing.T) { + read := func() (map[string]keeper.Planned, error) { + return map[string]keeper.Planned{ + "a": {Module: "a", Digest: "sha256:1", Revision: 3, Entries: []catalogue.BusStateEntry{{Name: "drafts"}}}, + "b": {Module: "b", Digest: "sha256:2", Revision: 4}, + }, nil + } + tool := plansTool(read) + out, err := tool.Run(map[string]any{"module": "a"}) + raw, _ := json.Marshal(out) + if err != nil || !strings.Contains(string(raw), `"entries":["drafts"]`) || !strings.Contains(string(raw), `"digest":"sha256:1"`) || strings.Contains(string(raw), `"module":"b"`) { + t.Errorf("nats_plans answered %s (%v)", raw, err) + } +} diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index c915c3f55..1f5e82f4b 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -197,5 +197,6 @@ func allTools() []stdio.Tool { out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...) rs := RequestsInState() out = append(out, snapshotListTool(rs, int(number(os.Getenv, "MESH_NATS_SNAPSHOT_KEEP_DAYS", keeper.DefaultSnapshotKeepDays)))...) + out = append(out, plansTool(PlansInState)) return append(out, requestTools(rs)...) } diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index 998cb2a9f..3e3214030 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -228,7 +228,7 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { } // spent, the warrants the keeper spent, is written by the keeper alone and only read by the tools (review round 2, // security F2: the tool runner's account, which writes requests, may never delete a spend record). - if !slices.Equal(states, []string{"asked", "requests", "spent", "findings"}) || !slices.Equal(readOnly, []string{"spent"}) { + if !slices.Equal(states, []string{"asked", "requests", "spent", "plans", "findings"}) || !slices.Equal(readOnly, []string{"spent", "plans"}) { t.Errorf("state is %v, read-only %v: asked and requests, old sections (ADR 0595 §10), and spent, read-only", states, readOnly) } if _, ok := nm.OwnSecrets["broker-keeper"]; !ok { @@ -238,7 +238,7 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { bus, _ := keeper["bus"].(map[string]any) health, _ := keeper["health"].(map[string]any) if keeper["type"] != "process" || keeper["artifact"] != "tools" || fmt.Sprint(keeper["run"]) != "[./nats-tools keep]" || - fmt.Sprint(bus["state"]) != "[asked requests spent findings]" || + fmt.Sprint(bus["state"]) != "[asked requests spent plans findings]" || health["kind"] != "tool" || health["tool"] != "nats_keeper_health" { t.Errorf("the keeper is %v: a process of the tools bundle, run as keep, keeping asked, requests and spent, judged by nats_keeper_health", keeper) } @@ -248,8 +248,8 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { t.Fatal(err) } for _, st := range whole.State { - if (st.Name == "spent") != st.ReadOnly { - t.Errorf("the SDK reads %s read-only %v: spent alone is read-only", st.Name, st.ReadOnly) + if (st.Name == "spent" || st.Name == "plans") != st.ReadOnly { + t.Errorf("the SDK reads %s read-only %v: spent and plans alone are read-only", st.Name, st.ReadOnly) } } if b := whole.ProcessBuses()["keeper"]; !b.Keeps { diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index a25ba9e08..a6b69975b 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -3,6 +3,7 @@ package keeper import ( "context" "crypto/rand" + "crypto/sha256" "encoding/hex" "encoding/json" "errors" @@ -26,6 +27,9 @@ const Module = "nats" const ( StateAsked = "asked" StateRequests = "requests" + // StatePlans holds, per module, the bus-state entries as last planned and their digest (design 47 G §538): what a + // verdict's entries are compared with, never the manifest's digest, which changes with any field. + StatePlans = "plans" // KeyStatus is the keeper's own status in StateAsked, read by its health tool: a key no module can have, since a // module's name holds no underscore. KeyStatus = "_keeper" @@ -92,6 +96,24 @@ type FirstView struct { Modules []Accepted `json:"modules"` } +// Planned is the value of a module's key in StatePlans: the entries as planned, their digest and the revision of the verdict. +type Planned struct { + V int `json:"v"` + Module string `json:"module"` + Digest string `json:"digest"` + Revision uint64 `json:"revision"` + Entries []catalogue.BusStateEntry `json:"entries"` + At time.Time `json:"at"` +} + +// EntriesDigest is the digest of bus-state entries alone (design 47 G §538): equal entries have an equal digest whatever +// else a manifest changed. +func EntriesDigest(entries []catalogue.BusStateEntry) string { + raw, _ := json.Marshal(entries) + sum := sha256.Sum256(raw) + return "sha256:" + hex.EncodeToString(sum[:]) +} + // Asked is what the keeper holds of one module, the value of its key in StateAsked: what its newest accepted manifest // asks for, and every entry the keeper made for it and the bus holds, by local name. type Asked struct { @@ -455,10 +477,22 @@ func (k *Keeper) Heard(a Accepted) { if held.Made == nil { held.Made = map[string]Made{} } + // It plans again only when its bus-state entries differ from those it last planned, never on the manifest's digest + // (design 47 G §538). + digest := EntriesDigest(a.BusState) + if p, ok := k.planned(a.Module); ok && found && p.Digest == digest { + held.Digest, held.At = a.Digest, k.now() + if a.Revision > held.Revision { + held.Revision = a.Revision + } + k.write(held) + return + } held.Digest, held.Entries, held.At = a.Digest, a.BusState, k.now() if a.Revision > held.Revision { held.Revision = a.Revision } + defer k.writePlan(a.Module, digest, held.Revision, a.BusState) asked := map[string]bool{} for _, e := range a.BusState { asked[e.Name] = true @@ -829,6 +863,23 @@ func EventID() string { return hex.EncodeToString(b[:]) } +func (k *Keeper) planned(module string) (Planned, bool) { + raw, _, found, err := k.Store.Get(StatePlans, module) + if err != nil || !found { + return Planned{}, false + } + var p Planned + return p, json.Unmarshal(raw, &p) == nil +} + +// writePlan keeps the entries a module was last planned for and their digest (§538). +func (k *Keeper) writePlan(module, digest string, revision uint64, entries []catalogue.BusStateEntry) { + raw, _ := json.Marshal(Planned{V: 1, Module: module, Digest: digest, Revision: revision, Entries: entries, At: k.now()}) + if _, err := k.Store.Put(StatePlans, module, raw); err != nil { + k.logf("the plan of %s cannot be written to its own %s: %v", module, StatePlans, err) + } +} + func (k *Keeper) asked(module string) (Asked, uint64, bool) { raw, rev, found, err := k.Store.Get(StateAsked, module) if err != nil || !found { @@ -848,6 +899,7 @@ func (k *Keeper) write(a Asked) { if err := k.Store.Delete(StateAsked, a.Module); err != nil { k.logf("the key of %s cannot be removed from its own %s: %v", a.Module, StateAsked, err) } + _ = k.Store.Delete(StatePlans, a.Module) return } raw, _ := json.Marshal(a) diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 30c07cb47..d1eab8b7a 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -40,6 +40,8 @@ type fakeBus struct { deletedStreams []string // limits are the bus's storage limits the fake answers. limits Limits + // reads counts the reads of an object's stream. + reads int } func (b *fakeBus) SnapshotStreams(path string) ([]string, error) { @@ -88,6 +90,7 @@ func newFakeBus() *fakeBus { func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) { b.mu.Lock() defer b.mu.Unlock() + b.reads++ s, ok := b.held[o.Stream()] return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil } @@ -575,6 +578,29 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { t.Errorf("with no account limit nothing is refused: %+v", k.Held()["m"].Made["log"]) } }}, + {"a verdict plans again only when its entries differ, never on the manifest's digest (§538)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) + reads := bus.reads + k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 2, BusState: []catalogue.BusStateEntry{drafts}}) + if bus.reads != reads { + t.Errorf("the bus was read %d times more for a manifest that changed outside its entries", bus.reads-reads) + } + raw, _, found, _ := k.Store.Get(StatePlans, "m") + var p Planned + _ = json.Unmarshal(raw, &p) + if !found || p.Digest != EntriesDigest([]catalogue.BusStateEntry{drafts}) || len(p.Entries) != 1 || k.Held()["m"].Digest != "d2" || k.Held()["m"].Revision != 2 { + t.Errorf("the plan kept %+v (%v), the digest and revision follow the verdict: %+v", p, found, k.Held()["m"]) + } + bigger := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "7"}) + k.Heard(Accepted{Module: "m", Digest: "d3", Revision: 3, BusState: []catalogue.BusStateEntry{bigger}}) + if bus.reads == reads || bus.updated == nil { + t.Errorf("a changed option plans the difference: reads %d (was %d), updated %v", bus.reads, reads, bus.updated) + } + k.Heard(Accepted{Module: "m", Digest: "d4", Revision: 4, BusState: nil}) + if _, _, found, _ := k.Store.Get(StatePlans, "m"); !found { + t.Errorf("an entry made and now unasked is an orphan the keeper still holds, with its plan") + } + }}, {"a state made before its objects store existed gets the store and its binding said again (§58)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { p, _ := PlanOf("m", drafts) bus.held["KV_m_drafts"] = p.Objects[0].Settings diff --git a/modules/nats/module.json b/modules/nats/module.json index 7acc5ebdd..ba571cf81 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -115,6 +115,11 @@ "ttl-seconds": 86400, "read-only": true }, + { + "name": "plans", + "history": 1, + "read-only": true + }, { "name": "findings", "history": 1 @@ -154,7 +159,8 @@ "nats_keeper_health", "nats_first_view_again", "nats_requests", - "nats_snapshots" + "nats_snapshots", + "nats_plans" ], "guards": [ 8222 @@ -287,6 +293,7 @@ "asked", "requests", "spent", + "plans", "findings" ], "asks": true, From 8d6b7fbbc99395466a0f56186df542bea01da5ab Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:02:13 +0100 Subject: [PATCH 05/15] A-112: the bus module's snapshots-cap and snapshot-keep-days are config values in a settings file the keeper reads at its start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §633, §681, lead answer A-112 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/nats/cmd/nats-tools/keep.go | 43 ++++++++++++----- modules/nats/cmd/nats-tools/main.go | 15 +++++- modules/nats/cmd/nats-tools/manifest_test.go | 49 ++++++++++++++++++++ modules/nats/files/settings.json | 4 ++ modules/nats/module.json | 31 +++++++++++-- 5 files changed, 127 insertions(+), 15 deletions(-) create mode 100644 modules/nats/files/settings.json diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index fa2f82cbd..ae96890b0 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -227,17 +227,10 @@ func percent(getenv func(string) string, name string, fallback int64) int64 { return fallback } -// number reads a positive whole number from the environment, or the fallback. -func number(getenv func(string) string, name string, fallback int64) int64 { - if n, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64); err == nil && n > 0 { - return n - } - return fallback -} - // runKeeper is the keeper's process: it returns when ctx ends, or when it has no credential. func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer) error { logger := log.New(logOut, "nats-keeper: ", log.LstdFlags) + settings := readSettings(getenv("MESH_NATS_SNAPSHOTS_DIR") + "/settings.json") account, err := process.ReadBrokerAccount(getenv) if err != nil { return err @@ -276,9 +269,10 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer Warrant: WarrantFrom(conn.Ask), Finding: writer, Logf: logger.Printf, Raise: percent(getenv, "MESH_NATS_FULL_RAISE", 75), Clear: percent(getenv, "MESH_NATS_FULL_CLEAR", 70), - // The settings snapshots-cap (20 GiB) and snapshot-keep-days (90) of the manifest (design 47 G §681). - SnapshotsCap: number(getenv, "MESH_NATS_SNAPSHOTS_CAP", keeper.DefaultSnapshotsCap), - SnapshotKeepDays: int(number(getenv, "MESH_NATS_SNAPSHOT_KEEP_DAYS", keeper.DefaultSnapshotKeepDays)), + // The settings snapshots-cap (20 GiB) and snapshot-keep-days (90) of the manifest, read from the file it declares + // at the start, its defaults when the file is missing (design 47 G §681, A-112). + SnapshotsCap: settings.SnapshotsCap, + SnapshotKeepDays: settings.SnapshotKeepDays, } // The process answers the seat mesh-broker's snapshot verbs itself (design 47 G §681, A-107), with the caller the bus proved. stopAnswers, err := conn.AnswerSeat(meshbroker.Seat, snapshotAnswers(k, heldRequest(k.Store), keeperSnapshots(nc, logger.Printf))) @@ -454,3 +448,30 @@ func requestsOf(c *process.Conn) func(context.Context, func()) error { }) } } + +// keeperSettings are the keeper's settings its manifest declares as a file (design 47 G A-112): the content of +// `${config:}`, read at its start. +type keeperSettings struct { + SnapshotsCap int64 `json:"snapshots-cap"` + SnapshotKeepDays int `json:"snapshot-keep-days"` +} + +// readSettings reads the settings file at path; a file that is missing, or a value that is not above 0, gives its default. +func readSettings(path string) keeperSettings { + s := keeperSettings{SnapshotsCap: keeper.DefaultSnapshotsCap, SnapshotKeepDays: keeper.DefaultSnapshotKeepDays} + raw, err := os.ReadFile(path) + if err != nil { + return s + } + var f keeperSettings + if json.Unmarshal(raw, &f) != nil { + return s + } + if f.SnapshotsCap > 0 { + s.SnapshotsCap = f.SnapshotsCap + } + if f.SnapshotKeepDays > 0 { + s.SnapshotKeepDays = f.SnapshotKeepDays + } + return s +} diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index 1f5e82f4b..a82f859bf 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -6,6 +6,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "time" @@ -196,7 +197,19 @@ func allTools() []stdio.Tool { out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz)) out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...) rs := RequestsInState() - out = append(out, snapshotListTool(rs, int(number(os.Getenv, "MESH_NATS_SNAPSHOT_KEEP_DAYS", keeper.DefaultSnapshotKeepDays)))...) + out = append(out, snapshotListTool(rs, bundleKeepDays())...) out = append(out, plansTool(PlansInState)) return append(out, requestTools(rs)...) } + +// bundleKeepDays is the setting snapshot-keep-days as this bundle's configuration holds it (design 47 G A-112), its +// default when the runtime gives none. +func bundleKeepDays() int { + if raw, err := stdio.Config("snapshot-keep-days"); err == nil { + var n int + if json.Unmarshal(raw, &n) == nil && n > 0 { + return n + } + } + return keeper.DefaultSnapshotKeepDays +} diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index 3e3214030..e492b319a 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -8,6 +8,8 @@ import ( "slices" "strings" "testing" + + "nats-tools/internal/keeper" ) // The shape ADR 0480 clauses 1, 2, 3 and 5 give the bus's manifest: the upstream image by digest, the mesh's two @@ -315,3 +317,50 @@ func TestTheKeepersProcessServesTheVerbsItAnswers(t *testing.T) { t.Errorf("the keeper process serves %v; its code answers %v", served, want) } } + +// The keeper's snapshot settings are config values of the manifest, written into a file the keeper reads at its start, +// never environment (design 47 G A-112, §681): a missing file, or a value not above 0, gives the default. +func TestTheKeepersSnapshotSettingsAreAFileItsManifestDeclares(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Config map[string]struct{ Default float64 } `json:"config"` + Resources []struct { + ID string `json:"id"` + Path string `json:"path"` + Owner string `json:"owner"` + TakesConfig []string `json:"takes-config"` + Env map[string]string `json:"env"` + } `json:"resources"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Config["snapshots-cap"].Default != 21474836480 || m.Config["snapshot-keep-days"].Default != 90 { + t.Errorf("config %v: snapshots-cap 20 GiB and snapshot-keep-days 90 are the defaults", m.Config) + } + found := false + for _, r := range m.Resources { + if r.ID == "keeper-settings" { + found = r.Path == "${dir:keeper-state}/settings.json" && r.Owner == "nats-keeper" && slices.Equal(r.TakesConfig, []string{"snapshots-cap", "snapshot-keep-days"}) + } + for k := range r.Env { + if strings.Contains(k, "SNAPSHOTS_CAP") || strings.Contains(k, "SNAPSHOT_KEEP_DAYS") { + t.Errorf("%s still sets %s in the environment", r.ID, k) + } + } + } + if !found { + t.Errorf("the keeper has no settings file taking snapshots-cap and snapshot-keep-days") + } + dir := t.TempDir() + if got := readSettings(dir + "/none.json"); got.SnapshotsCap != keeper.DefaultSnapshotsCap || got.SnapshotKeepDays != keeper.DefaultSnapshotKeepDays { + t.Errorf("a missing file gives %+v", got) + } + _ = os.WriteFile(dir+"/s.json", []byte(`{"snapshots-cap": 1000, "snapshot-keep-days": 0}`), 0o600) + if got := readSettings(dir + "/s.json"); got.SnapshotsCap != 1000 || got.SnapshotKeepDays != keeper.DefaultSnapshotKeepDays { + t.Errorf("the file gives %+v: the cap it names, the day count's default for 0", got) + } +} diff --git a/modules/nats/files/settings.json b/modules/nats/files/settings.json new file mode 100644 index 000000000..f973289e0 --- /dev/null +++ b/modules/nats/files/settings.json @@ -0,0 +1,4 @@ +{ + "snapshots-cap": 21474836480, + "snapshot-keep-days": 90 +} diff --git a/modules/nats/module.json b/modules/nats/module.json index ba571cf81..1e23a2e6d 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -71,6 +71,20 @@ "uses": [ "operator-channel" ], + "config": { + "snapshots-cap": { + "type": "number", + "description": "How many bytes the keeper's snapshots directory may hold (design 47 G §681): past 80 % of it the keeper tells a finding, and it never takes a snapshot past it. The keeper reads it at its start from its settings file.", + "minimum": 1, + "default": 21474836480 + }, + "snapshot-keep-days": { + "type": "number", + "description": "How many days a snapshot the keeper took before a drop or recreate, and its record, are kept (design 47 G §681, A-112). The keeper reads it at its start from its settings file; the tool nats_snapshots reads it as its configuration.", + "minimum": 1, + "default": 90 + } + }, "streams": [ { "name": "broker-events", @@ -236,6 +250,20 @@ "place": "mesh", "unknown": "program" }, + { + "id": "keeper-settings", + "trusted": false, + "type": "file", + "path": "${dir:keeper-state}/settings.json", + "mode": "0600", + "owner": "nats-keeper", + "takes-config": [ + "snapshots-cap", + "snapshot-keep-days" + ], + "merge": "json", + "template": "files/settings.json" + }, { "id": "server-conf", "type": "file", @@ -278,8 +306,6 @@ "MESH_BROKER_FILE": "${dir:keeper-state}/broker-keeper", "MESH_NATS_FULL_RAISE": "75", "MESH_NATS_FULL_CLEAR": "70", - "MESH_NATS_SNAPSHOTS_CAP": "21474836480", - "MESH_NATS_SNAPSHOT_KEEP_DAYS": "90", "MESH_NATS_SNAPSHOTS_DIR": "${dir:keeper-state}" }, "serves": [ @@ -389,7 +415,6 @@ "MESH_NATS_CONTAINER": "mesh-broker-nats", "MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf", "MESH_NATS_SNAPSHOTS_DIR": "${dir:snapshots}", - "MESH_NATS_SNAPSHOT_KEEP_DAYS": "90", "NATS_TOOLS_CREDENTIAL": "${dir:tools-secret}/broker-nats-tools" } }, From 0e3580895da133cd097030bf03db5aada1e2f370 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:03:13 +0100 Subject: [PATCH 06/15] mesh-conditions: ListBound is asked again on the bus module's created or a node-engine's running, not on a timer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §515, §663 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-conditions/bound.go | 26 ++++++++--- .../cmd/mesh-conditions/events.go | 6 ++- .../cmd/mesh-conditions/keeper.go | 6 +++ .../cmd/mesh-conditions/keeper_test.go | 44 +++++++++++++++++++ 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/bound.go b/modules/mesh-conditions/cmd/mesh-conditions/bound.go index edff8dac4..8bec1b32d 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/bound.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/bound.go @@ -7,7 +7,6 @@ package main import ( "encoding/json" "fmt" - "time" broker "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" ) @@ -48,10 +47,27 @@ func (k *Keeper) takeSource(module string) error { return nil } +// nudgeBound says, without waiting, that the bus module's entries may be read now (ListBound's retry). +func (k *Keeper) nudgeBound() { + k.mu.Lock() + k.init() + c := k.boundRetry + k.mu.Unlock() + select { + case c <- struct{}{}: + default: + } +} + // ListBound reads the entries the bus module holds at the keeper's start: every module with a `finding-events` entry is a -// source. A call that fails is asked again with a growing wait, the bus module being delivered after the keeper at times. +// source. A call that fails is asked again on the next event that says the bus module may answer, its `created` or a +// node-engine's `running` (the bus module being delivered after the keeper at times), never on a timer. func ListBound(k *Keeper, call func(key string, body any) (json.RawMessage, error)) { - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + k.mu.Lock() + k.init() + retry := k.boundRetry + k.mu.Unlock() + for { raw, err := call("seat:"+broker.Seat+"."+broker.VerbList, broker.ListInput{}) var answer broker.ListAnswer if err == nil { @@ -67,7 +83,7 @@ func ListBound(k *Keeper, call func(key string, body any) (json.RawMessage, erro } return } - k.Logf("the entries of the bus module are not read yet (%v); asked again in %s", err, wait) - time.Sleep(wait) + k.Logf("the entries of the bus module are not read yet (%v); asked again when it says created or a node-engine says running", err) + <-retry } } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/events.go b/modules/mesh-conditions/cmd/mesh-conditions/events.go index 4a22ca0b5..0c892aece 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/events.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/events.go @@ -72,13 +72,17 @@ func (k *Keeper) FindingEvent(e stdio.Envelope) error { if _, ok := emitterOf(e.Key, watchdogs.EventMachineBack); ok && strings.HasPrefix(e.Key, watchdogs.Seat+".") { return k.MachinePresence(e.Body, false) } - if _, _, _, ok := engineKey(e.Key); ok { + if event, _, _, ok := engineKey(e.Key); ok { + if event == engine.EventRunning { + k.nudgeBound() + } return k.EngineEvent(e) } if strings.HasPrefix(e.Key, meshhealers.Seat+".attempt-changed.") { return k.AttemptChanged(e.Key, e.Body) } if e.Key == broker.Seat+"."+broker.EventCreated { + k.nudgeBound() return k.BrokerCreated(e.Body) } return nil diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index 179e30b70..cc5682b2e 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -164,6 +164,9 @@ type Keeper struct { stopped map[string]bool // created are the modules whose `finding-events` entry the bus module made (bound.go). created map[string]bool + // boundRetry is told, once, when the bus module says `created` or a node-engine says `running`: the events after which + // the entries of the bus module may be read (ListBound). + boundRetry chan struct{} // followed are the finding streams the catalog half declared for each module, by module (catalog.go). followed map[string]string } @@ -179,6 +182,9 @@ func (k *Keeper) init() { k.halves, k.placed, k.stopped = map[string]map[string]catseat.CatalogEntry{}, map[string]bool{}, map[string]bool{} k.views, k.keyNode, k.whys = map[string]*sourceView{}, map[string]map[string]string{}, map[string]string{} } + if k.boundRetry == nil { + k.boundRetry = make(chan struct{}, 1) + } if k.Now == nil { k.Now = time.Now } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 2c69ba789..7fdaa964f 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -723,3 +723,47 @@ func TestTheKeepersOwnFindingIsTheContractsFindingByItsWireForm(t *testing.T) { t.Fatal("the condition stands after its finding was cleared") } } + +// ListBound reads the bus module's entries once and, when the call fails, again on the event that says the bus module may +// answer (its `created`, or a node-engine's `running`), never on a timer (design 47 G §515, §663). +func TestListBoundIsAskedAgainOnTheEventsNotATimer(t *testing.T) { + for _, event := range []string{"mesh-broker.created", "node-engine.running.ace.mesh-conditions"} { + t.Run(event, func(t *testing.T) { + var asked []map[string]any + k := &Keeper{Publish: func(_ string, b any) error { asked = append(asked, b.(map[string]any)); return nil }} + calls := make(chan int) + ended := make(chan struct{}) + go func() { + n := 0 + ListBound(k, func(string, any) (json.RawMessage, error) { + n++ + calls <- n + if n == 1 { + return nil, fmt.Errorf("the bus module is not delivered yet") + } + return mustJSON(t, map[string]any{"entries": []any{map[string]any{"module": "newpart", "template": FindingEventsTemplate}}}), nil + }) + close(ended) + }() + <-calls + select { + case <-ended: + t.Fatal("ListBound ended with no answer") + default: + } + _ = k.FindingEvent(stdio.Envelope{Key: event, Body: []byte(`{}`)}) + select { + case n := <-calls: + if n != 2 { + t.Fatalf("asked %d times", n) + } + case <-time.After(time.Second): + t.Fatal("not asked again on the event") + } + <-ended + if len(asked) != 1 || asked[0]["module"] != "newpart" { + t.Errorf("asked %v, want newpart once", asked) + } + }) + } +} From dcaa229c2b92a69d5dce2e7526ae513ed7269a0e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:03:35 +0100 Subject: [PATCH 07/15] mesh-issues: mesh-recovery is an owner of issues Builds: design 47 G recovery module (catalog review finding: the owners test of every code module) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-issues/files/settings.json | 1 + modules/mesh-issues/module.json | 1 + 2 files changed, 2 insertions(+) diff --git a/modules/mesh-issues/files/settings.json b/modules/mesh-issues/files/settings.json index dca5d6cf9..b833058f6 100644 --- a/modules/mesh-issues/files/settings.json +++ b/modules/mesh-issues/files/settings.json @@ -78,6 +78,7 @@ "mesh-catalog modules/mesh-provisioner", "mesh-catalog modules/mesh-provisioner-client", "mesh-catalog modules/mesh-reads", + "mesh-catalog modules/mesh-recovery", "mesh-catalog modules/mesh-self-check", "mesh-catalog modules/mesh-vault", "mesh-catalog modules/mesh-watcher", diff --git a/modules/mesh-issues/module.json b/modules/mesh-issues/module.json index 2ae2304be..ebf31730c 100644 --- a/modules/mesh-issues/module.json +++ b/modules/mesh-issues/module.json @@ -190,6 +190,7 @@ "mesh-catalog modules/mesh-provisioner", "mesh-catalog modules/mesh-provisioner-client", "mesh-catalog modules/mesh-reads", + "mesh-catalog modules/mesh-recovery", "mesh-catalog modules/mesh-self-check", "mesh-catalog modules/mesh-vault", "mesh-catalog modules/mesh-watcher", From c235abcb11122936ace71365bae6120fb890bbb6 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:04:06 +0100 Subject: [PATCH 08/15] mesh-recovery: the four bound settings take a minimum above 0, as the reader refuses 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §557, §561, §681 (catalog review finding) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-recovery/internal/recovery/manifest_test.go | 9 ++++++++- modules/mesh-recovery/module.json | 8 ++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/mesh-recovery/internal/recovery/manifest_test.go b/modules/mesh-recovery/internal/recovery/manifest_test.go index 185840f3d..686e7dd64 100644 --- a/modules/mesh-recovery/internal/recovery/manifest_test.go +++ b/modules/mesh-recovery/internal/recovery/manifest_test.go @@ -21,7 +21,8 @@ type manifest struct { Consumes []string `json:"consumes"` Invokes []string `json:"invokes"` Config map[string]struct { - Default float64 `json:"default"` + Default float64 `json:"default"` + Minimum *float64 `json:"minimum"` } `json:"config"` Data struct { Own []struct{ ID string } `json:"own"` @@ -75,6 +76,12 @@ func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { t.Errorf("setting %s defaults to %v, want %v", k, m.Config[k].Default, want) } } + // A bound of time is never 0, which the reader refuses: its minimum is above 0 (the manifest's minimum is inclusive). + for _, k := range []string{"point-keep-days", "dump-wait-minutes", "snapshot-restore-wait-minutes", "expire-every-hours"} { + if m.Config[k].Minimum == nil || *m.Config[k].Minimum < 1 { + t.Errorf("setting %s has the minimum %v: a bound of 0 is refused by the reader, so the manifest refuses it", k, m.Config[k].Minimum) + } + } // The keeper's snapshot verbs the recovery module asks, the cancel at its bound among them (§681). for _, inv := range []string{"seat:mesh-broker.snapshot-get", "seat:mesh-broker.snapshot-restore", "seat:mesh-broker.snapshot-restore-cancel"} { if !slices.Contains(m.Invokes, inv) { diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json index bbfe35474..acf9bfeb2 100644 --- a/modules/mesh-recovery/module.json +++ b/modules/mesh-recovery/module.json @@ -110,7 +110,7 @@ "point-keep-days": { "type": "number", "description": "days after which a restore point is removed, the newest of its kept data never (design 47 G sections 551, 557).", - "minimum": 0, + "minimum": 1, "maximum": 3650, "trusted": false, "default": 90 @@ -118,7 +118,7 @@ "dump-wait-minutes": { "type": "number", "description": "minutes an act waits on the dump of the bus's item before recovery-waits is written (design 47 G sections 561, 557).", - "minimum": 0, + "minimum": 1, "maximum": 10080, "trusted": false, "default": 60 @@ -126,7 +126,7 @@ "snapshot-restore-wait-minutes": { "type": "number", "description": "minutes a restore in place of a protected keeper's snapshot is waited on, and once more after its cancel (design 47 G section 681, A-138, A-140).", - "minimum": 0, + "minimum": 1, "maximum": 10080, "trusted": false, "default": 90 @@ -134,7 +134,7 @@ "expire-every-hours": { "type": "number", "description": "hours between two removals of the points past their keep days, beside the one at the start and at each snapshot-kept; timed by a bound key of the state bounds (design 47 G sections 551, 557, A-138).", - "minimum": 0, + "minimum": 1, "maximum": 8760, "trusted": false, "default": 24 From 741239a5068f20f0a4401887d96042ae9da2aaf2 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:04:58 +0100 Subject: [PATCH 09/15] EE-K1: the Stop hook advises mesh_wait on a wait's event and a reread after every answer, or a watcher, with no condition on a grant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 E §20 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/claude-code/cmd/claude-code/stop.go | 9 ++++----- modules/claude-code/cmd/claude-code/stop_test.go | 10 +++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/modules/claude-code/cmd/claude-code/stop.go b/modules/claude-code/cmd/claude-code/stop.go index d1328c778..2a7bab6b8 100644 --- a/modules/claude-code/cmd/claude-code/stop.go +++ b/modules/claude-code/cmd/claude-code/stop.go @@ -60,13 +60,12 @@ func waitBlocksStop(session string, w Wait, now time.Time) (bool, string) { "{\"op\":\"said\",\"session\":%q,\"by\":%q,\"id\":%q}.", w.ID, w.What, session, w.By, w.ID) } -// watcherAdvice is what a session is told to do to follow a wait (ADR 0489 §3, review round 1 item 3): a -// family is a candidate for mesh_wait only when the tool runner is granted it (ADR 0367 §2), which this hook -// cannot know, so it is offered conditionally; a watcher that rereads the read address always works. +// watcherAdvice is what a session is told to do to follow a wait (ADR 0489 §3, design 47 E §20): a wait with an event is +// followed by mesh_wait on it, rereading what it reads after every answer, or by a watcher that does; with no event, by a +// watcher that rereads the read address. The advice puts no condition on a grant. func watcherAdvice(w Wait) string { if w.Family != "" { - return fmt.Sprintf("mesh_wait on %s if this session's tool runner is granted it, otherwise start a watcher "+ - "that rereads %s", w.Family, w.Read.Address) + return fmt.Sprintf("mesh_wait on %s and reread %s after every answer, or start a watcher that does", w.Family, w.Read.Address) } return fmt.Sprintf("start a watcher that rereads %s", w.Read.Address) } diff --git a/modules/claude-code/cmd/claude-code/stop_test.go b/modules/claude-code/cmd/claude-code/stop_test.go index 71501a566..11ead0cf0 100644 --- a/modules/claude-code/cmd/claude-code/stop_test.go +++ b/modules/claude-code/cmd/claude-code/stop_test.go @@ -300,15 +300,15 @@ func TestStopAllowsWhenTheRefusalCounterCannotBeWritten(t *testing.T) { } } -// Review round 1, item 3 (1743/AC1, §3 with ADR 0367 §2): a family is offered only conditionally (the tool -// runner may not be granted it); without one, the advice is only to start a watcher. -func TestWaitBlocksStopAdvisesMeshWaitOnlyConditionallyOtherwiseAWatcher(t *testing.T) { +// Design 47 E §20: a wait with an event is followed by mesh_wait on it and a reread after every answer, or a watcher +// that does, with no condition on a grant; without an event, the advice is only to start a watcher. +func TestWaitBlocksStopAdvisesMeshWaitOnItsEventOtherwiseAWatcher(t *testing.T) { withFamily := openWait("w-f", 10) withFamily.Read = WaitRead{Address: "mesh-delivery.recheck"} withFamily.Family = "mesh-delivery.decided" _, reason := waitBlocksStop("sess-f", withFamily, stopT0.Add(time.Minute)) - if !strings.Contains(reason, "mesh_wait on mesh-delivery.decided if this session's tool runner is granted it") || - !strings.Contains(reason, "otherwise start a watcher that rereads mesh-delivery.recheck") { + if !strings.Contains(reason, "mesh_wait on mesh-delivery.decided and reread mesh-delivery.recheck after every answer, or start a watcher that does") || + strings.Contains(reason, "granted") { t.Fatalf("%s", reason) } From 2d5358b8b95e2cf6b2511b2885ebff3db4bd315e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:05:08 +0100 Subject: [PATCH 10/15] AM-K1: the zsh module uses operator-channel for its marked execute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 46 7-approval-mark §21 (milestone G approval mark) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/zsh/module.json | 3 +++ modules/zsh/test/manifest.test.ts | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/modules/zsh/module.json b/modules/zsh/module.json index 8d965e972..028f2213d 100644 --- a/modules/zsh/module.json +++ b/modules/zsh/module.json @@ -14,6 +14,9 @@ ] } ], + "uses": [ + "operator-channel" + ], "tools": [ "zsh_config" ], diff --git a/modules/zsh/test/manifest.test.ts b/modules/zsh/test/manifest.test.ts index 019e94eb0..9530b9779 100644 --- a/modules/zsh/test/manifest.test.ts +++ b/modules/zsh/test/manifest.test.ts @@ -28,6 +28,10 @@ test("it claims node-login-shell serving execute, and declares no seat", () => { assert.deepEqual(m.claims, [{ name: "node-login-shell", scope: "node", serves: ["execute"] }]); }); +test("it uses operator-channel, since its seat's verb execute is marked needs-approval (design 46/7 section 21)", () => { + assert.deepEqual(m.uses, ["operator-channel"]); +}); + test("the .zshrc block is at the start and holds the three slots in order, after the defaults' header", () => { const rc = resource("rc"); assert.equal(rc.path, "${machine:account-home}/.zshrc"); From 00be4c506327c7363a6256edb74f0658d5631ac3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:06:03 +0100 Subject: [PATCH 11/15] GIT17: the forge module deletes seen keys past 30 days on each step-ended and at its start; git_seen_expire runs it now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §581, §585, §487 (TypeScript tests not run here: no installed dependencies) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/gitea/failedsteps.ts | 24 ++++++++++++++++++++++++ modules/gitea/index.ts | 3 +++ modules/gitea/test/failedsteps.test.ts | 22 ++++++++++++++++++++++ modules/gitea/tools/failedsteps.ts | 18 ++++++++++++++++++ 4 files changed, 67 insertions(+) diff --git a/modules/gitea/failedsteps.ts b/modules/gitea/failedsteps.ts index bbdb5f89c..c7edd8eea 100644 --- a/modules/gitea/failedsteps.ts +++ b/modules/gitea/failedsteps.ts @@ -24,6 +24,10 @@ export const SEEN = "seen."; /** The template state's cap (design 47 *The runner's kept data*): the fill is shown against it. */ export const CAP_BYTES = 64 * 1024 * 1024; +/** How long a `seen` key is kept after its `step-ended`, and how many go on one event (§581). */ +export const SEEN_DAYS = 30; +export const SEEN_MOST_PER_EVENT = 100; + /** What a `seen` key holds: a commit's fingerprints at one module. */ export interface Seen { v: 1; @@ -79,6 +83,11 @@ export class FailedSteps { * a kept failure's step clears it, and nothing else does. */ stepEnded(ev: StepEndedEvent): Promise { + // The `seen` keys past their 30 days are judged on each step end, with no timer (§581). + return this.record(ev).then(() => this.expireSeen(new Date(Date.parse(String(ev?.at ?? "")) || Date.now())).then(() => {})); + } + + private record(ev: StepEndedEvent): Promise { const r = ev?.result; const fingerprint = r?.fingerprint || ev?.fp || ""; if (!r || !SHA.test(String(ev.commit ?? "")) || !fingerprint || !ev.module) return Promise.resolve(); @@ -163,6 +172,21 @@ export class FailedSteps { }); } + /** + * Deletes the `seen` keys whose `at` is more than 30 days before `now`, the oldest first, at most `most` of them (§581): run on + * each `step-ended`, at the start, and by the tool `git_seen_expire`. A key with no readable `at` is kept. Answers the keys deleted. + */ + expireSeen(now: Date = new Date(), most = SEEN_MOST_PER_EVENT): Promise { + return this.turn(async () => { + const { seen } = await this.all(); + const limit = now.getTime() - SEEN_DAYS * 24 * 3600 * 1000; + const old = [...seen].filter(([, s]) => Date.parse(s.at) < limit).sort(([, a], [, b]) => Date.parse(a.at) - Date.parse(b.at)).slice(0, most); + for (const [k] of old) await this.st.delete(k); + if (old.length) this.log(`${old.length} seen key(s) past ${SEEN_DAYS} days deleted`); + return old.map(([k]) => k); + }); + } + /** Drops one held key, a failure by its key or a `seen` key; false when it is not there. */ drop(key: string): Promise { return this.turn(async () => { diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 0f61601bd..d26359575 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -402,6 +402,9 @@ const heldHeads = new Heads(); let reloadHeads: () => Promise = async () => {}; const failedSteps = new FailedSteps(state("failed-steps"), (line) => console.log(`[gitea] ${line}`), (commit) => heldHeads.has(commit)); +// The `seen` keys past their 30 days are deleted at the start, as on each step-ended (design 47 §581). +void failedSteps.expireSeen().catch((err) => console.log(`[gitea] the seen keys past 30 days could not be deleted at the start: ${err instanceof Error ? err.message : String(err)}`)); + // gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729). The seat git's verbs are // served here, beside the one writer of a head's statuses: the seat's status takes its turn on the head's chain, after the // announcement's pending and a trunk's move, so a recheck's pending never lands over a later verdict (issue 456). diff --git a/modules/gitea/test/failedsteps.test.ts b/modules/gitea/test/failedsteps.test.ts index d6883b617..e6e499887 100644 --- a/modules/gitea/test/failedsteps.test.ts +++ b/modules/gitea/test/failedsteps.test.ts @@ -96,3 +96,25 @@ test("git_failed_steps shows what is held and its fill, and git_failed_steps_dro assert.deepEqual([done.dropped, acts.length, acts[0].verb], [true, 1, "git_failed_steps_drop"]); assert.equal(await f.judge(C2), null); }); + +test("a seen key past 30 days is deleted at the next step-ended, a younger one kept, at most 100 oldest first, and git_seen_expire runs the same now (§581)", async () => { + const { failedStepsTools } = await import("../tools/failedsteps.js"); + const { runAs } = await import("@novox/mesh-sdk/callers"); + const { st, kv } = memory() as any; + const f = new FailedSteps(st as any); + // The step-ended is at 2026-11-11T12:00:00Z, so 30 days before it is 2026-10-12. + const now = new Date("2026-11-11T12:00:00Z"); + const old = (n: number) => new Date(Date.parse("2026-10-01T00:00:00Z") - n * 60000).toISOString(); + await st.put(seenKey(C1, "old"), { v: 1, commit: C1, module: "old", fingerprints: ["x"], at: old(0) }); + await st.put(seenKey(C2, "young"), { v: 1, commit: C2, module: "young", fingerprints: ["x"], at: "2026-10-20T00:00:00Z" }); + await f.stepEnded(ended(C3, "passed")); + assert.equal(kv.has(seenKey(C1, "old")), false, "past 30 days: deleted at the next step-ended"); + assert.equal(kv.has(seenKey(C2, "young")), true, "younger: kept"); + assert.equal(kv.has(seenKey(C3, "nats")), true, "the step's own key is new"); + for (let i = 0; i < 101; i++) await st.put(seenKey(C1, `m${String(i).padStart(3, "0")}`), { v: 1, commit: C1, module: `m${i}`, fingerprints: ["x"], at: old(i) }); + assert.equal((await f.expireSeen(now)).length, 100, "at most 100 for each event"); + const acts: any[] = []; + const tool = failedStepsTools(f, async (a) => { acts.push(a); }).find((t) => t.name === "git_seen_expire")!; + const done: any = await runAs({ "mesh/caller": "operator" }, {}, () => tool.run({ why: "stale" })); + assert.deepEqual([done.deleted.length, acts[0].verb], [1, "git_seen_expire"]); +}); diff --git a/modules/gitea/tools/failedsteps.ts b/modules/gitea/tools/failedsteps.ts index 430aaaa61..c13ba2067 100644 --- a/modules/gitea/tools/failedsteps.ts +++ b/modules/gitea/tools/failedsteps.ts @@ -40,5 +40,23 @@ export function failedStepsTools(kept: FailedSteps, acts: (act: HandAct) => Prom return { dropped: await kept.drop(key), key, why, by }; }, }, + { + name: "git_seen_expire", + description: "Deletes the seen keys of the state failed-steps that are more than 30 days old, the oldest first, at most 100: the removal the forge module does on each step-ended, now, with why. " + + "Written in the hand-act log first (design 47 §581).", + input: { type: "object", required: ["why"], properties: { why: { type: "string", description: "why it is run now, in plain words" } } }, + run: async (args: Record) => { + const why = typeof args.why === "string" ? args.why.trim() : ""; + if (!why) throw new Error("git_seen_expire says why"); + const by = callerOf(args); + try { + await acts({ what: "deleted the seen keys of the forge module's failed-steps past 30 days", why: `${why} (${by})`, cause: "seen-expired", verb: "git_seen_expire" }); + } catch (err) { + throw new Error(`the hand act was not recorded, so nothing was deleted: ${err instanceof Error ? err.message : String(err)}`); + } + const deleted = await kept.expireSeen(); + return { deleted, why, by }; + }, + }, ]; } From e723d498e760d2ed90808ff4c2a239fc3515f059 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:08:17 +0100 Subject: [PATCH 12/15] NATS1, NATS6: the bus module's whole test makes the plans state and restores a state's objects store with it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §58, §538 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/nats/cmd/nats-tools/keeper_integration_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/nats/cmd/nats-tools/keeper_integration_test.go b/modules/nats/cmd/nats-tools/keeper_integration_test.go index 63e38779f..832b9505a 100644 --- a/modules/nats/cmd/nats-tools/keeper_integration_test.go +++ b/modules/nats/cmd/nats-tools/keeper_integration_test.go @@ -158,7 +158,7 @@ func TestTheBusModuleRunsWhole(t *testing.T) { t.Fatal(err) } defer pconn.Close() - for _, st := range []string{keeper.StateAsked, keeper.StateRequests, keeper.StateSpent, "findings"} { + for _, st := range []string{keeper.StateAsked, keeper.StateRequests, keeper.StateSpent, keeper.StatePlans, "findings"} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "nats_" + st, History: 5}); err != nil { t.Fatal(err) } @@ -521,8 +521,11 @@ func TestTheBusModuleRunsWhole(t *testing.T) { if a := restore("live"); !strings.Contains(a.Error, "names live") { t.Errorf("a restore over a live stream answered %+v", a) } - if err := js.DeleteStream(ctx, "KV_m_drafts"); err != nil { - t.Fatal(err) + // The state's snapshot holds its objects store beside it (design 47 G §58): a restore goes over neither. + for _, name := range []string{"KV_m_drafts", "OBJ_m_drafts-objects"} { + if err := js.DeleteStream(ctx, name); err != nil { + t.Fatal(err) + } } a := restore("it was removed by mistake") if a.Accepted == "" { From 64669575c8ad82db943c381115b7f34d56b8f066 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:12:20 +0100 Subject: [PATCH 13/15] mesh-recovery: the four bound settings take a fractional minimum of 0.01, so a whole test sets seconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A minimum of 1 refused the fractions A-138 allows; 0 stays refused, as the reader refuses it. Builds: design 47 G §557, §551, A-138 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-recovery/internal/recovery/manifest_test.go | 7 ++++--- modules/mesh-recovery/module.json | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/mesh-recovery/internal/recovery/manifest_test.go b/modules/mesh-recovery/internal/recovery/manifest_test.go index 686e7dd64..3b8a7dd54 100644 --- a/modules/mesh-recovery/internal/recovery/manifest_test.go +++ b/modules/mesh-recovery/internal/recovery/manifest_test.go @@ -76,10 +76,11 @@ func TestTheManifestSaysWhatTheModuleIs(t *testing.T) { t.Errorf("setting %s defaults to %v, want %v", k, m.Config[k].Default, want) } } - // A bound of time is never 0, which the reader refuses: its minimum is above 0 (the manifest's minimum is inclusive). + // A bound of time is never 0, which the reader refuses, and may be a fraction (A-138), so a whole test sets + // seconds: its minimum is above 0 and below 1 (the manifest's minimum is inclusive). for _, k := range []string{"point-keep-days", "dump-wait-minutes", "snapshot-restore-wait-minutes", "expire-every-hours"} { - if m.Config[k].Minimum == nil || *m.Config[k].Minimum < 1 { - t.Errorf("setting %s has the minimum %v: a bound of 0 is refused by the reader, so the manifest refuses it", k, m.Config[k].Minimum) + if min := m.Config[k].Minimum; min == nil || *min <= 0 || *min >= 1 { + t.Errorf("setting %s has the minimum %v: a bound of 0 is refused by the reader and a fraction is allowed (A-138), so its minimum is above 0 and below 1", k, m.Config[k].Minimum) } } // The keeper's snapshot verbs the recovery module asks, the cancel at its bound among them (§681). diff --git a/modules/mesh-recovery/module.json b/modules/mesh-recovery/module.json index acf9bfeb2..179c4ed97 100644 --- a/modules/mesh-recovery/module.json +++ b/modules/mesh-recovery/module.json @@ -110,7 +110,7 @@ "point-keep-days": { "type": "number", "description": "days after which a restore point is removed, the newest of its kept data never (design 47 G sections 551, 557).", - "minimum": 1, + "minimum": 0.01, "maximum": 3650, "trusted": false, "default": 90 @@ -118,7 +118,7 @@ "dump-wait-minutes": { "type": "number", "description": "minutes an act waits on the dump of the bus's item before recovery-waits is written (design 47 G sections 561, 557).", - "minimum": 1, + "minimum": 0.01, "maximum": 10080, "trusted": false, "default": 60 @@ -126,7 +126,7 @@ "snapshot-restore-wait-minutes": { "type": "number", "description": "minutes a restore in place of a protected keeper's snapshot is waited on, and once more after its cancel (design 47 G section 681, A-138, A-140).", - "minimum": 1, + "minimum": 0.01, "maximum": 10080, "trusted": false, "default": 90 @@ -134,7 +134,7 @@ "expire-every-hours": { "type": "number", "description": "hours between two removals of the points past their keep days, beside the one at the start and at each snapshot-kept; timed by a bound key of the state bounds (design 47 G sections 551, 557, A-138).", - "minimum": 1, + "minimum": 0.01, "maximum": 8760, "trusted": false, "default": 24 From 9fe3f41ac0aeced17f890b93cd38046118d271f9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:17:44 +0100 Subject: [PATCH 14/15] AM-K2: the forge module's gitea_branch_protection_loosen alone removes a trunk's protection or lets a force-push onto it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool is marked in the manifest's needs-approval, so each call runs on the operator's Approve; the manifest uses operator-channel; set_full keeps refusing a force-push onto a trunk and names the new tool; both clients gain deleteBranchProtection. Builds: design 46 approval mark §21 (milestone G) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/gitea/client.ts | 14 ++++++ modules/gitea/module.json | 6 +++ modules/gitea/protectfull.ts | 17 ++++--- modules/gitea/test/protection-full.test.ts | 39 ++++++++++++++++ modules/gitea/test/token.test.ts | 2 +- modules/gitea/tools/index.ts | 52 +++++++++++++++++++++- 6 files changed, 122 insertions(+), 8 deletions(-) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index d7402f378..7720ff58d 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -794,6 +794,12 @@ export class GiteaClient { { method: "PATCH", body: JSON.stringify(body) }); } + /** Remove the protection rule named rule (gitea_branch_protection_loosen, design 46 approval mark §21). As this + * client's account, which the forge refuses where it is no administrator. */ + async deleteBranchProtection(owner: string, repo: string, rule: string): Promise { + await this.request(`${repoPath(owner, repo)}/branch_protections/${pathSegments("branch", rule)}`, { method: "DELETE" }); + } + async createPullRequest( owner: string, repo: string, @@ -1097,6 +1103,14 @@ export class GiteaAdmin { return res.body as BranchProtection; } + /** Remove the protection rule named rule, as the forge's administrator (gitea_branch_protection_loosen, design 46 + * approval mark §21): the forge answers 204. */ + async deleteBranchProtection(owner: string, repo: string, rule: string): Promise { + const path = `${repoPath(owner, repo)}/branch_protections/${pathSegments("branch", rule)}`; + const res = await this.request(path, { method: "DELETE" }); + if (res.status !== 204 && res.status !== 200) GiteaAdmin.fail(path, res); + } + /** A repository as the forge answers it to its administrator: every setting (gitea_repo_settings_set_full). */ async repoSettings(owner: string, repo: string): Promise> { const path = repoPath(owner, repo); diff --git a/modules/gitea/module.json b/modules/gitea/module.json index e149a5518..24da1123d 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -162,6 +162,12 @@ ] } ], + "uses": [ + "operator-channel" + ], + "needs-approval": [ + "gitea_branch_protection_loosen" + ], "data": { "own": [ { diff --git a/modules/gitea/protectfull.ts b/modules/gitea/protectfull.ts index ddb173163..b288f0ca7 100644 --- a/modules/gitea/protectfull.ts +++ b/modules/gitea/protectfull.ts @@ -218,16 +218,23 @@ export function trunksOf(): { names: string[]; from: string } { return { names: [MAIN_BRANCH], from: TRUNK_FROM }; } +/** The tool that alone removes a trunk's protection or allows a force-push onto a trunk, marked in the manifest's + * `needs-approval`, so it runs only on the operator's Approve of that one call (design 46 approval mark §21). */ +export const LOOSEN_TOOL = "gitea_branch_protection_loosen"; + +/** The two acts LOOSEN_TOOL performs (design 46 approval mark §21). */ +export const LOOSEN_ACTS = ["remove-protection", "allow-force-push"] as const; + /** Why a call is the operator's, or null: it lets a force-push onto a rule covering a trunk (the operator, 2026-10-28 - * 16:4x: a call waits for their Approve only if it removes branch protection or allows a force-push to a trunk). No - * tool of this module can ask for that Approve yet, so such a call is refused with nothing sent. */ + * 16:4x: a call waits for their Approve only if it removes branch protection or allows a force-push to a trunk). Those + * two acts are LOOSEN_TOOL's alone (design 46 approval mark §21), so such a call is refused with nothing sent, naming it. */ export function forcePushToTrunk(rule: string, body: Record, trunks: string[]): string | null { const covered = trunks.filter((t) => ruleCovers(rule, t)); if (!covered.length) return null; const opens = body.enable_force_push === true || body.force_push_allowlist_deploy_keys === true || (["force_push_allowlist_usernames", "force_push_allowlist_teams"] as const).some((k) => Array.isArray(body[k]) && (body[k] as unknown[]).length > 0); if (!opens) return null; - return `the rule ${rule} covers ${covered.join(", ")}, and this call lets a force-push onto it: a force-push to a trunk is ` + - `the operator's Approve (the operator, 2026-10-28), and no tool of this module can ask it yet, so it is the operator's ` + - `at the forge. Nothing was sent`; + return `the rule ${rule} covers ${covered.join(", ")}, and this call lets a force-push onto it: removing a trunk's protection ` + + `or allowing a force-push onto a trunk is ${LOOSEN_TOOL}'s alone, which runs on the operator's Approve of that one call. ` + + `Nothing was sent`; } diff --git a/modules/gitea/test/protection-full.test.ts b/modules/gitea/test/protection-full.test.ts index 5da3d6016..8830e8885 100644 --- a/modules/gitea/test/protection-full.test.ts +++ b/modules/gitea/test/protection-full.test.ts @@ -79,6 +79,12 @@ const server = createServer((req, res) => { giteaEdit(r, body); return send(200, r); } + if (req.method === "DELETE" && m[1]) { + const i = rules.findIndex((x) => x.rule_name === decodeURIComponent(m[1])); + if (i < 0) return send(404, { message: "no rule" }); + rules.splice(i, 1); + res.writeHead(204); return res.end(); + } if (req.method === "POST" && !m[1]) { const r = { rule_name: body.rule_name }; giteaEdit(r, body); rules.push(r); return send(201, r); } send(405, { message: "no" }); }); @@ -132,6 +138,39 @@ test("an unknown field, a field of the wrong kind, no field, no reason or a logi }); // judgedRule's cases (mesh-control cmd/mesh-controller/build_source.go), each in its own words. +// Design 46 approval mark §21: a trunk's protection is loosened only by gitea_branch_protection_loosen, marked in the +// manifest's needs-approval; gitea_branch_protection_set_full refuses a force-push onto a trunk, naming it. +const loosen = (args: Record) => + tools.find((t) => t.name === "gitea_branch_protection_loosen")!.run({ owner: "novox", repo: "de-spiegel", branch: "main", why: "L2", ...args }); + +test("set_full refuses a force-push onto a trunk, nothing sent, naming gitea_branch_protection_loosen", async () => { + for (const args of [{ enable_force_push: true }, { enable_force_push: true, enable_force_push_allowlist: true, force_push_allowlist_usernames: ["jschoubben"] }]) { + await assert.rejects(setFull(args), /gitea_branch_protection_loosen's alone/); + } + assert.deepEqual(sent, []); +}); + +test("gitea_branch_protection_loosen removes a trunk's protection, or lets a force-push onto it, and says the trunk rule no longer holds", async () => { + const forced: any = await loosen({ act: "allow-force-push" }); + assert.deepEqual(sent, [{ method: "PATCH", path: "/repos/novox/de-spiegel/branch_protections/main", body: { enable_force_push: true } }]); + assert.equal(forced.before.fields.enable_force_push, false); + assert.equal(forced.after.fields.enable_force_push, true); + sent = []; + const removed: any = await loosen({ act: "remove-protection" }); + assert.deepEqual(sent, [{ method: "DELETE", path: "/repos/novox/de-spiegel/branch_protections/main", body: undefined }]); + assert.equal(removed.after, null); + assert.deepEqual(rules, []); + assert.equal(removed.trunk_rule.holds, false); + assert.match(removed.trunk_rule.why, /no protection rule covers main/); +}); + +test("gitea_branch_protection_loosen refuses an unknown act, no reason or a rule the repository lacks, nothing sent", async () => { + await assert.rejects(loosen({ act: "unprotect" }), /one of remove-protection, allow-force-push/); + await assert.rejects(loosen({ act: "remove-protection", why: "" }), /say why/); + await assert.rejects(loosen({ act: "remove-protection", branch: "release/*" }), /no protection rule named release\/\*/); + assert.deepEqual(sent, []); +}); + test("the trunk rule is judged as the controller judges it", () => { const ok = { rule_name: "main", enable_push: false, enable_merge_whitelist: true, merge_whitelist_usernames: ["mesh-tools"], block_admin_merge_override: true }; const cases: [any[], string, boolean, RegExp][] = [ diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index 4e9fd2af0..a33477b82 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -539,7 +539,7 @@ test("the tools register once there is a way to a token, and the first call mint "gitea_get_file", "gitea_list_branches", "gitea_delete_branch", - "gitea_branch_protection_get", "gitea_team_rights", "gitea_branch_protection_set", "gitea_branch_protection_set_full", "gitea_repo_settings_set_full", + "gitea_branch_protection_get", "gitea_team_rights", "gitea_branch_protection_set", "gitea_branch_protection_set_full", "gitea_branch_protection_loosen", "gitea_repo_settings_set_full", "gitea_default_branch_set", "gitea_pull_base_set", "gitea_commit_statuses", "gitea_open_pulls", "gitea_commit_trees", "gitea_contains", "gitea_list_labels", "gitea_create_label", diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index 8c7e73e2b..86ad0d0c9 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -12,7 +12,7 @@ import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools" import { emit } from "@novox/mesh-sdk/events"; import { GiteaAdmin, GiteaClient, type BranchProtection } from "../client.js"; import { readTeamRights } from "../teamrights.js"; -import { PROTECTION_FIELDS, REPO_FIELDS, alsoChanged, forcePushToTrunk, fullProtectionBody, trunksOf, givenFields, trunkRule, unapplied } from "../protectfull.js"; +import { LOOSEN_ACTS, LOOSEN_TOOL, PROTECTION_FIELDS, REPO_FIELDS, alsoChanged, forcePushToTrunk, fullProtectionBody, trunksOf, givenFields, trunkRule, unapplied } from "../protectfull.js"; import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js"; import { MERGE_METHODS, Refused, checkName, judgeApiCall, judgeContentsWrite, judgeDelete, judgeProtectionChange, pathSegments, routedPath, sentQuery, @@ -484,7 +484,7 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined "enable_push_whitelist and push_whitelist_deploy_keys need enable_push: true in the same call, the force-push allowlist " + "flags enable_force_push: true (the forge ignores them otherwise). The rule is read back after the write: a field the " + "forge did not apply fails the call, naming it; also_changed names what the forge changed beside the call. Generic: it " + - "tightens or loosens as told (the operator, 2026-10-28), so a reason (why) is required and logged; but a call that lets a force-push onto a rule covering a trunk is refused, nothing sent, until the module can ask the operator's Approve for it. A repository's trunk, the branch the mesh installs from, is main, the mesh's one trunk of every repository (said in trunks.from). As the forge's " + + "tightens or loosens as told (the operator, 2026-10-28), so a reason (why) is required and logged; but a call that lets a force-push onto a rule covering a trunk is refused, nothing sent, naming gitea_branch_protection_loosen, which alone removes a trunk's protection or lets a force-push onto it, on the operator's Approve. A repository's trunk, the branch the mesh installs from, is main, the mesh's one trunk of every repository (said in trunks.from). As the forge's " + "administrator. Answers the rule before and after, the repository's trunks, and trunk_rule: whether the branch is a trunk (is_trunk) and whether the controller now registers a module from " + "this branch (judgedRule, mesh-control build_source.go: no push, a merge whitelist naming mesh-tools and no " + "agent-, no administrator override) and why not.", @@ -530,6 +530,54 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined }, }, + { + name: LOOSEN_TOOL, + description: "Loosen a branch's protection rule in one of the two ways gitea_branch_protection_set_full refuses: remove the rule " + + "(act remove-protection), or let a force-push onto it (act allow-force-push, enable_force_push true). Marked in the " + + "module's needs-approval, so a call runs only on the operator's Approve of that one call (design 46 approval mark " + + "section 21); tightening stays gitea_branch_protection_set_full's, unasked. A reason (why) is required and logged. As " + + "the forge's administrator. Answers the rule before and after (null once removed), the repository's trunks, and " + + "trunk_rule: whether the controller still registers a module from the branch, and why not.", + input: { + owner: { type: "string", description: "the repository owner" }, + repo: { type: "string", description: "the repository name" }, + branch: { type: "string", description: "the branch (the rule's name, or a glob) whose rule is loosened" }, + act: { type: "string", enum: [...LOOSEN_ACTS], description: "remove-protection removes the rule; allow-force-push lets a force-push onto it" }, + why: { type: "string", description: "why, in a sentence: logged and answered" }, + }, + run: async (args) => { + const owner = String(args.owner), repo = String(args.repo), branch = String(args.branch ?? "").trim(); + const act = String(args.act ?? ""), why = String(args.why ?? "").trim(); + if (!branch) throw new Error("name the branch whose rule is loosened"); + if (!(LOOSEN_ACTS as readonly string[]).includes(act)) throw new Error(`act is ${JSON.stringify(act)}: it is one of ${LOOSEN_ACTS.join(", ")}`); + if (!why) throw new Error("say why (why): a branch's protection is loosened for a reason, which is logged"); + pathSegments("branch", branch); + const forge = admin ? admin() : gitea; + const rules = await forge.branchProtections(owner, repo); + const before = rules.find((p) => (p.rule_name ?? p.branch_name) === branch) ?? null; + if (!before) throw new Refused(`${owner}/${repo} has no protection rule named ${branch}: there is nothing to loosen. Nothing was sent`); + console.log(`[gitea] ${LOOSEN_TOOL} ${owner}/${repo} ${branch}: ${act} — why: ${why}`); + if (act === "remove-protection") await forge.deleteBranchProtection(owner, repo, branch); + else await forge.writeBranchProtection(owner, repo, branch, { enable_force_push: true }, false); + const after = await forge.branchProtections(owner, repo); + const read = after.find((p) => (p.rule_name ?? p.branch_name) === branch) ?? null; + if (act === "remove-protection" && read) { + throw new Error(`the forge answered the removal of ${owner}/${repo} ${branch}'s rule but still holds it: ${JSON.stringify(summarised(read))}`); + } + if (act === "allow-force-push") { + const missed = read ? unapplied({ enable_force_push: true }, read) : ["the rule itself (the forge no longer holds it)"]; + if (missed.length) throw new Error(`the forge answered the write of ${owner}/${repo} ${branch}'s rule but did not apply ${missed.join("; ")}`); + } + const trunks = trunksOf(); + return { + branch, act, why, + before: { ...summarised(before), fields: before }, + after: read ? { ...summarised(read), fields: read } : null, + trunks, trunk_rule: trunkRule(after, branch, trunks.names), + }; + }, + }, + { name: "gitea_repo_settings_set_full", description: "Set a repository's settings as the forge's administrator — its default branch (e.g. master to main), " + From bf60d6a8bb1671fa7d9c5b0fbe202e8e2b20d61a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:45:49 +0100 Subject: [PATCH 15/15] NATS3 (rest): the bus module plans from the catalog half of every watched branch, not from manifest-accepted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The keeper follows meshcatalogclient's branch follower and the seat list on its views reader, and plans again at each move of a branch's view, the registrations or the seat list. Each local name is planned from the first of the module's watched branches in its registration's order; another branch asking it with other settings is said drifted, naming the branch. A module of the new form whose resolved seat definition has a verb or an event gets its sent store; one including only parts, and a foundation part, none. A module in no watched branch's newest revision loses its plans key, its entries orphans. The sum of the caps counts the old sections the entries declare. The manifest consumes the catalog half's and the seat list's events and invokes their verbs; manifest-accepted and indexer_bus_state go. Builds: design 47 G §344, §433, §532, §533, §537, §538, §547, §591 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/nats/cmd/nats-tools/broker.go | 20 +- modules/nats/cmd/nats-tools/broker_test.go | 9 +- modules/nats/cmd/nats-tools/catalogview.go | 104 ++++++- modules/nats/cmd/nats-tools/keep.go | 279 +++++++++++------- modules/nats/cmd/nats-tools/keep_test.go | 94 +++--- modules/nats/cmd/nats-tools/keeper_health.go | 2 +- .../cmd/nats-tools/keeper_integration_test.go | 140 ++++++--- modules/nats/cmd/nats-tools/main.go | 2 +- modules/nats/cmd/nats-tools/manifest_test.go | 21 +- modules/nats/cmd/nats-tools/requests.go | 2 +- modules/nats/go.mod | 2 + modules/nats/internal/keeper/keeper.go | 236 ++++++++++----- modules/nats/internal/keeper/keeper_test.go | 109 +++++-- modules/nats/internal/keeper/plan.go | 9 +- modules/nats/internal/keeper/requests.go | 14 +- modules/nats/module.json | 16 +- 16 files changed, 746 insertions(+), 313 deletions(-) diff --git a/modules/nats/cmd/nats-tools/broker.go b/modules/nats/cmd/nats-tools/broker.go index a78b45d97..86421c683 100644 --- a/modules/nats/cmd/nats-tools/broker.go +++ b/modules/nats/cmd/nats-tools/broker.go @@ -48,13 +48,17 @@ func HeldInState() (map[string]keeper.Asked, error) { return out, nil } -// capsOf is the sum of the bus's caps against its limits (design 47 G §547, §661), in MiB; a limit the keeper has not read, -// or that is none, is absent. The sum is over every module, whatever list is narrowed to. -func capsOf(all map[string]keeper.Asked, limits func() (keeper.Limits, error)) *meshbroker.ListAnswerCaps { - c := &meshbroker.ListAnswerCaps{SumMiB: keeper.CapSum(all, "", "") / keeper.MiB} - if l, err := limits(); err == nil { - c.AccountLimitMiB, c.ServerLimitMiB = l.AccountBytes/keeper.MiB, l.ServerBytes/keeper.MiB +// capsOf is the sum of the bus's caps against its limits (design 47 G §547, §591, §661), in MiB: every entry made and +// every old section the catalog graph's entries declare, as the keeper's status last says them; a limit the keeper has +// not read, or that is none, is absent. The sum is over every module, whatever list is narrowed to. +func capsOf(all map[string]keeper.Asked, status func() (keeper.Status, error)) *meshbroker.ListAnswerCaps { + sum := keeper.CapSum(all, "", "") + c := &meshbroker.ListAnswerCaps{} + if s, err := status(); err == nil { + sum += s.OldBytes + c.AccountLimitMiB, c.ServerLimitMiB = s.Limits.AccountBytes/keeper.MiB, s.Limits.ServerBytes/keeper.MiB } + c.SumMiB = sum / keeper.MiB return c } @@ -167,7 +171,7 @@ func anyOf(m map[string]json.RawMessage) map[string]any { } // brokerTools are the four reading verbs and nats_history. -func brokerTools(read heldEntries, limits func() (keeper.Limits, error), jsz func() ([]byte, error), history func(module, local string, limit int) ([]keeper.Event, error)) []stdio.Tool { +func brokerTools(read heldEntries, status func() (keeper.Status, error), jsz func() ([]byte, error), history func(module, local string, limit int) ([]keeper.Event, error)) []stdio.Tool { both := func() (map[string]keeper.Asked, map[string]busStream, error) { entries, err := read() if err != nil { @@ -208,7 +212,7 @@ func brokerTools(read heldEntries, limits func() (keeper.Limits, error), jsz fun if err != nil { return nil, err } - return meshbroker.ListAnswer{Caps: capsOf(all, limits), Entries: each(entries, held, func(keeper.Made) bool { return true })}, nil + return meshbroker.ListAnswer{Caps: capsOf(all, status), Entries: each(entries, held, func(keeper.Made) bool { return true })}, nil }}, {Name: "mesh-broker.show", Description: "One entry of a module's kept data on the bus: its bus name, template and options as asked, the settings the bus holds of each of its objects, and its fill against its cap. Whether it is refused, waiting, drifted, full or orphaned is what nats_history says of it.", diff --git a/modules/nats/cmd/nats-tools/broker_test.go b/modules/nats/cmd/nats-tools/broker_test.go index a30b5407b..348b7b4eb 100644 --- a/modules/nats/cmd/nats-tools/broker_test.go +++ b/modules/nats/cmd/nats-tools/broker_test.go @@ -42,8 +42,8 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) { {"name":"m_log","config":{"name":"m_log","retention":"limits","discard":"new","max_bytes":33554432,"max_msg_size":262144},"state":{"messages":9,"bytes":9000}}]}]}` tools := map[string]stdio.Tool{} for _, tl := range brokerTools(func() (map[string]keeper.Asked, error) { return held, nil }, - func() (keeper.Limits, error) { - return keeper.Limits{AccountBytes: 2048 << 20, ServerBytes: 4096 << 20}, nil + func() (keeper.Status, error) { + return keeper.Status{Limits: keeper.Limits{AccountBytes: 2048 << 20, ServerBytes: 4096 << 20}, OldBytes: 256 << 20}, nil }, func() ([]byte, error) { return []byte(jsz), nil }, func(module, local string, limit int) ([]keeper.Event, error) { @@ -62,8 +62,9 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) { `"KV_m_drafts":{"max-bytes":67108864`, `"fill":{"bytes":1048576,"cap":83886080,"count":3}`, `"options":{"max-mib":16}`, // The binding of an entry the bus holds is the one its created carries (design 25 bindings-first-view §2). `"binding":{"bucket":"m_drafts","local":"drafts","objects":"m_drafts-objects","storage":"key-value","template":"state"}`}, ""}, - // The sum of the caps against both limits (design 47 G §547): drafts 64+16, log 16, old 1024 (orphaned, still held); later is not made. - {"mesh-broker.list", nil, []string{`"caps":{"account-limit-mib":2048,"server-limit-mib":4096,"sum-mib":1120`}, ""}, + // The sum of the caps against both limits (design 47 G §547, §591): drafts 64+16, log 16, old 1024 (orphaned, still + // held), and the old sections the catalog graph's entries declare, 256; later is not made. + {"mesh-broker.list", nil, []string{`"caps":{"account-limit-mib":2048,"server-limit-mib":4096,"sum-mib":1376`}, ""}, {"mesh-broker.show", map[string]any{"module": "m", "local-name": "log"}, []string{`"bus-name":"m_log"`, `"template":"log"`, `"m_log":{"max-bytes":33554432`}, ""}, {"mesh-broker.show", map[string]any{"module": "m", "local-name": "later"}, []string{`"local-name":"later"`}, ""}, diff --git a/modules/nats/cmd/nats-tools/catalogview.go b/modules/nats/cmd/nats-tools/catalogview.go index ee8fb77fe..4bff15451 100644 --- a/modules/nats/cmd/nats-tools/catalogview.go +++ b/modules/nats/cmd/nats-tools/catalogview.go @@ -1,10 +1,13 @@ package main import ( + "cmp" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" + "maps" + "slices" catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -12,15 +15,18 @@ import ( "nats-tools/internal/keeper" ) -// The keeper reads a module's asks from the module's entry in the catalog half of its revision view, not from the -// indexer's event (design 47 G §344, §433). +// The keeper reads a module's asks from the module's entry in the catalog half of each watched branch, not from the +// indexer's event (design 47 G §344, §433, §532). // Asks is what one catalog graph entry says of a module for the keeper: what its manifest asks of the bus as an accepted -// verdict, the seats it claims, and whether it declares an event of its own (the facts keeper.Sends reads). +// verdict, the seats it claims, whether it declares an event of its own (the facts keeper.Sends reads), whether it is a +// foundation part (an old `objects` section `sent-objects`, §533), and the caps of its old sections in bytes (§547, §591). type Asks struct { - Accepted keeper.Accepted - Claims []string - EmitsOwn bool + Accepted keeper.Accepted + Claims []string + EmitsOwn bool + Foundation bool + OldBytes int64 } // AsksOf reads one entry. An entry whose manifest is not accepted asks nothing of the bus now, as an accepted verdict of @@ -48,9 +54,95 @@ func AsksOf(e catalog.CatalogEntry) (Asks, error) { for _, c := range m.Claims { out.Claims = append(out.Claims, c.Name) } + // The old sections' caps (design 47 G §547, §591): an objects section its max-mib, a log or a stream its max-mib or + // the catalogue's default. + for _, o := range m.Objects { + out.Foundation = out.Foundation || o.Name == keeper.SentLocal + out.OldBytes += int64(o.MaxMiB) * keeper.MiB + } + for _, l := range m.Logs { + out.OldBytes += int64(cmp.Or(l.MaxMiB, catalogue.LogDefaultMiB)) * keeper.MiB + } + for _, st := range m.Streams { + out.OldBytes += int64(cmp.Or(st.MaxMiB, catalogue.StreamDefaultMiB)) * keeper.MiB + } return out, nil } +// Branch is one watched branch's newest revision of the catalog half: its name, its revision and its entries by module. +type Branch struct { + Name string + Revision int64 + Entries map[string]catalog.CatalogEntry +} + +// ViewOf is the catalog half as the keeper plans from it (design 47 G §532): every module some branch's newest revision +// lists, its kept data planned from each such branch, each local name once, from the first of the module's watched +// branches in its registration's order (order: the registration's branches; a branch it does not name comes after, in +// the order given) whose entry asks it; another branch asking that local name with other settings is a drift naming +// the branch, and plans nothing. The module sends when a branch's entry says so; it is a foundation part, and its old +// sections' caps are read, from the first branch. +func ViewOf(branches []Branch, order func(module string) []string) (keeper.CatalogView, error) { + modules := map[string]bool{} + for _, b := range branches { + for m := range b.Entries { + modules[m] = true + } + } + var view keeper.CatalogView + for _, module := range slices.Sorted(maps.Keys(modules)) { + var listing []Branch + var named []string + if order != nil { + named = order(module) + } + for _, name := range named { + for _, b := range branches { + if _, ok := b.Entries[module]; ok && b.Name == name { + listing = append(listing, b) + } + } + } + for _, b := range branches { + if _, ok := b.Entries[module]; ok && !slices.Contains(named, b.Name) { + listing = append(listing, b) + } + } + var cm keeper.CatalogModule + from := map[string]string{} + asked := map[string]string{} + for i, b := range listing { + a, err := AsksOf(b.Entries[module]) + if err != nil { + return keeper.CatalogView{}, fmt.Errorf("the branch %s: %w", b.Name, err) + } + if i == 0 { + cm = keeper.CatalogModule{Accepted: keeper.Accepted{Module: module, Digest: b.Entries[module].ManifestDigest, Revision: uint64(max(b.Revision, 0))}, + Foundation: a.Foundation, OldBytes: a.OldBytes} + } + cm.EmitsOwn = cm.EmitsOwn || a.EmitsOwn + for _, c := range a.Claims { + if !slices.Contains(cm.Claims, c) { + cm.Claims = append(cm.Claims, c) + } + } + for _, e := range a.Accepted.BusState { + settings := string(mustJSON(e)) + if f, ok := from[e.Name]; ok { + if asked[e.Name] != settings { + view.Drifts = append(view.Drifts, keeper.Drift{Module: module, Local: e.Name, Branch: b.Name, From: f}) + } + continue + } + from[e.Name], asked[e.Name] = b.Name, settings + cm.BusState = append(cm.BusState, e) + } + } + view.Modules = append(view.Modules, cm) + } + return view, nil +} + func mustJSON(v any) []byte { b, _ := json.Marshal(v) return b diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index ae96890b0..ed3625b2b 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -14,6 +14,8 @@ import ( "syscall" "time" + "sync" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -22,11 +24,14 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/sdk/sourcebranches" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatdef" + "git.novox.be/novox/mesh-sdk/go/seatlist" "git.novox.be/novox/mesh-sdk/go/views" "git.novox.be/novox/mesh-sdk/go/bussnap" @@ -34,33 +39,10 @@ import ( ) // The keeper (`nats-tools keep`): the bus module's process on its own account (own secret broker-keeper, ADR 0436 §1), -// started by the node-engine as the manifest's process keeper. It reads the fill every minute from its start, asks the -// indexer's first view once, then hears manifest-accepted and the operator's decided on its durable consumer, makes what -// each verdict asks for, and acts on the persons' requests its tools put in its own state requests (requests.go) as they -// are written or answered. It answers the seat mesh-broker's snapshot verbs itself (snapshots.go; design 47 G A-107). - -// AcceptedOf reads a manifest-accepted body: the seat mesh-catalog's typed event, with the indexer's revision of the -// verdict when the event carries it (0 from an indexer that does not say it). -func AcceptedOf(body []byte) (keeper.Accepted, error) { - var a catalog.ManifestAcceptedEvent - if err := json.Unmarshal(body, &a); err != nil { - return keeper.Accepted{}, err - } - if a.Module == "" { - return keeper.Accepted{}, fmt.Errorf("a manifest-accepted names no module") - } - var state []catalogue.BusStateEntry // the seat types an entry as the manifest wrote it; the keeper reads the catalogue's form - if raw, err := json.Marshal(a.BusState); err != nil || json.Unmarshal(raw, &state) != nil { - return keeper.Accepted{}, fmt.Errorf("the bus-state of the manifest-accepted of %s is not readable", a.Module) - } - if len(a.BusState) == 0 { - state = nil - } - return keeper.Accepted{Module: a.Module, Digest: a.Digest, Revision: uint64(max(a.Revision, 0)), BusState: state}, nil -} - -// AcceptedSubject is the seat mesh-catalog's manifest-accepted on the shared events stream. -var AcceptedSubject = catalog.SubjectManifestAccepted +// started by the node-engine as the manifest's process keeper. It reads the fill every minute from its start, follows the +// catalog half of every watched branch through meshcatalogclient's branch follower and the seat list (design 47 G §344, +// §433, §532, §614), plans again at each of their changes, hears the operator's decided on its durable consumer, and acts +// on the persons' requests its tools put in its own state requests (requests.go) as they are written or answered. It answers the seat mesh-broker's snapshot verbs itself (snapshots.go; design 47 G A-107). // The router's records of asks, read under this module's own name (ADR 0259 §3), as the mail sender reads them. const asksBucket = "messenger_asks" @@ -190,36 +172,6 @@ func sayer(js jetstream.JetStream) func(oc.Condition) error { } } -// firstView asks the indexer's indexer_bus_state as the keeper's own account. -func firstView(c *process.Conn) func() (keeper.FirstView, error) { - return func() (keeper.FirstView, error) { - raw, err := c.Call(keeper.FirstViewOwner+"."+keeper.FirstViewVerb, map[string]any{}) - if err != nil { - return keeper.FirstView{}, err - } - return FirstViewOf(raw) - } -} - -// FirstViewOf reads indexer_bus_state's answer, as the tool runner hands it back ({result, error} or bare). -func FirstViewOf(raw []byte) (keeper.FirstView, error) { - var wrapped struct { - Result json.RawMessage `json:"result"` - Error string `json:"error"` - } - if json.Unmarshal(raw, &wrapped) == nil && (wrapped.Result != nil || wrapped.Error != "") { - if wrapped.Error != "" { - return keeper.FirstView{}, fmt.Errorf("%s", wrapped.Error) - } - raw = wrapped.Result - } - var v keeper.FirstView - if err := json.Unmarshal(raw, &v); err != nil { - return keeper.FirstView{}, fmt.Errorf("the answer is not a first view: %w", err) - } - return v, nil -} - func percent(getenv func(string) string, name string, fallback int64) int64 { if n, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64); err == nil && n > 0 && n < 100 { return n @@ -261,7 +213,7 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer k := &keeper.Keeper{ Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR")), NC: nc, Dir: getenv("MESH_NATS_SNAPSHOTS_DIR"), Log: logger.Printf, Server: serverStorageLimit(LiveMonitor())}, - Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn), + Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), Act: func(what, why, cause string) error { _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause}) return err @@ -274,25 +226,168 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer SnapshotsCap: settings.SnapshotsCap, SnapshotKeepDays: settings.SnapshotKeepDays, } + // The catalog half (design 47 G §433, §532, §614): meshcatalogclient's branch follower on the keeper's views reader, which + // holds its finding writer; and the seat list, each definition as the seat answers that one seat, resolved (§537). Each + // change of a branch's view, of the registrations or of the seat list plans again. + moved := make(chan struct{}, 1) + wakeMoved := func() { + select { + case moved <- struct{}{}: + default: + } + } + follow, err := followCatalog(ctx, conn, writer, wakeMoved, logger.Printf) + if err != nil { + return err + } + k.Catalog, k.Seats, k.Again = follow.view, follow.seats, follow.again // The process answers the seat mesh-broker's snapshot verbs itself (design 47 G §681, A-107), with the caller the bus proved. stopAnswers, err := conn.AnswerSeat(meshbroker.Seat, snapshotAnswers(k, heldRequest(k.Store), keeperSnapshots(nc, logger.Printf))) if err != nil { return fmt.Errorf("the seat %s's snapshot verbs cannot be answered: %w", meshbroker.Seat, err) } defer stopAnswers() - logger.Printf("started on the account %s: asking %s.%s for the first view", account.User, keeper.FirstViewOwner, keeper.FirstViewVerb) - return Keep(ctx, k, hearer(js, account, logger.Printf), requestsOf(conn), logger.Printf) + logger.Printf("started on the account %s: following the catalog half of every watched branch through the seat %s", account.User, catalog.Seat) + return Keep(ctx, k, hearer(js, account, follow.take, follow.bound, logger.Printf), requestsOf(conn), moved, logger.Printf) } -// hearer binds the keeper's durable consumer on EVENTS (_nats_keeper, filtered on manifest-accepted) and hands -// each message to handle, which acknowledges it once applied. -func hearer(js jetstream.JetStream, a process.Account, logf func(string, ...any)) func(context.Context, func(subject string, body []byte) error) error { +// catalogFollow is the keeper's reading of the catalog half and the seat list (design 47 G §532, §614): the reader, the +// branch follower over it, the seat list, and the events of both, routed from the keeper's one durable consumer. +type catalogFollow struct { + ctx context.Context + reader *views.ProcessReader + branches *meshcatalogclient.Branches + seatList *seatlist.Reader + routes *eventRoutes +} + +// followCatalog wires the branch follower and the seat list, and asks their first views; moved is told of each change. +func followCatalog(ctx context.Context, conn *process.Conn, writer views.WireWriter, moved func(), logf func(string, ...any)) (*catalogFollow, error) { + f := &catalogFollow{ctx: ctx, routes: &eventRoutes{by: map[string]func(jetstream.Msg){}}} + f.reader = &views.ProcessReader{Module: keeper.Module, Call: conn.Call, Logf: logf, Findings: func() views.WireWriter { return writer }} + branches, err := meshcatalogclient.FollowBranchesProcess(f.reader, conn.Call) + if err != nil { + return nil, fmt.Errorf("the catalog half cannot be followed: %w", err) + } + branches.Logf = logf + branches.Changed = func(string, string, json.RawMessage, bool, time.Time) { moved() } + // A registration that changes its watched branches' order moves what is planned from which branch (§532). + reconcile := branches.Registrations.Follower.Changed + branches.Registrations.Follower.Changed = func(key string, value json.RawMessage, deleted bool, at time.Time) { + reconcile(key, value, deleted, at) + moved() + } + f.branches = branches + f.seatList, err = seatlist.Start(seatlist.Config{Bus: seatlist.ProcessBus(conn.Call, f.routes), Bootstrap: seatlist.Bootstrapped(), + OnChange: func([]catalogue.Seat) { moved() }}) + if err != nil { + logf("the seat list cannot be followed (%v): a module's sent store waits for it", err) + } + f.reader.Ask(ctx, nil) + return f, nil +} + +// view is the catalog half as the keeper plans from it: an error while a watched branch has no first view yet. +func (f *catalogFollow) view() (keeper.CatalogView, error) { + if err := f.branches.Ready(); err != nil { + return keeper.CatalogView{}, err + } + var bs []Branch + for _, name := range f.branches.Followed() { + if c, ok := f.branches.Catalog(name); ok { + if half, ok := c.View(); ok { + bs = append(bs, Branch{Name: name, Revision: half.Revision.Revision, Entries: half.Entries}) + } + } + } + return ViewOf(bs, func(module string) []string { + reg, ok := f.branches.Registrations.Registration(module) + if !ok { + return nil + } + return reg.Branches + }) +} + +// seats are the seat list's definitions once the seat has answered; before, none, so no sent store is planned yet +// (ADR 0636 §1: a reader that has not heard acts on nothing new). +func (f *catalogFollow) seats() map[string]seatdef.Definition { + if f.seatList == nil || !f.seatList.Status().Heard { + return nil + } + return f.seatList.Definitions() +} + +// again asks every first view of the catalog half and the seat list again, by hand. +func (f *catalogFollow) again() error { + var errs []error + for _, fl := range f.reader.Followers() { + errs = append(errs, fl.Again(f.ctx, "asked again by hand (nats_first_view_again)")) + } + if f.seatList != nil { + errs = append(errs, f.seatList.Reload()) + } + return errors.Join(errs...) +} + +// take hands a message of the catalog half or the seat list to its reader, which acknowledges it; false for any other. +func (f *catalogFollow) take(m jetstream.Msg) bool { + if h := f.routes.get(m.Subject()); h != nil { + h(m) + return true + } + if strings.HasPrefix(m.Subject(), "mesh.seat."+catalog.Seat+".event.") || strings.HasPrefix(m.Subject(), seatdef.EventSubject(sourcebranches.Seat, sourcebranches.EventSourceMoved)) { + f.reader.Hear(f.ctx)(m) + return true + } + return false +} + +// bound tells each view the bus made the keeper's consumer then, so a first view taken before is asked again. +func (f *catalogFollow) bound(created time.Time) { + for _, fl := range f.reader.Followers() { + fl.ConsumerMade(f.ctx, created) + } +} + +// eventRoutes are the seat list's handlers of its events by subject, on the keeper's one durable consumer. +type eventRoutes struct { + mu sync.Mutex + by map[string]func(jetstream.Msg) +} + +func (r *eventRoutes) Handle(subject string, handle func(jetstream.Msg)) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.by[subject] != nil { + return fmt.Errorf("the events %q are handled already", subject) + } + r.by[subject] = handle + return nil +} + +func (r *eventRoutes) get(subject string) func(jetstream.Msg) { + r.mu.Lock() + defer r.mu.Unlock() + return r.by[subject] +} + +// hearer binds the keeper's durable consumer on EVENTS (_nats_keeper, filtered on its consumes): a message of the +// catalog half or the seat list goes to take, which acknowledges it; every other to handle, acknowledged once applied. +// bound is told when the bus made the consumer. +func hearer(js jetstream.JetStream, a process.Account, take func(jetstream.Msg) bool, bound func(time.Time), logf func(string, ...any)) func(context.Context, func(subject string, body []byte) error) error { return func(ctx context.Context, handle func(string, []byte) error) error { cons, err := js.Consumer(ctx, process.EventsStream, a.QueueConsumerName()) if err != nil { return fmt.Errorf("the keeper's consumer %s on %s: %w", a.QueueConsumerName(), process.EventsStream, err) } + if info, err := cons.Info(ctx); err == nil && bound != nil { + bound(info.Created.UTC()) + } cc, err := process.HearEvents(cons, func(m jetstream.Msg) { + if take != nil && take(m) { + return + } if err := handle(m.Subject(), m.Data()); err != nil { logf("%v: delivered again", err) _ = m.NakWithDelay(30 * time.Second) @@ -316,39 +411,25 @@ var FillEvery = time.Minute var DecidedSubject = oc.SubjectDecided(keeper.Module) // Keep is the keeper's loop, apart from the bus so a test drives it: a fill pass at once and every minute, from its -// start, so the store's one guard runs while the first view is still asked (design 25 § 12.3); the first view asked -// until it answers, on its backoff; then hear, which applies each manifest-accepted and wakes the persons' requests on -// the operator channel's decided for the bus module's own asks; the waiting entries made again on the same backoff; and -// the persons' requests acted on as they are written or answered, never on a timer. It returns when ctx ends. +// start, so the store's one guard runs while the catalog half's first views are still asked (design 25 § 12.3); a plan +// from the catalog half at once and at each move of it (moved: a branch's view, the registrations or the seat list +// changed), nothing planned until every watched branch has its first view (design 47 G §532, §614); hear, which wakes the +// persons' requests on the operator channel's decided for the bus module's own asks; the waiting entries made again on +// a backoff; and the persons' requests acted on as they are written or answered, never on a timer. It returns when ctx +// ends. func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func(string, []byte) error) error, - requests func(context.Context, func()) error, logf func(string, ...any)) error { + requests func(context.Context, func()) error, moved <-chan struct{}, logf func(string, ...any)) error { tick := time.NewTicker(FillEvery) defer tick.Stop() k.FillPass() k.SnapshotsAtStart() k.EndCutOffRestores() - var wait time.Duration - for { - next, err := k.AskFirstView(wait) - if err == nil { - break - } - logf("the first view did not answer (%v): nothing is created or orphaned until it does; asked again in %s", err, next) - wait = next - again := time.After(next) - waiting: - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-tick.C: - k.FillPass() - case <-again: - break waiting - } + plan := func() { + if err := k.PlanCatalog(); err != nil { + logf("nothing is planned yet: %v", err) } } - logf("the first view answered at revision %d", k.Status().FirstView.Revision) + plan() wake := func(c chan struct{}) func() { return func() { select { @@ -357,31 +438,20 @@ func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func } } } - heard, asked := make(chan struct{}, 1), make(chan struct{}, 1) + asked := make(chan struct{}, 1) for backoff := 10 * time.Second; ; backoff = min(backoff*2, 5*time.Minute) { err := hear(ctx, func(subject string, body []byte) error { if subject == DecidedSubject { // The operator answered one of the bus module's asks: the waiting requests are read again now. wake(asked)() - return nil } - if subject != AcceptedSubject && !strings.HasSuffix(subject, ".event."+catalog.EventManifestAccepted) { - return nil - } - a, err := AcceptedOf(body) - if err != nil { - logf("a manifest-accepted cannot be read (%v): taken, and nothing made of it", err) - return nil - } - k.Heard(a) - wake(heard)() return nil }) if err == nil { break } - logf("manifest-accepted cannot be heard yet (%v): asked again in %s", err, backoff) - // The fill pass runs on while the consumer is bound again (review round 2, running F1). + logf("the keeper's consumer cannot be heard yet (%v): asked again in %s", err, backoff) + // The fill pass and the plans run on while the consumer is bound again (review round 2, running F1). bound := time.After(backoff) binding: for { @@ -390,6 +460,8 @@ func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func return ctx.Err() case <-tick.C: k.FillPass() + case <-moved: + plan() case <-bound: break binding } @@ -415,7 +487,8 @@ func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func k.FillPass() case <-asked: k.ActOnRequests() - case <-heard: + case <-moved: + plan() case <-again: again = nil k.CheckAll() diff --git a/modules/nats/cmd/nats-tools/keep_test.go b/modules/nats/cmd/nats-tools/keep_test.go index 40a78a304..a3d2035fc 100644 --- a/modules/nats/cmd/nats-tools/keep_test.go +++ b/modules/nats/cmd/nats-tools/keep_test.go @@ -6,67 +6,73 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-sdk/go/catalogue" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "nats-tools/internal/keeper" ) -// The keeper's inputs (design 25 § 12.2; CAT-N2): a manifest-accepted read with the indexer's revision when it carries -// one (0 when not yet), its entries whole; the first view of indexer_bus_state read bare or as the tool runner wraps it, -// an error answer an error; anything else refused, naming why. -func TestTheKeepersInputsAreRead(t *testing.T) { +// The catalog half as the keeper plans from it (design 47 G §532, §533, §547, §591): every module some watched branch's +// newest revision lists, each local name from the first of its watched branches in its registration's order whose entry +// asks it, another branch asking it with other settings a drift naming that branch; a branch the registration does not +// name after; a verdict not accepted asks nothing; the claims of every branch; a foundation part and its old sections' +// caps from the first branch. +func TestTheCatalogHalfIsReadPerWatchedBranch(t *testing.T) { + entryOf := func(verdict string, manifest map[string]any) catalog.CatalogEntry { + manifest["module"] = "m" + return catalog.CatalogEntry{Module: "m", Verdict: verdict, ManifestDigest: "sha256:" + verdict, Manifest: manifest} + } + state := func(opts string) map[string]any { + e := map[string]any{"name": "drafts", "storage": "key-value", "template": "state"} + if opts != "" { + e["options"] = map[string]any{"history": opts} + } + return e + } + log := map[string]any{"name": "log", "storage": "stream", "template": "log"} + main := entryOf("accepted", map[string]any{"bus-state": []any{state("")}, "claims": []any{map[string]any{"name": "a"}}}) + feature := entryOf("accepted", map[string]any{"bus-state": []any{state("5"), log}, "claims": []any{map[string]any{"name": "b"}}}) for _, row := range []struct { - name, body, want, refused string - view bool + name string + branches []Branch + order []string + want string }{ - {name: "accepted with its revision", body: `{"v":1,"module":"m","digest":"sha256:a","judge":"j","revision":7,"bus-state":[{"name":"drafts","storage":"key-value","template":"state"}]}`, - want: "m sha256:a 7 [drafts/key-value/state]"}, - {name: "accepted before the event carries a revision", body: `{"v":1,"module":"m","digest":"sha256:b","judge":"j","bus-state":[]}`, - want: "m sha256:b 0 []"}, - {name: "accepted naming no module", body: `{"v":1,"digest":"x"}`, refused: "names no module"}, - {name: "first view as the tool runner answers it", view: true, - body: `{"result":{"revision":12,"modules":[{"module":"m","digest":"d","revision":11,"bus-state":[{"name":"log","storage":"stream","template":"log"}]}]}}`, - want: "12: m d 11 [log/stream/log]"}, - {name: "first view bare", view: true, body: `{"revision":3,"modules":[]}`, want: "3:"}, - {name: "first view answering an error", view: true, body: `{"error":"not granted"}`, refused: "not granted"}, - {name: "first view that is none", view: true, body: `[1,2]`, refused: "is not a first view"}, + {"one branch", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}}, []string{"main"}, + "m 3 [drafts/key-value/state] [a] false 0;"}, + {"a feature branch asking a local name with other settings drifts, its new one planned", + []Branch{{Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}, {Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}}, + []string{"main", "feature"}, "m 3 [drafts/key-value/state log/stream/log] [a b] false 0; drifted m drafts feature from main"}, + {"the registration's order decides", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}, {Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}}, + []string{"feature"}, "m 9 [drafts/key-value/state log/stream/log] [b a] false 0; drifted m drafts main from feature"}, + {"a verdict not accepted asks nothing", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("refused", map[string]any{"bus-state": []any{state("")}})}}}, []string{"main"}, + "m 3 [] [] false 0;"}, + {"a foundation part, its old sections counted", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("accepted", map[string]any{ + "objects": []any{map[string]any{"name": "sent-objects", "max-mib": 256}}, "logs": []any{map[string]any{"name": "l", "max-mib": 4}}, "streams": []any{map[string]any{"name": "s"}}})}}}, []string{"main"}, + "m 3 [] [] true 1284;"}, } { t.Run(row.name, func(t *testing.T) { + v, err := ViewOf(row.branches, func(string) []string { return row.order }) + if err != nil { + t.Fatal(err) + } var got string - var err error - if row.view { - v, e := FirstViewOf([]byte(row.body)) - err = e - got = fmt.Sprintf("%d:", v.Revision) - for _, m := range v.Modules { - got += " " + line(m.Module, m.Digest, m.Revision, m.BusState) + for _, m := range v.Modules { + es := []string{} + for _, e := range m.BusState { + es = append(es, e.Name+"/"+e.Storage+"/"+e.Template) } - } else { - a, e := AcceptedOf([]byte(row.body)) - err = e - got = line(a.Module, a.Digest, a.Revision, a.BusState) + got += fmt.Sprintf("%s %d [%s] %v %v %d;", m.Module, m.Revision, strings.Join(es, " "), m.Claims, m.Foundation, m.OldBytes/keeper.MiB) } - if row.refused != "" { - if err == nil || !strings.Contains(err.Error(), row.refused) { - t.Fatalf("read %s (%v): refused, naming %q", got, err, row.refused) - } - return + for _, d := range v.Drifts { + got += fmt.Sprintf(" drifted %s %s %s from %s", d.Module, d.Local, d.Branch, d.From) } - if err != nil || got != row.want { - t.Errorf("read %q (%v), want %q", got, err, row.want) + if got != row.want { + t.Errorf("read %q, want %q", got, row.want) } }) } } -func line(module, digest string, revision uint64, entries []catalogue.BusStateEntry) string { - es := []string{} - for _, e := range entries { - es = append(es, e.Name+"/"+e.Storage+"/"+e.Template) - } - return fmt.Sprintf("%s %s %d [%s]", module, digest, revision, strings.Join(es, " ")) -} - // The keeper's health tool (CAT-N6): healthy naming its pass; unhealthy, naming why, when its pass is old or verdicts // wait on its consumer `_nats_keeper` past the bound. func TestTheKeepersHealthToolReadsItsStatusAndItsConsumer(t *testing.T) { diff --git a/modules/nats/cmd/nats-tools/keeper_health.go b/modules/nats/cmd/nats-tools/keeper_health.go index 66f8ef75e..3a1acbbde 100644 --- a/modules/nats/cmd/nats-tools/keeper_health.go +++ b/modules/nats/cmd/nats-tools/keeper_health.go @@ -82,7 +82,7 @@ func keeperHealth(status func() (keeper.Status, error), read heldEntries, jsz fu func keeperHealthTool(status func() (keeper.Status, error), read heldEntries, jsz func() ([]byte, error)) stdio.Tool { return stdio.Tool{Name: "nats_keeper_health", - Description: "The bus module's keeper's health, as the node-engine asks it: unhealthy, naming why, when its fill pass is more than three minutes old, its first view (the indexer's indexer_bus_state) has not answered, or accepted verdicts wait on its consumer unanswered past three minutes; it names beside the entries it could not create, the ones waiting, and those past the raise mark (another module's full entry raises that module's condition and leaves the keeper healthy).", + Description: "The bus module's keeper's health, as the node-engine asks it: unhealthy, naming why, when its fill pass is more than three minutes old, the catalog half's first views of its watched branches (the seat mesh-catalog's revision) have not answered, or events wait on its consumer unanswered past three minutes; it names beside the entries it could not create, the ones waiting, and those past the raise mark (another module's full entry raises that module's condition and leaves the keeper healthy).", Run: func(map[string]any) (any, error) { return keeperHealth(status, read, jsz, func() time.Time { return time.Now().UTC() }), nil }} diff --git a/modules/nats/cmd/nats-tools/keeper_integration_test.go b/modules/nats/cmd/nats-tools/keeper_integration_test.go index 832b9505a..6cc5cbb94 100644 --- a/modules/nats/cmd/nats-tools/keeper_integration_test.go +++ b/modules/nats/cmd/nats-tools/keeper_integration_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "os" "slices" "strings" @@ -24,8 +25,10 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/seatdef" "git.novox.be/novox/mesh-sdk/go/views" "git.novox.be/novox/mesh-sdk/go/bussnap" @@ -108,7 +111,8 @@ func eventually(t *testing.T, what string, ok func() bool) { } // The bus module runs whole (design 25 § 12.7, "The bus module runs whole" and "The bus module's first view"; CAT-N10): -// its keeper against the real bus server, hearing manifest-accepted on its durable consumer, creating on the bus, +// its keeper against the real bus server, planning from the catalog half of every watched branch and the seat list +// (design 47 G §344, §532, §533, §537, §538) at each move of them, creating on the bus, // saying its events on its own stream BROKER_EVENTS, keeping its state in its own buckets, and acting on a person's // requests. One server for every case, in order, as the mesh runs them. func TestTheBusModuleRunsWhole(t *testing.T) { @@ -144,7 +148,7 @@ func TestTheBusModuleRunsWhole(t *testing.T) { // The keeper's consumer hears the indexer's verdicts and the operator channel's decided for the bus module's own // asks, as the grants part composes it from its consumes and the seats it uses. if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: account.QueueConsumerName(), - FilterSubjects: []string{AcceptedSubject, DecidedSubject}, AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 5}); err != nil { + FilterSubjects: []string{DecidedSubject}, AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 5}); err != nil { t.Fatal(err) } // The router's own record of the asks, which the keeper reads under its own process account (process.Conn.Ask), never @@ -166,7 +170,18 @@ func TestTheBusModuleRunsWhole(t *testing.T) { store := kvStore{js} snapshots := t.TempDir() var mu sync.Mutex - view := keeper.FirstView{Revision: 1} + // The catalog half as meshcatalogclient's branch follower holds it, branch by branch in the registrations' order, and + // the seat list's definitions; each change moves the keeper, as the follower's and the seat list's changes do. + branches := []Branch{{Name: "main", Revision: 1, Entries: map[string]catalog.CatalogEntry{}}} + order := []string{"main"} + defs := map[string]seatdef.Definition{} + moved := make(chan struct{}, 1) + move := func() { + select { + case moved <- struct{}{}: + default: + } + } var acts []string // The bus module's findings go through a held writer over its state findings, started as the keeper's process starts it. writer, err := views.NewHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ @@ -182,7 +197,13 @@ func TestTheBusModuleRunsWhole(t *testing.T) { newKeeper := func() *keeper.Keeper { return &keeper.Keeper{Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, snapshots), NC: nc, Dir: snapshots, Log: t.Logf}, Store: store, Finding: writer, Emit: streamEmitter(js, account), Say: sayer(js), - View: func() (keeper.FirstView, error) { mu.Lock(); defer mu.Unlock(); return view, nil }, + Catalog: func() (keeper.CatalogView, error) { + mu.Lock() + defer mu.Unlock() + return ViewOf(branches, func(string) []string { return order }) + }, + Seats: func() map[string]seatdef.Definition { mu.Lock(); defer mu.Unlock(); return maps.Clone(defs) }, + Again: func() error { return nil }, Act: func(what, why, cause string) error { mu.Lock() defer mu.Unlock() @@ -208,15 +229,28 @@ func TestTheBusModuleRunsWhole(t *testing.T) { }() return nil } - go func() { _ = Keep(kctx, k, hearer(js, account, t.Logf), watch, t.Logf); close(done) }() + go func() { _ = Keep(kctx, k, hearer(js, account, nil, nil, t.Logf), watch, moved, t.Logf); close(done) }() return func() { stop(); <-done } } - accepted := func(module string, revision uint64, entries ...catalogue.BusStateEntry) { - body, _ := json.Marshal(map[string]any{"v": 1, "module": module, "digest": fmt.Sprintf("sha256:%d", revision), - "judge": "j", "revision": revision, "bus-state": entries}) - if _, err := js.Publish(ctx, AcceptedSubject, body); err != nil { - t.Fatal(err) + // onBranch puts a module's entry in a branch's newest revision (none: the module leaves it), and moves the keeper. + onBranch := func(branch, module string, revision int64, manifest map[string]any) { + mu.Lock() + for i := range branches { + if branches[i].Name == branch { + branches[i].Revision = revision + if manifest == nil { + delete(branches[i].Entries, module) + } else { + manifest["module"] = module + branches[i].Entries[module] = catalog.CatalogEntry{Module: module, Verdict: "accepted", ManifestDigest: fmt.Sprintf("sha256:%d", revision), Manifest: manifest} + } + } } + mu.Unlock() + move() + } + accepted := func(module string, revision uint64, entries ...catalogue.BusStateEntry) { + onBranch("main", module, int64(revision), map[string]any{"bus-state": entries}) } // saidOn reads what one stream holds of the bus module's events, by its subjects' prefix: its own stream (its // history), or EVENTS on the seat mesh-broker's subjects (what every consumer of mesh-broker. hears). @@ -264,11 +298,13 @@ func TestTheBusModuleRunsWhole(t *testing.T) { // one guard never waits on the indexer. t.Run("the fill pass runs while the first view is asked", func(t *testing.T) { down := newKeeper() - down.View = func() (keeper.FirstView, error) { return keeper.FirstView{}, errors.New("no responders") } + down.Catalog = func() (keeper.CatalogView, error) { + return keeper.CatalogView{}, errors.New("the catalog half has no first view of main yet") + } stopDown := run(down) eventually(t, "a fill pass with no first view", func() bool { return !down.Status().Pass.IsZero() && down.Status().FirstView.Answered.IsZero() }) stopDown() - // And while its consumer of manifest-accepted is bound again on its backoff (review round 2, running F1). + // And while its consumer is bound again on its backoff (review round 2, running F1). was := FillEvery FillEvery = 20 * time.Millisecond defer func() { FillEvery = was }() @@ -276,7 +312,7 @@ func TestTheBusModuleRunsWhole(t *testing.T) { kctx, stopDeaf := context.WithCancel(ctx) done := make(chan struct{}) go func() { - _ = Keep(kctx, deaf, func(context.Context, func(string, []byte) error) error { return errors.New("no consumer yet") }, nil, t.Logf) + _ = Keep(kctx, deaf, func(context.Context, func(string, []byte) error) error { return errors.New("no consumer yet") }, nil, nil, t.Logf) close(done) }() eventually(t, "the first view answered", func() bool { return !deaf.Status().FirstView.Answered.IsZero() }) @@ -309,27 +345,6 @@ func TestTheBusModuleRunsWhole(t *testing.T) { stop = run(k) eventually(t, "created n jobs", saidHas("created n jobs")) }) - t.Run("a replay older than the first view is not applied; a set-aside module keeps its key", func(t *testing.T) { - stop() - mu.Lock() - view = keeper.FirstView{Revision: 9, Modules: []keeper.Accepted{{Module: "p", Revision: 9, BusState: []catalogue.BusStateEntry{state}}}} - mu.Unlock() - accepted("p", 4) - k = newKeeper() - stop = run(k) - eventually(t, "created p drafts", saidHas("created p drafts")) - eventually(t, "the replay heard", func() bool { - info, _ := js.Consumer(ctx, "EVENTS", account.QueueConsumerName()) - i, _ := info.Info(ctx) - return i.NumPending == 0 && i.NumAckPending == 0 - }) - if saidHas("orphaned p drafts")() { - t.Error("the replay at revision 4 orphaned what the first view at 9 asked") - } - if _, _, found, _ := store.Get(keeper.StateAsked, "m"); !found { - t.Error("m, missing from the first view, lost its key") - } - }) t.Run("drift is said, a growth applied and the drift cleared", func(t *testing.T) { accepted("m", 5, state, entry("log", "stream", "log", map[string]string{"max-mib": "1", "max-age-days": "2"})) eventually(t, "drifted m log", saidHas("drifted m log")) @@ -554,6 +569,63 @@ func TestTheBusModuleRunsWhole(t *testing.T) { t.Error("the snapshot's record outlived its file") } }) + // §532: a branch registered while the keeper runs is followed with no restart; a feature branch asking a local name with + // other settings than the default branch's is said drifted, naming it, while the default branch's are planned, and its + // new local name is planned from it. + t.Run("a branch registered while it runs is planned from, a local name with other settings drifted", func(t *testing.T) { + mu.Lock() + branches = append(branches, Branch{Name: "feature", Revision: 4, Entries: map[string]catalog.CatalogEntry{"m": {Module: "m", Verdict: "accepted", + Manifest: map[string]any{"module": "m", "bus-state": []any{state, entry("log", "stream", "log", map[string]string{"max-mib": "2"}), entry("extra", "key-value", "state", nil)}}}}}) + order = []string{"main", "feature"} + mu.Unlock() + move() + eventually(t, "created m extra", saidHas("created m extra")) + eventually(t, "drifted m log", saidHas("drifted m log")) + if info, _ := held("m_log"); info.Config.MaxBytes != 4<<20 { + t.Errorf("m_log's cap is %d: the default branch's 4 MiB planned", info.Config.MaxBytes) + } + mu.Lock() + branches, order = branches[:1], []string{"main"} + mu.Unlock() + move() + eventually(t, "orphaned m extra", saidHas("orphaned m extra")) + }) + // §344, §533, §537: a module of the new form claiming a seat with a verb gets a sent store, one with neither none, a seat + // whose verbs come only from an included part one, a seat including only bodies none, a foundation part none; a + // definition gaining a verb gives its holder one. §538: a module in no watched branch's newest revision loses its plans key. + t.Run("the sent stores follow the catalog half and the seat list", func(t *testing.T) { + mu.Lock() + defs["verbed"] = seatdef.Definition{Seat: "verbed", Verbs: []seatdef.Verb{{Name: "go"}}} + defs["bare"] = seatdef.Definition{Seat: "bare"} + defs["resolved"] = seatdef.Definition{Seat: "resolved", Includes: []string{"part"}, Verbs: []seatdef.Verb{{Name: "from-the-part"}}} + defs["bodies"] = seatdef.Definition{Seat: "bodies", Includes: []string{"shared-bodies"}} + mu.Unlock() + claims := func(seat string) map[string]any { return map[string]any{"claims": []any{map[string]any{"name": seat}}} } + onBranch("main", "sa", 5, claims("verbed")) + onBranch("main", "sb", 5, claims("bare")) + onBranch("main", "sc", 5, claims("resolved")) + onBranch("main", "sd", 5, claims("bodies")) + f := claims("verbed") + f["objects"] = []any{map[string]any{"name": "sent-objects", "max-mib": 256}} + onBranch("main", "se", 5, f) + eventually(t, "sa and sc's sent stores", func() bool { + _, a := held("OBJ_sa_sent-objects") + _, c := held("OBJ_sc_sent-objects") + return a && c + }) + for _, none := range []string{"sb", "sd", "se"} { + if _, ok := held("OBJ_" + none + "_sent-objects"); ok { + t.Errorf("%s has a sent store", none) + } + } + mu.Lock() + defs["bodies"] = seatdef.Definition{Seat: "bodies", Includes: []string{"shared-bodies"}, Verbs: []seatdef.Verb{{Name: "new"}}} + mu.Unlock() + move() + eventually(t, "sd's sent store once its seat gains a verb", func() bool { _, ok := held("OBJ_sd_sent-objects"); return ok }) + onBranch("main", "sb", 6, nil) + eventually(t, "sb's plan deleted", func() bool { _, _, found, _ := store.Get(keeper.StatePlans, "sb"); return !found }) + }) t.Run("nats_first_view_again asks at once and records the hand act", func(t *testing.T) { raw, _ := json.Marshal(keeper.Request{ID: "r3", Verb: keeper.VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"}) _, _ = store.Put(keeper.StateRequests, "r3", raw) diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index a82f859bf..d60eae4fc 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -193,7 +193,7 @@ func allTools() []stdio.Tool { } out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState)...) out = append(out, expiryTools(LiveBus(), SaidByTheRuntime, RecordedBySeat)...) - out = append(out, brokerTools(HeldInState, func() (keeper.Limits, error) { s, err := StatusInState(); return s.Limits, err }, jsz, StreamHistory(LiveBus()))...) + out = append(out, brokerTools(HeldInState, StatusInState, jsz, StreamHistory(LiveBus()))...) out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz)) out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...) rs := RequestsInState() diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index e492b319a..a6330a88a 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -161,8 +161,8 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { } } // The bus module provides bus state (design 25 § 12.2-12.3, ADR 0595 §4-§6; CAT-N1): its keeper is a process of - // its own on its own account (own secret broker-keeper, ADR 0436 §1), creating only bus state; it hears the indexer's - // manifest-accepted, keeps what it heard in its own old state `asked` (ADR 0595 §10) and the persons' requests in + // its own on its own account (own secret broker-keeper, ADR 0436 §1), creating only bus state; it reads the catalog half + // of every watched branch and the seat list (design 47 G §433, §532, §614), keeps what it planned in its own old state `asked` (ADR 0595 §10) and the persons' requests in // `requests`; its nine events are its own (ADR 0620 §2), on its own stream that keeps its history (ADR 0592 §6, the manifest // validator's predefined form) at 256 MiB; the seat's four // reading verbs are served, its four acting verbs are the keeper's own work and no other account's; its health is @@ -194,17 +194,22 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { if len(nm.Claims) != 1 || nm.Claims[0].Name != "mesh-broker" || !slices.Equal(nm.Claims[0].Serves, append(slices.Clone(ReadingVerbs), SnapshotVerbs...)) { t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v and its snapshot verbs %v alone", nm.Claims, ReadingVerbs, SnapshotVerbs) } - if !slices.Equal(nm.Consumes, []string{"mesh-catalog.manifest-accepted", "mesh-conditions.findings-asked"}) { - t.Errorf("consumes %v: the seat mesh-catalog's manifest-accepted (ADR 0595 §4) and the finding writer's findings-asked (design 47 G A-17)", nm.Consumes) + // The catalog half replaces manifest-accepted and indexer_bus_state (design 47 G §433, §614), and the seat list is read + // through the seat mesh-catalog (§537). + catalogHalf := []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", + "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"} + if !slices.Equal(nm.Consumes, append(slices.Clone(catalogHalf), "mesh-conditions.findings-asked")) { + t.Errorf("consumes %v: the catalog half's and the seat list's events %v, and the finding writer's findings-asked (design 47 G A-17)", nm.Consumes, catalogHalf) } - if AcceptedSubject != "mesh.seat.mesh-catalog.event.manifest-accepted" { - t.Errorf("the keeper's durable consumer filters on %s, not on the seat mesh-catalog's event", AcceptedSubject) - } - for _, want := range []string{"mesh-delivery-indexer.indexer_bus_state", "seat:mesh-hand-acts.hand-act"} { + for _, want := range []string{"seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", + "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act"} { if !slices.Contains(nm.Invokes, want) { t.Errorf("invokes %v lack %s", nm.Invokes, want) } } + if strings.Contains(string(raw), "indexer_bus_state") || strings.Contains(string(raw), "manifest-accepted") { + t.Error("the manifest still names indexer_bus_state or manifest-accepted (design 47 G §433)") + } if !slices.Contains(nm.Uses, "operator-channel") { t.Errorf("uses %v: the operator channel, for the warrant of recreate and drop", nm.Uses) } diff --git a/modules/nats/cmd/nats-tools/requests.go b/modules/nats/cmd/nats-tools/requests.go index 78a85822b..2602ab542 100644 --- a/modules/nats/cmd/nats-tools/requests.go +++ b/modules/nats/cmd/nats-tools/requests.go @@ -209,7 +209,7 @@ func requestTools(rs requestStore) []stdio.Tool { }} } firstView := stdio.Tool{Name: "nats_first_view_again", - Description: "Ask the bus module's keeper to ask the indexer's indexer_bus_state, its first view of every accepted manifest's bus state, at once (design 25 § 12.2): by hand, beside its own backoff, recorded as a hand act; answers what the first view answered and its revision.", + Description: "Ask the bus module's keeper to ask the catalog half's first views of every watched branch and the seat list again at once, and plan from them (design 47 G §614): by hand, recorded as a hand act; answers whether they answered and the newest revision planned from.", Input: map[string]any{"why": str("why, in plain words: kept with the act"), "request": str("a request made before: its end is read")}, Run: func(a map[string]any) (any, error) { if id := arg(a, "request"); id != "" { diff --git a/modules/nats/go.mod b/modules/nats/go.mod index 2bc42e0f8..9ef355f80 100644 --- a/modules/nats/go.mod +++ b/modules/nats/go.mod @@ -8,7 +8,9 @@ require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/klauspost/compress v1.20.1 diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index a6b69975b..3b1af6d94 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -17,6 +17,7 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/membership" + "git.novox.be/novox/mesh-sdk/go/seatdef" ) // Module is the bus module's own name: the publisher of its events and its conditions. @@ -81,8 +82,8 @@ type EventFill struct { Cap int64 `json:"cap"` } -// Accepted is a manifest-accepted the keeper heard, or a module of its first view: the module, its manifest's digest, -// the indexer's revision of the verdict (0 when the event does not say it yet), and its bus-state entries whole. +// Accepted is one module's entry as the catalog half's watched branches give it (design 47 G §344, §433, §532): the +// module, its manifest's digest, the catalog revision of the branch it was read from, and its bus-state entries whole. type Accepted struct { Module string `json:"module"` Digest string `json:"digest"` @@ -90,10 +91,33 @@ type Accepted struct { BusState []catalogue.BusStateEntry `json:"bus-state"` } -// FirstView is the indexer's answer of indexer_bus_state: every accepted manifest's bus-state entries, by module. -type FirstView struct { - Revision uint64 `json:"revision"` - Modules []Accepted `json:"modules"` +// CatalogModule is one module of the catalog half for the bus module (design 47 G §344, §532, §533, §547): what its +// watched branches' newest revisions ask of the bus, the seats they claim, whether one declares an event of its own, +// whether it is a foundation part (it declares an old `objects` section `sent-objects`), and the caps of the old +// sections it declares, in bytes. +type CatalogModule struct { + Accepted + Claims []string `json:"claims,omitempty"` + EmitsOwn bool `json:"emits-own,omitempty"` + Foundation bool `json:"foundation,omitempty"` + OldBytes int64 `json:"old-bytes,omitempty"` +} + +// Drift is a local name a later watched branch asks with other settings than the branch it is planned from (design 47 +// G §532): said as the bus module's drifted, naming the branch, and planned from nothing of that branch. +type Drift struct { + Module string `json:"module"` + Local string `json:"local-name"` + Branch string `json:"branch"` + // From is the branch whose settings are planned. + From string `json:"from"` +} + +// CatalogView is the catalog half as the bus module plans from it: every module in a watched branch's newest revision, +// and the drifts between branches. +type CatalogView struct { + Modules []CatalogModule + Drifts []Drift } // Planned is the value of a module's key in StatePlans: the entries as planned, their digest and the revision of the verdict. @@ -123,7 +147,9 @@ type Asked struct { Revision uint64 `json:"revision"` Entries []catalogue.BusStateEntry `json:"entries"` Made map[string]Made `json:"made,omitempty"` - At time.Time `json:"at"` + // Drifts are the local names a later watched branch asks with other settings, as last said (design 47 G §532). + Drifts map[string]Drift `json:"drifts,omitempty"` + At time.Time `json:"at"` } // Made is one entry as the keeper made it: its plan and what stands of it now. @@ -151,9 +177,13 @@ type Status struct { Pass time.Time `json:"pass"` // Limits are the bus's storage limits as last read (design 47 G §547): what `list` shows against the sum of the caps. Limits Limits `json:"limits,omitempty"` - // FirstView is the indexer's first view: when asked, when answered and at which revision, or why not. + // OldBytes is the sum of the caps of every old section the catalog graph's entries declare, in bytes, as last planned + // (design 47 G §547, §591): counted in the sum of the bus's caps beside the entries made. + OldBytes int64 `json:"old-sections-bytes,omitempty"` + // FirstView is the catalog half's first views (design 47 G §614): when last planned from, when answered and at which + // revision, or what it waits for. FirstView FirstViewStatus `json:"first-view"` - // Heard is when the last manifest-accepted was heard, and Unanswered since when one has waited unhandled. + // Heard is when the catalog half last moved, and Unanswered since when an event has waited unhandled. Heard time.Time `json:"heard,omitempty"` Unanswered time.Time `json:"unanswered,omitempty"` // Unsaid are the events the bus did not take, kept with their id and the subjects still to say them on, and @@ -191,10 +221,11 @@ type FirstViewStatus struct { Next time.Time `json:"next,omitempty"` } -// The first view's owner and verb (design 25 § 12.2). +// The first view's seat and verb: the catalog half of each watched branch, read through the seat mesh-catalog's +// `revision` (design 47 G §433, §614). const ( - FirstViewOwner = "mesh-delivery-indexer" - FirstViewVerb = "indexer_bus_state" + FirstViewOwner = "mesh-catalog" + FirstViewVerb = "revision" ) // Bus is the bus as the keeper's own account reaches it. @@ -257,7 +288,7 @@ func (k *Keeper) capRoom(ctx context.Context, held *Asked, m *Made) error { } all := k.Held() all[held.Module] = *held - sum := CapSum(all, held.Module, m.Plan.Local) + sum := CapSum(all, held.Module, m.Plan.Local) + k.Status().OldBytes for _, o := range m.Plan.Objects { sum += o.Settings.MaxBytes } @@ -297,8 +328,12 @@ type Keeper struct { // Emit says one of the bus module's events (Saying); Say raises or clears a condition through the condition keeper. Emit func(s *Saying) error Say func(oc.Condition) error - // View asks the indexer's indexer_bus_state. - View func() (FirstView, error) + // Catalog reads the catalog half of every watched branch as it stands (design 47 G §433, §532): an error while a + // branch has no first view yet, when the keeper plans nothing. Seats are the seat list's definitions, each resolved + // with the parts it includes (§537). Again asks every first view of the catalog half again, by hand. + Catalog func() (CatalogView, error) + Seats func() map[string]seatdef.Definition + Again func() error // Act records a hand act through the seat mesh-hand-acts; Warrant reads the router's record of an ask. Act func(what, why, cause string) error Warrant func(askID string) (*oc.Ask, *oc.Warrant, error) @@ -417,81 +452,148 @@ func (k *Keeper) statusWritten(s Status, err error) { } } -// AskFirstView asks the indexer's first view once and applies it (design 25 § 12.2): every module it answers is -// applied as an accepted verdict, a module the keeper holds and the view leaves out is kept (a set-aside manifest is -// missing from it, never removed), and then check runs over what is held. It answers the error and when to ask again, -// on the backoff from 10 s doubling to 5 min; the keeper's health is unhealthy meanwhile, naming the verb. -func (k *Keeper) AskFirstView(wait time.Duration) (time.Duration, error) { +// PlanCatalog plans from the catalog half as it stands (design 47 G §344, §433, §532, §537, §538): each module in a +// watched branch's newest revision is applied (Heard), a module of the new form that sends gets its sent store, a drift +// between branches is said once, and a module in no watched branch's newest revision has its plans key deleted, its +// entries then orphans. Before every watched branch has its first view it plans nothing, and says what it waits for in +// its status; the first plan checks what is held. +func (k *Keeper) PlanCatalog() error { at := k.now() - k.setStatus(func(s *Status) { s.FirstView.Asked = at }) - view, err := k.View() + view, err := k.Catalog() if err != nil { - next := wait * 2 - if wait <= 0 { - next = 10 * time.Second - } - if next > 5*time.Minute { - next = 5 * time.Minute - } - k.setStatus(func(s *Status) { - s.FirstView.Error = fmt.Sprintf("%s.%s did not answer: %v", FirstViewOwner, FirstViewVerb, err) - s.FirstView.Next = at.Add(next) - }) - return next, err + k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Error = at, err.Error() }) + return err } k.mu.Lock() + first := !k.viewed k.viewed = true k.mu.Unlock() - k.setStatus(func(s *Status) { - s.FirstView.Answered, s.FirstView.Revision, s.FirstView.Error, s.FirstView.Next = k.now(), view.Revision, "", time.Time{} - }) - for _, a := range view.Modules { - k.Heard(a) + var revision uint64 + var old int64 + for _, m := range view.Modules { + revision = max(revision, m.Revision) + old += m.OldBytes } - k.CheckAll() - return 0, nil + k.setStatus(func(s *Status) { + s.FirstView.Asked, s.FirstView.Revision, s.FirstView.Error, s.FirstView.Next = at, revision, "", time.Time{} + if s.FirstView.Answered.IsZero() { + s.FirstView.Answered = at + } + s.OldBytes = old + }) + var defs map[string]seatdef.Definition + if k.Seats != nil { + defs = k.Seats() + } + listed := map[string]bool{} + for _, m := range view.Modules { + listed[m.Module] = true + k.Heard(m.Accepted) + if !m.Foundation && Sends(m.Claims, m.EmitsOwn, defs) { + k.Sends(m.Module) + } + } + k.drifts(view.Drifts) + gone := map[string]bool{} + for module := range k.Held() { + gone[module] = !listed[module] + } + if keys, err := k.Store.Keys(StatePlans); err == nil { + for _, module := range keys { + gone[module] = !listed[module] + } + } + for _, module := range sortedKeys(gone) { + if !gone[module] { + continue + } + if _, ok := k.planned(module); ok { + k.Heard(Accepted{Module: module, Revision: revision}) + } + if err := k.Store.Delete(StatePlans, module); err != nil { + k.logf("the plan of %s cannot be removed from its own %s: %v", module, StatePlans, err) + } + } + if first { + k.CheckAll() + } + return nil } -// Heard applies one accepted verdict (manifest-accepted, or a module of the first view): applied only when newer than -// what the keeper holds for that module by the indexer's revision alone, so a replay never takes an entry back (a -// verdict that says no revision is applied when its digest differs). Each entry asked is made where the bus lacks it; -// each entry made that is no longer asked is an orphan. An empty verdict ("this module asks nothing now") ends the -// module's key once nothing it made is held. +// drifts says each drift once (design 47 G §532): kept in the module's key, said again only when the branch or the +// settings planned change. +func (k *Keeper) drifts(ds []Drift) { + by := map[string]map[string]Drift{} + for _, d := range ds { + if by[d.Module] == nil { + by[d.Module] = map[string]Drift{} + } + by[d.Module][d.Local] = d + } + for module, a := range k.Held() { + if by[module] == nil && len(a.Drifts) > 0 { + by[module] = map[string]Drift{} + } + } + for _, module := range sortedKeys(by) { + held, _, found := k.asked(module) + if !found { + held = Asked{V: 1, Module: module, At: k.now()} + } + now := by[module] + changed := len(now) != len(held.Drifts) + for local, d := range now { + if held.Drifts[local] != d { + changed = true + var p Plan + for _, e := range held.Entries { + if e.Name == local { + p, _ = PlanOf(module, e) + } + } + p.Module, p.Local = module, local + k.emit(EventDrifted, p, held.Revision, fmt.Sprintf("the branch %s asks %s of %s with other settings than the branch %s, whose are planned: nothing of %s is planned", d.Branch, local, module, d.From, d.Branch), nil, nil) + } + } + if !changed { + continue + } + held.Drifts = nil + if len(now) > 0 { + held.Drifts = now + } + if found || len(now) > 0 { + k.write(held) + } + } +} + +// Heard applies one module's entry of the catalog half: it plans again only when its bus-state entries differ from those +// it last planned (design 47 G §538), never on the manifest's digest, which changes with any field. Each entry asked is +// made where the bus lacks it; each entry made that is no longer asked is an orphan. An entry that asks nothing ("this +// module asks nothing now") ends the module's key once nothing it made is held. func (k *Keeper) Heard(a Accepted) { k.setStatus(func(s *Status) { s.Heard = k.now() }) if !k.Viewed() { - k.logf("manifest-accepted of %s heard before the first view: kept on the consumer until it is answered", a.Module) + k.logf("the entry of %s is heard before the catalog half's first views: nothing is planned until they are in", a.Module) return } held, _, found := k.asked(a.Module) - if found && a.Revision != 0 && a.Revision <= held.Revision { - k.logf("manifest-accepted of %s at revision %d is not newer than the %d held: not applied", a.Module, a.Revision, held.Revision) - return - } - if found && a.Revision == 0 && a.Digest != "" && a.Digest == held.Digest { - return - } if !found { held = Asked{V: 1, Module: a.Module} } if held.Made == nil { held.Made = map[string]Made{} } - // It plans again only when its bus-state entries differ from those it last planned, never on the manifest's digest - // (design 47 G §538). digest := EntriesDigest(a.BusState) if p, ok := k.planned(a.Module); ok && found && p.Digest == digest { - held.Digest, held.At = a.Digest, k.now() - if a.Revision > held.Revision { - held.Revision = a.Revision + if held.Digest != a.Digest || held.Revision != a.Revision { + held.Digest, held.Revision, held.At = a.Digest, a.Revision, k.now() + k.write(held) } - k.write(held) return } - held.Digest, held.Entries, held.At = a.Digest, a.BusState, k.now() - if a.Revision > held.Revision { - held.Revision = a.Revision - } + held.Digest, held.Entries, held.Revision, held.At = a.Digest, a.BusState, a.Revision, k.now() defer k.writePlan(a.Module, digest, held.Revision, a.BusState) asked := map[string]bool{} for _, e := range a.BusState { @@ -512,10 +614,10 @@ func (k *Keeper) Heard(a Accepted) { } m.Orphaned = true held.Made[local] = m - k.emit(EventOrphaned, m.Plan, held.Revision, "no accepted manifest of "+a.Module+" asks for it now", nil, nil) + k.emit(EventOrphaned, m.Plan, held.Revision, "no watched branch's newest revision of "+a.Module+" asks for it now", nil, nil) k.raise(conditionOrphaned, a.Module, local, occ.SeverityWarning, "Kept data no manifest asks for", "the operator's drop of it, or a manifest asking for it again", - fmt.Sprintf("%s of %s is on the bus and no accepted manifest asks for it: it goes only by the bus module's drop, on your warrant (nats_drop), or a manifest asking for it again keeps it", BusName(a.Module, local), a.Module)) + fmt.Sprintf("%s of %s is on the bus and no watched branch's newest revision asks for it: it goes only by the bus module's drop, on your warrant (nats_drop), or a manifest asking for it again keeps it", BusName(a.Module, local), a.Module)) } k.write(held) } @@ -895,7 +997,7 @@ func (k *Keeper) asked(module string) (Asked, uint64, bool) { // write keeps a module's key, or removes it when its manifest asks for nothing and nothing it made is held: a key goes // only so, never because a module is missing from a first view. func (k *Keeper) write(a Asked) { - if len(a.Entries) == 0 && len(a.Made) == 0 { + if len(a.Entries) == 0 && len(a.Made) == 0 && len(a.Drifts) == 0 { if err := k.Store.Delete(StateAsked, a.Module); err != nil { k.logf("the key of %s cannot be removed from its own %s: %v", a.Module, StateAsked, err) } diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index d1eab8b7a..90d499d5a 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -302,12 +302,12 @@ func (f *findingLog) Clear(_ context.Context, scope, id, kind, _ string) error { return nil } -func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) { +func newKeeper(view CatalogView) (*Keeper, *fakeBus, *memStore, *said) { bus, store, out := newFakeBus(), newMemStore(), &said{} at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) k := &Keeper{Bus: bus, Store: store, Finding: &findingLog{}, Emit: out.emit, Say: out.say, Files: &memFiles{size: map[string]int64{}}, - View: func() (FirstView, error) { return view, nil }, - Now: func() time.Time { return at }} + Catalog: func() (CatalogView, error) { return view, nil }, + Now: func() time.Time { return at }} return k, bus, store, out } @@ -445,7 +445,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { if len(bus.created) != 0 { t.Fatalf("created %v before the first view", bus.created) } - if _, err := k.AskFirstView(0); err != nil { + if err := k.PlanCatalog(); err != nil { t.Fatal(err) } k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}}) @@ -492,11 +492,66 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { t.Errorf("the sent store was orphaned or removed: %+v %v", held.Made[SentLocal], bus.deleted) } }}, - {"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { - k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}) - k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil}) - if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned { - t.Errorf("held %+v: the replay at 4 took back what 9 asked", a) + // §344, §533, §537: a module of the new form claiming a seat whose resolved definition has a verb or an event, or + // declaring an event of its own, gets a sent store; a seat including only bodies, and a foundation part, none; a + // definition gaining a verb gives its holder one at the next plan. + {"the catalog half and the seat list plan the sent stores", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + defs := map[string]seatdef.Definition{ + "verbed": {Seat: "verbed", Verbs: []seatdef.Verb{{Name: "go"}}}, + "resolved": {Seat: "resolved", Includes: []string{"part"}, Verbs: []seatdef.Verb{{Name: "from-the-part"}}}, + "bodies": {Seat: "bodies", Includes: []string{"shared-bodies"}}, + } + k.Seats = func() map[string]seatdef.Definition { return defs } + k.Catalog = func() (CatalogView, error) { + return CatalogView{Modules: []CatalogModule{ + {Accepted: Accepted{Module: "a"}, Claims: []string{"verbed"}}, + {Accepted: Accepted{Module: "b"}, Claims: []string{"resolved"}}, + {Accepted: Accepted{Module: "c"}, Claims: []string{"bodies"}}, + {Accepted: Accepted{Module: "d"}, Claims: []string{"verbed"}, Foundation: true}, + {Accepted: Accepted{Module: "e"}, EmitsOwn: true}, + }}, nil + } + _ = k.PlanCatalog() + if want := "[OBJ_a_sent-objects OBJ_b_sent-objects OBJ_e_sent-objects]"; fmt.Sprint(bus.created) != want { + t.Fatalf("created %v, want %s", bus.created, want) + } + defs["bodies"] = seatdef.Definition{Seat: "bodies", Includes: []string{"shared-bodies"}, Verbs: []seatdef.Verb{{Name: "new"}}} + _ = k.PlanCatalog() + if want := "[OBJ_a_sent-objects OBJ_b_sent-objects OBJ_e_sent-objects OBJ_c_sent-objects]"; fmt.Sprint(bus.created) != want { + t.Errorf("created %v, want %s: the holder of a definition gaining a verb gets its store", bus.created, want) + } + }}, + // §532: a later branch asking a local name with other settings is said drifted once, naming the branch; §547: the + // old sections' caps count in the sum. + {"a drift between branches is said once, naming the branch", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + view := CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}}, OldBytes: 8 * MiB}}, + Drifts: []Drift{{Module: "m", Local: "drafts", Branch: "feature", From: "main"}}} + k.Catalog = func() (CatalogView, error) { return view, nil } + _ = k.PlanCatalog() + _ = k.PlanCatalog() + if fmt.Sprint(out.events) != "[created m drafts drifted m drafts]" || !strings.Contains(out.bodies[1].Why, "the branch feature asks drafts") { + t.Errorf("said %v (%+v): created from main, drifted once naming feature", out.events, out.bodies) + } + if k.Status().OldBytes != 8*MiB { + t.Errorf("old sections %d: 8 MiB counted", k.Status().OldBytes) + } + }}, + // §538: a module in no watched branch's newest revision has its plans key deleted, its entries then orphans. + {"a module no watched branch lists loses its plan, its entries orphans", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + k.Catalog = func() (CatalogView, error) { + return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil + } + _ = k.PlanCatalog() + if _, ok := k.planned("m"); !ok { + t.Fatal("m's plan is not kept") + } + k.Catalog = func() (CatalogView, error) { return CatalogView{}, nil } + _ = k.PlanCatalog() + if _, ok := k.planned("m"); ok { + t.Error("m's plan is kept while no watched branch lists it") + } + if a := k.Held()["m"]; !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 { + t.Errorf("held %+v, deleted %v: drafts an orphan, kept on the bus", a, bus.deleted) } }}, {"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { @@ -641,12 +696,14 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { }}, {"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) - k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil } - if _, err := k.AskFirstView(0); err != nil { - t.Fatal(err) + k.Catalog = func() (CatalogView, error) { + return CatalogView{}, errors.New("the catalog half has no first view of main yet") + } + if err := k.PlanCatalog(); err == nil { + t.Fatal("planned while a watched branch has no first view") } if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned { - t.Errorf("held %+v: kept as asked, never orphaned by a first view", a) + t.Errorf("held %+v: kept as asked, never orphaned while a first view waits", a) } bus.refuse["KV_n_x"] = errors.New("no") k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}}) @@ -687,19 +744,21 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") { t.Errorf("health %+v: names the verdicts waiting", h) } - k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") } + k.Catalog = func() (CatalogView, error) { + return CatalogView{}, errors.New("the catalog half has no first view of main yet") + } k.mu.Lock() k.status.FirstView.Answered = time.Time{} k.mu.Unlock() - if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second { - t.Fatalf("asked again in %s (%v): on the backoff", next, err) + if err := k.PlanCatalog(); err == nil { + t.Fatal("planned with no first view") } // The first view unanswered is unhealthy only past the bound since the start (review round 1: health.go:41). if h := HealthOf(k.Status(), nil, 0, at); !h.Healthy { t.Errorf("health %+v: healthy within the bound while the first view is asked", h) } k.FillPass() - if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") { + if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "mesh-catalog.revision, has not answered: the catalog half has no first view of main yet") { t.Errorf("health %+v: unhealthy past the bound, naming the verb", h) } // What waits on its consumer unread is never skipped silently (review round 1: health.go:47). @@ -1157,24 +1216,26 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { store := k.Store.(*memStore) var acts []string k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil } - k.View = func() (FirstView, error) { - return FirstView{Revision: 8, Modules: []Accepted{{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}, nil + k.Catalog = func() (CatalogView, error) { + return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil } + again := 0 + k.Again = func() error { again++; return nil } raw, _ := json.Marshal(Request{ID: "r9", Verb: VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"}) _, _ = store.Put(StateRequests, "r9", raw) k.ActOnRequests() raw, _, _, _ = store.Get(StateRequests, "r9") var got Request _ = json.Unmarshal(raw, &got) - if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts OBJ_m_drafts-objects]" || - len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked mesh-delivery-indexer.indexer_bus_state again by hand") { + if got.Outcome != "the first views answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts OBJ_m_drafts-objects]" || again != 1 || + len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked the catalog half's first views (mesh-catalog.revision) again by hand") { t.Errorf("ended %+v, created %v, recorded %v", got, bus.created, acts) } }}, } { t.Run(row.name, func(t *testing.T) { - k, bus, _, out := newKeeper(FirstView{Revision: 1}) - if _, err := k.AskFirstView(0); err != nil { + k, bus, _, out := newKeeper(CatalogView{}) + if err := k.PlanCatalog(); err != nil { t.Fatal(err) } row.run(t, k, bus, out) @@ -1197,7 +1258,7 @@ func TestSends(t *testing.T) { }{ {"a seat with a verb", []string{"verbed"}, false, true}, {"a seat with an event", []string{"evented"}, false, true}, - {"a seat that includes parts", []string{"included"}, false, true}, + {"a seat that only includes parts (§537: resolved by the seat list, including alone counts for nothing)", []string{"included"}, false, false}, {"a seat with neither", []string{"bare"}, false, false}, {"a seat the list does not hold", []string{"unknown"}, false, false}, {"an event of its own", nil, true, true}, diff --git a/modules/nats/internal/keeper/plan.go b/modules/nats/internal/keeper/plan.go index baa6a3e88..5880e14bc 100644 --- a/modules/nats/internal/keeper/plan.go +++ b/modules/nats/internal/keeper/plan.go @@ -1,8 +1,7 @@ // Package keeper is the bus module's keeper (novox/hq design 25 § 12.2-12.3, ADR 0595 §4-§6): it creates the bus state // every accepted manifest asks for in `requires`, compares what the bus holds with what was asked, finds orphans and // reads the fill of what refuses when full, and says each on the bus module's own events. It reads nothing of another -// part's state: what to create comes on the indexer's manifest-accepted, and its first view from the indexer's tool -// indexer_bus_state. Everything it touches it reaches through the interfaces in keeper.go; this file is pure. +// part's state: what to create comes from the catalog half of each watched branch (design 47 G §433, §532). Everything it touches it reaches through the interfaces in keeper.go; this file is pure. package keeper import ( @@ -55,14 +54,14 @@ const StateObjectsCapMiB = 16 // Sends says whether a module sends (design 47 G §344): it claims a seat whose definition has a verb or an event, or it // declares an event of its own (emitsOwn). claims are the seats its catalog graph entry claims, defs the seat list's -// definitions by seat; a claimed seat the list does not hold counts for nothing. A definition that includes parts takes -// their verbs and events whole (seatdef.Resolve), so one counts as having some. +// definitions by seat, each as the seat list answers that one seat, resolved with the parts it includes; a claimed seat +// the list does not hold counts for nothing, and including parts alone counts for nothing (design 47 G §537). func Sends(claims []string, emitsOwn bool, defs map[string]seatdef.Definition) bool { if emitsOwn { return true } for _, c := range claims { - if d, ok := defs[c]; ok && (len(d.Verbs) > 0 || len(d.Events) > 0 || len(d.Includes) > 0) { + if d, ok := defs[c]; ok && (len(d.Verbs) > 0 || len(d.Events) > 0) { return true } } diff --git a/modules/nats/internal/keeper/requests.go b/modules/nats/internal/keeper/requests.go index b9b87803e..9367b5546 100644 --- a/modules/nats/internal/keeper/requests.go +++ b/modules/nats/internal/keeper/requests.go @@ -24,7 +24,7 @@ const ( VerbApply = "apply" VerbRecreate = "recreate" VerbDrop = "drop" - // VerbFirstViewAgain asks the indexer's first view at once, by hand (design 25 § 12.2), beside the backoff. + // VerbFirstViewAgain asks the catalog half's first views again at once, by hand (design 47 G §614). VerbFirstViewAgain = "first-view-again" ) @@ -175,13 +175,19 @@ func (k *Keeper) ActOnRequests() { func (k *Keeper) act(r *Request) (string, error) { if r.Verb == VerbFirstViewAgain { - _, err := k.AskFirstView(0) + err := fmt.Errorf("the keeper follows no catalog half") + if k.Again != nil { + err = k.Again() + } + if err == nil { + err = k.PlanCatalog() + } s := k.Status().FirstView - k.record(r, fmt.Sprintf("asked %s.%s again by hand: answered %v at revision %d", s.Owner, s.Verb, err == nil, s.Revision)) + k.record(r, fmt.Sprintf("asked the catalog half's first views (%s.%s) again by hand: answered %v at revision %d", s.Owner, s.Verb, err == nil, s.Revision)) if err != nil { return "asked, not answered", err } - return fmt.Sprintf("the first view answered at revision %d", s.Revision), nil + return fmt.Sprintf("the first views answered at revision %d", s.Revision), nil } if !k.Viewed() { return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant) diff --git a/modules/nats/module.json b/modules/nats/module.json index 1e23a2e6d..ab20bcd8a 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -62,10 +62,17 @@ ], "invokes": [ "seat:mesh-hand-acts.hand-act", - "mesh-delivery-indexer.indexer_bus_state" + "seat:mesh-catalog.revision", + "seat:mesh-catalog.registrations", + "seat:mesh-delivery.source", + "seat:mesh-catalog.seat-definitions" ], "consumes": [ - "mesh-catalog.manifest-accepted", + "mesh-catalog.catalog-revision", + "mesh-catalog.registration-changed", + "mesh-delivery.source-moved", + "mesh-catalog.seat-definition-changed", + "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked" ], "uses": [ @@ -325,7 +332,10 @@ "asks": true, "keeps": true, "invokes": [ - "mesh-delivery-indexer.indexer_bus_state", + "seat:mesh-catalog.revision", + "seat:mesh-catalog.registrations", + "seat:mesh-delivery.source", + "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act" ] },