NATS3 (rest): the bus module plans from the catalog half of every watched branch, not from manifest-accepted

The keeper follows meshcatalogclient's branch follower and the seat list on its views reader, and plans again at each
move of a branch's view, the registrations or the seat list. Each local name is planned from the first of the module's
watched branches in its registration's order; another branch asking it with other settings is said drifted, naming
the branch. A module of the new form whose resolved seat definition has a verb or an event gets its sent store; one
including only parts, and a foundation part, none. A module in no watched branch's newest revision loses its plans
key, its entries orphans. The sum of the caps counts the old sections the entries declare. The manifest consumes the
catalog half's and the seat list's events and invokes their verbs; manifest-accepted and indexer_bus_state go.

Builds: design 47 G §344, §433, §532, §533, §537, §538, §547, §591

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
2026-11-12 05:45:49 +01:00
parent 9fe3f41ac0
commit bf60d6a8bb
16 changed files with 746 additions and 313 deletions
+12 -8
View File
@@ -48,13 +48,17 @@ func HeldInState() (map[string]keeper.Asked, error) {
return out, nil
}
// capsOf is the sum of the bus's caps against its limits (design 47 G §547, §661), in MiB; a limit the keeper has not read,
// or that is none, is absent. The sum is over every module, whatever list is narrowed to.
func capsOf(all map[string]keeper.Asked, limits func() (keeper.Limits, error)) *meshbroker.ListAnswerCaps {
c := &meshbroker.ListAnswerCaps{SumMiB: keeper.CapSum(all, "", "") / keeper.MiB}
if l, err := limits(); err == nil {
c.AccountLimitMiB, c.ServerLimitMiB = l.AccountBytes/keeper.MiB, l.ServerBytes/keeper.MiB
// capsOf is the sum of the bus's caps against its limits (design 47 G §547, §591, §661), in MiB: every entry made and
// every old section the catalog graph's entries declare, as the keeper's status last says them; a limit the keeper has
// not read, or that is none, is absent. The sum is over every module, whatever list is narrowed to.
func capsOf(all map[string]keeper.Asked, status func() (keeper.Status, error)) *meshbroker.ListAnswerCaps {
sum := keeper.CapSum(all, "", "")
c := &meshbroker.ListAnswerCaps{}
if s, err := status(); err == nil {
sum += s.OldBytes
c.AccountLimitMiB, c.ServerLimitMiB = s.Limits.AccountBytes/keeper.MiB, s.Limits.ServerBytes/keeper.MiB
}
c.SumMiB = sum / keeper.MiB
return c
}
@@ -167,7 +171,7 @@ func anyOf(m map[string]json.RawMessage) map[string]any {
}
// brokerTools are the four reading verbs and nats_history.
func brokerTools(read heldEntries, limits func() (keeper.Limits, error), jsz func() ([]byte, error), history func(module, local string, limit int) ([]keeper.Event, error)) []stdio.Tool {
func brokerTools(read heldEntries, status func() (keeper.Status, error), jsz func() ([]byte, error), history func(module, local string, limit int) ([]keeper.Event, error)) []stdio.Tool {
both := func() (map[string]keeper.Asked, map[string]busStream, error) {
entries, err := read()
if err != nil {
@@ -208,7 +212,7 @@ func brokerTools(read heldEntries, limits func() (keeper.Limits, error), jsz fun
if err != nil {
return nil, err
}
return meshbroker.ListAnswer{Caps: capsOf(all, limits), Entries: each(entries, held, func(keeper.Made) bool { return true })}, nil
return meshbroker.ListAnswer{Caps: capsOf(all, status), Entries: each(entries, held, func(keeper.Made) bool { return true })}, nil
}},
{Name: "mesh-broker.show",
Description: "One entry of a module's kept data on the bus: its bus name, template and options as asked, the settings the bus holds of each of its objects, and its fill against its cap. Whether it is refused, waiting, drifted, full or orphaned is what nats_history says of it.",
+5 -4
View File
@@ -42,8 +42,8 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) {
{"name":"m_log","config":{"name":"m_log","retention":"limits","discard":"new","max_bytes":33554432,"max_msg_size":262144},"state":{"messages":9,"bytes":9000}}]}]}`
tools := map[string]stdio.Tool{}
for _, tl := range brokerTools(func() (map[string]keeper.Asked, error) { return held, nil },
func() (keeper.Limits, error) {
return keeper.Limits{AccountBytes: 2048 << 20, ServerBytes: 4096 << 20}, nil
func() (keeper.Status, error) {
return keeper.Status{Limits: keeper.Limits{AccountBytes: 2048 << 20, ServerBytes: 4096 << 20}, OldBytes: 256 << 20}, nil
},
func() ([]byte, error) { return []byte(jsz), nil },
func(module, local string, limit int) ([]keeper.Event, error) {
@@ -62,8 +62,9 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) {
`"KV_m_drafts":{"max-bytes":67108864`, `"fill":{"bytes":1048576,"cap":83886080,"count":3}`, `"options":{"max-mib":16}`,
// The binding of an entry the bus holds is the one its created carries (design 25 bindings-first-view §2).
`"binding":{"bucket":"m_drafts","local":"drafts","objects":"m_drafts-objects","storage":"key-value","template":"state"}`}, ""},
// The sum of the caps against both limits (design 47 G §547): drafts 64+16, log 16, old 1024 (orphaned, still held); later is not made.
{"mesh-broker.list", nil, []string{`"caps":{"account-limit-mib":2048,"server-limit-mib":4096,"sum-mib":1120`}, ""},
// The sum of the caps against both limits (design 47 G §547, §591): drafts 64+16, log 16, old 1024 (orphaned, still
// held), and the old sections the catalog graph's entries declare, 256; later is not made.
{"mesh-broker.list", nil, []string{`"caps":{"account-limit-mib":2048,"server-limit-mib":4096,"sum-mib":1376`}, ""},
{"mesh-broker.show", map[string]any{"module": "m", "local-name": "log"}, []string{`"bus-name":"m_log"`, `"template":"log"`,
`"m_log":{"max-bytes":33554432`}, ""},
{"mesh-broker.show", map[string]any{"module": "m", "local-name": "later"}, []string{`"local-name":"later"`}, ""},
+98 -6
View File
@@ -1,10 +1,13 @@
package main
import (
"cmp"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"maps"
"slices"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -12,15 +15,18 @@ import (
"nats-tools/internal/keeper"
)
// The keeper reads a module's asks from the module's entry in the catalog half of its revision view, not from the
// indexer's event (design 47 G §344, §433).
// The keeper reads a module's asks from the module's entry in the catalog half of each watched branch, not from the
// indexer's event (design 47 G §344, §433, §532).
// Asks is what one catalog graph entry says of a module for the keeper: what its manifest asks of the bus as an accepted
// verdict, the seats it claims, and whether it declares an event of its own (the facts keeper.Sends reads).
// verdict, the seats it claims, whether it declares an event of its own (the facts keeper.Sends reads), whether it is a
// foundation part (an old `objects` section `sent-objects`, §533), and the caps of its old sections in bytes (§547, §591).
type Asks struct {
Accepted keeper.Accepted
Claims []string
EmitsOwn bool
Accepted keeper.Accepted
Claims []string
EmitsOwn bool
Foundation bool
OldBytes int64
}
// AsksOf reads one entry. An entry whose manifest is not accepted asks nothing of the bus now, as an accepted verdict of
@@ -48,9 +54,95 @@ func AsksOf(e catalog.CatalogEntry) (Asks, error) {
for _, c := range m.Claims {
out.Claims = append(out.Claims, c.Name)
}
// The old sections' caps (design 47 G §547, §591): an objects section its max-mib, a log or a stream its max-mib or
// the catalogue's default.
for _, o := range m.Objects {
out.Foundation = out.Foundation || o.Name == keeper.SentLocal
out.OldBytes += int64(o.MaxMiB) * keeper.MiB
}
for _, l := range m.Logs {
out.OldBytes += int64(cmp.Or(l.MaxMiB, catalogue.LogDefaultMiB)) * keeper.MiB
}
for _, st := range m.Streams {
out.OldBytes += int64(cmp.Or(st.MaxMiB, catalogue.StreamDefaultMiB)) * keeper.MiB
}
return out, nil
}
// Branch is one watched branch's newest revision of the catalog half: its name, its revision and its entries by module.
type Branch struct {
Name string
Revision int64
Entries map[string]catalog.CatalogEntry
}
// ViewOf is the catalog half as the keeper plans from it (design 47 G §532): every module some branch's newest revision
// lists, its kept data planned from each such branch, each local name once, from the first of the module's watched
// branches in its registration's order (order: the registration's branches; a branch it does not name comes after, in
// the order given) whose entry asks it; another branch asking that local name with other settings is a drift naming
// the branch, and plans nothing. The module sends when a branch's entry says so; it is a foundation part, and its old
// sections' caps are read, from the first branch.
func ViewOf(branches []Branch, order func(module string) []string) (keeper.CatalogView, error) {
modules := map[string]bool{}
for _, b := range branches {
for m := range b.Entries {
modules[m] = true
}
}
var view keeper.CatalogView
for _, module := range slices.Sorted(maps.Keys(modules)) {
var listing []Branch
var named []string
if order != nil {
named = order(module)
}
for _, name := range named {
for _, b := range branches {
if _, ok := b.Entries[module]; ok && b.Name == name {
listing = append(listing, b)
}
}
}
for _, b := range branches {
if _, ok := b.Entries[module]; ok && !slices.Contains(named, b.Name) {
listing = append(listing, b)
}
}
var cm keeper.CatalogModule
from := map[string]string{}
asked := map[string]string{}
for i, b := range listing {
a, err := AsksOf(b.Entries[module])
if err != nil {
return keeper.CatalogView{}, fmt.Errorf("the branch %s: %w", b.Name, err)
}
if i == 0 {
cm = keeper.CatalogModule{Accepted: keeper.Accepted{Module: module, Digest: b.Entries[module].ManifestDigest, Revision: uint64(max(b.Revision, 0))},
Foundation: a.Foundation, OldBytes: a.OldBytes}
}
cm.EmitsOwn = cm.EmitsOwn || a.EmitsOwn
for _, c := range a.Claims {
if !slices.Contains(cm.Claims, c) {
cm.Claims = append(cm.Claims, c)
}
}
for _, e := range a.Accepted.BusState {
settings := string(mustJSON(e))
if f, ok := from[e.Name]; ok {
if asked[e.Name] != settings {
view.Drifts = append(view.Drifts, keeper.Drift{Module: module, Local: e.Name, Branch: b.Name, From: f})
}
continue
}
from[e.Name], asked[e.Name] = b.Name, settings
cm.BusState = append(cm.BusState, e)
}
}
view.Modules = append(view.Modules, cm)
}
return view, nil
}
func mustJSON(v any) []byte {
b, _ := json.Marshal(v)
return b
+176 -103
View File
@@ -14,6 +14,8 @@ import (
"syscall"
"time"
"sync"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -22,11 +24,14 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/sourcebranches"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/bussnap"
@@ -34,33 +39,10 @@ import (
)
// The keeper (`nats-tools keep`): the bus module's process on its own account (own secret broker-keeper, ADR 0436 §1),
// started by the node-engine as the manifest's process keeper. It reads the fill every minute from its start, asks the
// indexer's first view once, then hears manifest-accepted and the operator's decided on its durable consumer, makes what
// each verdict asks for, and acts on the persons' requests its tools put in its own state requests (requests.go) as they
// are written or answered. It answers the seat mesh-broker's snapshot verbs itself (snapshots.go; design 47 G A-107).
// AcceptedOf reads a manifest-accepted body: the seat mesh-catalog's typed event, with the indexer's revision of the
// verdict when the event carries it (0 from an indexer that does not say it).
func AcceptedOf(body []byte) (keeper.Accepted, error) {
var a catalog.ManifestAcceptedEvent
if err := json.Unmarshal(body, &a); err != nil {
return keeper.Accepted{}, err
}
if a.Module == "" {
return keeper.Accepted{}, fmt.Errorf("a manifest-accepted names no module")
}
var state []catalogue.BusStateEntry // the seat types an entry as the manifest wrote it; the keeper reads the catalogue's form
if raw, err := json.Marshal(a.BusState); err != nil || json.Unmarshal(raw, &state) != nil {
return keeper.Accepted{}, fmt.Errorf("the bus-state of the manifest-accepted of %s is not readable", a.Module)
}
if len(a.BusState) == 0 {
state = nil
}
return keeper.Accepted{Module: a.Module, Digest: a.Digest, Revision: uint64(max(a.Revision, 0)), BusState: state}, nil
}
// AcceptedSubject is the seat mesh-catalog's manifest-accepted on the shared events stream.
var AcceptedSubject = catalog.SubjectManifestAccepted
// started by the node-engine as the manifest's process keeper. It reads the fill every minute from its start, follows the
// catalog half of every watched branch through meshcatalogclient's branch follower and the seat list (design 47 G §344,
// §433, §532, §614), plans again at each of their changes, hears the operator's decided on its durable consumer, and acts
// on the persons' requests its tools put in its own state requests (requests.go) as they are written or answered. It answers the seat mesh-broker's snapshot verbs itself (snapshots.go; design 47 G A-107).
// The router's records of asks, read under this module's own name (ADR 0259 §3), as the mail sender reads them.
const asksBucket = "messenger_asks"
@@ -190,36 +172,6 @@ func sayer(js jetstream.JetStream) func(oc.Condition) error {
}
}
// firstView asks the indexer's indexer_bus_state as the keeper's own account.
func firstView(c *process.Conn) func() (keeper.FirstView, error) {
return func() (keeper.FirstView, error) {
raw, err := c.Call(keeper.FirstViewOwner+"."+keeper.FirstViewVerb, map[string]any{})
if err != nil {
return keeper.FirstView{}, err
}
return FirstViewOf(raw)
}
}
// FirstViewOf reads indexer_bus_state's answer, as the tool runner hands it back ({result, error} or bare).
func FirstViewOf(raw []byte) (keeper.FirstView, error) {
var wrapped struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
if json.Unmarshal(raw, &wrapped) == nil && (wrapped.Result != nil || wrapped.Error != "") {
if wrapped.Error != "" {
return keeper.FirstView{}, fmt.Errorf("%s", wrapped.Error)
}
raw = wrapped.Result
}
var v keeper.FirstView
if err := json.Unmarshal(raw, &v); err != nil {
return keeper.FirstView{}, fmt.Errorf("the answer is not a first view: %w", err)
}
return v, nil
}
func percent(getenv func(string) string, name string, fallback int64) int64 {
if n, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64); err == nil && n > 0 && n < 100 {
return n
@@ -261,7 +213,7 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer
k := &keeper.Keeper{
Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR")), NC: nc,
Dir: getenv("MESH_NATS_SNAPSHOTS_DIR"), Log: logger.Printf, Server: serverStorageLimit(LiveMonitor())},
Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn),
Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js),
Act: func(what, why, cause string) error {
_, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause})
return err
@@ -274,25 +226,168 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer
SnapshotsCap: settings.SnapshotsCap,
SnapshotKeepDays: settings.SnapshotKeepDays,
}
// The catalog half (design 47 G §433, §532, §614): meshcatalogclient's branch follower on the keeper's views reader, which
// holds its finding writer; and the seat list, each definition as the seat answers that one seat, resolved (§537). Each
// change of a branch's view, of the registrations or of the seat list plans again.
moved := make(chan struct{}, 1)
wakeMoved := func() {
select {
case moved <- struct{}{}:
default:
}
}
follow, err := followCatalog(ctx, conn, writer, wakeMoved, logger.Printf)
if err != nil {
return err
}
k.Catalog, k.Seats, k.Again = follow.view, follow.seats, follow.again
// The process answers the seat mesh-broker's snapshot verbs itself (design 47 G §681, A-107), with the caller the bus proved.
stopAnswers, err := conn.AnswerSeat(meshbroker.Seat, snapshotAnswers(k, heldRequest(k.Store), keeperSnapshots(nc, logger.Printf)))
if err != nil {
return fmt.Errorf("the seat %s's snapshot verbs cannot be answered: %w", meshbroker.Seat, err)
}
defer stopAnswers()
logger.Printf("started on the account %s: asking %s.%s for the first view", account.User, keeper.FirstViewOwner, keeper.FirstViewVerb)
return Keep(ctx, k, hearer(js, account, logger.Printf), requestsOf(conn), logger.Printf)
logger.Printf("started on the account %s: following the catalog half of every watched branch through the seat %s", account.User, catalog.Seat)
return Keep(ctx, k, hearer(js, account, follow.take, follow.bound, logger.Printf), requestsOf(conn), moved, logger.Printf)
}
// hearer binds the keeper's durable consumer on EVENTS (<node>_nats_keeper, filtered on manifest-accepted) and hands
// each message to handle, which acknowledges it once applied.
func hearer(js jetstream.JetStream, a process.Account, logf func(string, ...any)) func(context.Context, func(subject string, body []byte) error) error {
// catalogFollow is the keeper's reading of the catalog half and the seat list (design 47 G §532, §614): the reader, the
// branch follower over it, the seat list, and the events of both, routed from the keeper's one durable consumer.
type catalogFollow struct {
ctx context.Context
reader *views.ProcessReader
branches *meshcatalogclient.Branches
seatList *seatlist.Reader
routes *eventRoutes
}
// followCatalog wires the branch follower and the seat list, and asks their first views; moved is told of each change.
func followCatalog(ctx context.Context, conn *process.Conn, writer views.WireWriter, moved func(), logf func(string, ...any)) (*catalogFollow, error) {
f := &catalogFollow{ctx: ctx, routes: &eventRoutes{by: map[string]func(jetstream.Msg){}}}
f.reader = &views.ProcessReader{Module: keeper.Module, Call: conn.Call, Logf: logf, Findings: func() views.WireWriter { return writer }}
branches, err := meshcatalogclient.FollowBranchesProcess(f.reader, conn.Call)
if err != nil {
return nil, fmt.Errorf("the catalog half cannot be followed: %w", err)
}
branches.Logf = logf
branches.Changed = func(string, string, json.RawMessage, bool, time.Time) { moved() }
// A registration that changes its watched branches' order moves what is planned from which branch (§532).
reconcile := branches.Registrations.Follower.Changed
branches.Registrations.Follower.Changed = func(key string, value json.RawMessage, deleted bool, at time.Time) {
reconcile(key, value, deleted, at)
moved()
}
f.branches = branches
f.seatList, err = seatlist.Start(seatlist.Config{Bus: seatlist.ProcessBus(conn.Call, f.routes), Bootstrap: seatlist.Bootstrapped(),
OnChange: func([]catalogue.Seat) { moved() }})
if err != nil {
logf("the seat list cannot be followed (%v): a module's sent store waits for it", err)
}
f.reader.Ask(ctx, nil)
return f, nil
}
// view is the catalog half as the keeper plans from it: an error while a watched branch has no first view yet.
func (f *catalogFollow) view() (keeper.CatalogView, error) {
if err := f.branches.Ready(); err != nil {
return keeper.CatalogView{}, err
}
var bs []Branch
for _, name := range f.branches.Followed() {
if c, ok := f.branches.Catalog(name); ok {
if half, ok := c.View(); ok {
bs = append(bs, Branch{Name: name, Revision: half.Revision.Revision, Entries: half.Entries})
}
}
}
return ViewOf(bs, func(module string) []string {
reg, ok := f.branches.Registrations.Registration(module)
if !ok {
return nil
}
return reg.Branches
})
}
// seats are the seat list's definitions once the seat has answered; before, none, so no sent store is planned yet
// (ADR 0636 §1: a reader that has not heard acts on nothing new).
func (f *catalogFollow) seats() map[string]seatdef.Definition {
if f.seatList == nil || !f.seatList.Status().Heard {
return nil
}
return f.seatList.Definitions()
}
// again asks every first view of the catalog half and the seat list again, by hand.
func (f *catalogFollow) again() error {
var errs []error
for _, fl := range f.reader.Followers() {
errs = append(errs, fl.Again(f.ctx, "asked again by hand (nats_first_view_again)"))
}
if f.seatList != nil {
errs = append(errs, f.seatList.Reload())
}
return errors.Join(errs...)
}
// take hands a message of the catalog half or the seat list to its reader, which acknowledges it; false for any other.
func (f *catalogFollow) take(m jetstream.Msg) bool {
if h := f.routes.get(m.Subject()); h != nil {
h(m)
return true
}
if strings.HasPrefix(m.Subject(), "mesh.seat."+catalog.Seat+".event.") || strings.HasPrefix(m.Subject(), seatdef.EventSubject(sourcebranches.Seat, sourcebranches.EventSourceMoved)) {
f.reader.Hear(f.ctx)(m)
return true
}
return false
}
// bound tells each view the bus made the keeper's consumer then, so a first view taken before is asked again.
func (f *catalogFollow) bound(created time.Time) {
for _, fl := range f.reader.Followers() {
fl.ConsumerMade(f.ctx, created)
}
}
// eventRoutes are the seat list's handlers of its events by subject, on the keeper's one durable consumer.
type eventRoutes struct {
mu sync.Mutex
by map[string]func(jetstream.Msg)
}
func (r *eventRoutes) Handle(subject string, handle func(jetstream.Msg)) error {
r.mu.Lock()
defer r.mu.Unlock()
if r.by[subject] != nil {
return fmt.Errorf("the events %q are handled already", subject)
}
r.by[subject] = handle
return nil
}
func (r *eventRoutes) get(subject string) func(jetstream.Msg) {
r.mu.Lock()
defer r.mu.Unlock()
return r.by[subject]
}
// hearer binds the keeper's durable consumer on EVENTS (<node>_nats_keeper, filtered on its consumes): a message of the
// catalog half or the seat list goes to take, which acknowledges it; every other to handle, acknowledged once applied.
// bound is told when the bus made the consumer.
func hearer(js jetstream.JetStream, a process.Account, take func(jetstream.Msg) bool, bound func(time.Time), logf func(string, ...any)) func(context.Context, func(subject string, body []byte) error) error {
return func(ctx context.Context, handle func(string, []byte) error) error {
cons, err := js.Consumer(ctx, process.EventsStream, a.QueueConsumerName())
if err != nil {
return fmt.Errorf("the keeper's consumer %s on %s: %w", a.QueueConsumerName(), process.EventsStream, err)
}
if info, err := cons.Info(ctx); err == nil && bound != nil {
bound(info.Created.UTC())
}
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
if take != nil && take(m) {
return
}
if err := handle(m.Subject(), m.Data()); err != nil {
logf("%v: delivered again", err)
_ = m.NakWithDelay(30 * time.Second)
@@ -316,39 +411,25 @@ var FillEvery = time.Minute
var DecidedSubject = oc.SubjectDecided(keeper.Module)
// Keep is the keeper's loop, apart from the bus so a test drives it: a fill pass at once and every minute, from its
// start, so the store's one guard runs while the first view is still asked (design 25 § 12.3); the first view asked
// until it answers, on its backoff; then hear, which applies each manifest-accepted and wakes the persons' requests on
// the operator channel's decided for the bus module's own asks; the waiting entries made again on the same backoff; and
// the persons' requests acted on as they are written or answered, never on a timer. It returns when ctx ends.
// start, so the store's one guard runs while the catalog half's first views are still asked (design 25 § 12.3); a plan
// from the catalog half at once and at each move of it (moved: a branch's view, the registrations or the seat list
// changed), nothing planned until every watched branch has its first view (design 47 G §532, §614); hear, which wakes the
// persons' requests on the operator channel's decided for the bus module's own asks; the waiting entries made again on
// a backoff; and the persons' requests acted on as they are written or answered, never on a timer. It returns when ctx
// ends.
func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func(string, []byte) error) error,
requests func(context.Context, func()) error, logf func(string, ...any)) error {
requests func(context.Context, func()) error, moved <-chan struct{}, logf func(string, ...any)) error {
tick := time.NewTicker(FillEvery)
defer tick.Stop()
k.FillPass()
k.SnapshotsAtStart()
k.EndCutOffRestores()
var wait time.Duration
for {
next, err := k.AskFirstView(wait)
if err == nil {
break
}
logf("the first view did not answer (%v): nothing is created or orphaned until it does; asked again in %s", err, next)
wait = next
again := time.After(next)
waiting:
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-tick.C:
k.FillPass()
case <-again:
break waiting
}
plan := func() {
if err := k.PlanCatalog(); err != nil {
logf("nothing is planned yet: %v", err)
}
}
logf("the first view answered at revision %d", k.Status().FirstView.Revision)
plan()
wake := func(c chan struct{}) func() {
return func() {
select {
@@ -357,31 +438,20 @@ func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func
}
}
}
heard, asked := make(chan struct{}, 1), make(chan struct{}, 1)
asked := make(chan struct{}, 1)
for backoff := 10 * time.Second; ; backoff = min(backoff*2, 5*time.Minute) {
err := hear(ctx, func(subject string, body []byte) error {
if subject == DecidedSubject {
// The operator answered one of the bus module's asks: the waiting requests are read again now.
wake(asked)()
return nil
}
if subject != AcceptedSubject && !strings.HasSuffix(subject, ".event."+catalog.EventManifestAccepted) {
return nil
}
a, err := AcceptedOf(body)
if err != nil {
logf("a manifest-accepted cannot be read (%v): taken, and nothing made of it", err)
return nil
}
k.Heard(a)
wake(heard)()
return nil
})
if err == nil {
break
}
logf("manifest-accepted cannot be heard yet (%v): asked again in %s", err, backoff)
// The fill pass runs on while the consumer is bound again (review round 2, running F1).
logf("the keeper's consumer cannot be heard yet (%v): asked again in %s", err, backoff)
// The fill pass and the plans run on while the consumer is bound again (review round 2, running F1).
bound := time.After(backoff)
binding:
for {
@@ -390,6 +460,8 @@ func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func
return ctx.Err()
case <-tick.C:
k.FillPass()
case <-moved:
plan()
case <-bound:
break binding
}
@@ -415,7 +487,8 @@ func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func
k.FillPass()
case <-asked:
k.ActOnRequests()
case <-heard:
case <-moved:
plan()
case <-again:
again = nil
k.CheckAll()
+50 -44
View File
@@ -6,67 +6,73 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"nats-tools/internal/keeper"
)
// The keeper's inputs (design 25 § 12.2; CAT-N2): a manifest-accepted read with the indexer's revision when it carries
// one (0 when not yet), its entries whole; the first view of indexer_bus_state read bare or as the tool runner wraps it,
// an error answer an error; anything else refused, naming why.
func TestTheKeepersInputsAreRead(t *testing.T) {
// The catalog half as the keeper plans from it (design 47 G §532, §533, §547, §591): every module some watched branch's
// newest revision lists, each local name from the first of its watched branches in its registration's order whose entry
// asks it, another branch asking it with other settings a drift naming that branch; a branch the registration does not
// name after; a verdict not accepted asks nothing; the claims of every branch; a foundation part and its old sections'
// caps from the first branch.
func TestTheCatalogHalfIsReadPerWatchedBranch(t *testing.T) {
entryOf := func(verdict string, manifest map[string]any) catalog.CatalogEntry {
manifest["module"] = "m"
return catalog.CatalogEntry{Module: "m", Verdict: verdict, ManifestDigest: "sha256:" + verdict, Manifest: manifest}
}
state := func(opts string) map[string]any {
e := map[string]any{"name": "drafts", "storage": "key-value", "template": "state"}
if opts != "" {
e["options"] = map[string]any{"history": opts}
}
return e
}
log := map[string]any{"name": "log", "storage": "stream", "template": "log"}
main := entryOf("accepted", map[string]any{"bus-state": []any{state("")}, "claims": []any{map[string]any{"name": "a"}}})
feature := entryOf("accepted", map[string]any{"bus-state": []any{state("5"), log}, "claims": []any{map[string]any{"name": "b"}}})
for _, row := range []struct {
name, body, want, refused string
view bool
name string
branches []Branch
order []string
want string
}{
{name: "accepted with its revision", body: `{"v":1,"module":"m","digest":"sha256:a","judge":"j","revision":7,"bus-state":[{"name":"drafts","storage":"key-value","template":"state"}]}`,
want: "m sha256:a 7 [drafts/key-value/state]"},
{name: "accepted before the event carries a revision", body: `{"v":1,"module":"m","digest":"sha256:b","judge":"j","bus-state":[]}`,
want: "m sha256:b 0 []"},
{name: "accepted naming no module", body: `{"v":1,"digest":"x"}`, refused: "names no module"},
{name: "first view as the tool runner answers it", view: true,
body: `{"result":{"revision":12,"modules":[{"module":"m","digest":"d","revision":11,"bus-state":[{"name":"log","storage":"stream","template":"log"}]}]}}`,
want: "12: m d 11 [log/stream/log]"},
{name: "first view bare", view: true, body: `{"revision":3,"modules":[]}`, want: "3:"},
{name: "first view answering an error", view: true, body: `{"error":"not granted"}`, refused: "not granted"},
{name: "first view that is none", view: true, body: `[1,2]`, refused: "is not a first view"},
{"one branch", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}}, []string{"main"},
"m 3 [drafts/key-value/state] [a] false 0;"},
{"a feature branch asking a local name with other settings drifts, its new one planned",
[]Branch{{Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}, {Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}},
[]string{"main", "feature"}, "m 3 [drafts/key-value/state log/stream/log] [a b] false 0; drifted m drafts feature from main"},
{"the registration's order decides", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}, {Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}},
[]string{"feature"}, "m 9 [drafts/key-value/state log/stream/log] [b a] false 0; drifted m drafts main from feature"},
{"a verdict not accepted asks nothing", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("refused", map[string]any{"bus-state": []any{state("")}})}}}, []string{"main"},
"m 3 [] [] false 0;"},
{"a foundation part, its old sections counted", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("accepted", map[string]any{
"objects": []any{map[string]any{"name": "sent-objects", "max-mib": 256}}, "logs": []any{map[string]any{"name": "l", "max-mib": 4}}, "streams": []any{map[string]any{"name": "s"}}})}}}, []string{"main"},
"m 3 [] [] true 1284;"},
} {
t.Run(row.name, func(t *testing.T) {
v, err := ViewOf(row.branches, func(string) []string { return row.order })
if err != nil {
t.Fatal(err)
}
var got string
var err error
if row.view {
v, e := FirstViewOf([]byte(row.body))
err = e
got = fmt.Sprintf("%d:", v.Revision)
for _, m := range v.Modules {
got += " " + line(m.Module, m.Digest, m.Revision, m.BusState)
for _, m := range v.Modules {
es := []string{}
for _, e := range m.BusState {
es = append(es, e.Name+"/"+e.Storage+"/"+e.Template)
}
} else {
a, e := AcceptedOf([]byte(row.body))
err = e
got = line(a.Module, a.Digest, a.Revision, a.BusState)
got += fmt.Sprintf("%s %d [%s] %v %v %d;", m.Module, m.Revision, strings.Join(es, " "), m.Claims, m.Foundation, m.OldBytes/keeper.MiB)
}
if row.refused != "" {
if err == nil || !strings.Contains(err.Error(), row.refused) {
t.Fatalf("read %s (%v): refused, naming %q", got, err, row.refused)
}
return
for _, d := range v.Drifts {
got += fmt.Sprintf(" drifted %s %s %s from %s", d.Module, d.Local, d.Branch, d.From)
}
if err != nil || got != row.want {
t.Errorf("read %q (%v), want %q", got, err, row.want)
if got != row.want {
t.Errorf("read %q, want %q", got, row.want)
}
})
}
}
func line(module, digest string, revision uint64, entries []catalogue.BusStateEntry) string {
es := []string{}
for _, e := range entries {
es = append(es, e.Name+"/"+e.Storage+"/"+e.Template)
}
return fmt.Sprintf("%s %s %d [%s]", module, digest, revision, strings.Join(es, " "))
}
// The keeper's health tool (CAT-N6): healthy naming its pass; unhealthy, naming why, when its pass is old or verdicts
// wait on its consumer `<node>_nats_keeper` past the bound.
func TestTheKeepersHealthToolReadsItsStatusAndItsConsumer(t *testing.T) {
+1 -1
View File
@@ -82,7 +82,7 @@ func keeperHealth(status func() (keeper.Status, error), read heldEntries, jsz fu
func keeperHealthTool(status func() (keeper.Status, error), read heldEntries, jsz func() ([]byte, error)) stdio.Tool {
return stdio.Tool{Name: "nats_keeper_health",
Description: "The bus module's keeper's health, as the node-engine asks it: unhealthy, naming why, when its fill pass is more than three minutes old, its first view (the indexer's indexer_bus_state) has not answered, or accepted verdicts wait on its consumer unanswered past three minutes; it names beside the entries it could not create, the ones waiting, and those past the raise mark (another module's full entry raises that module's condition and leaves the keeper healthy).",
Description: "The bus module's keeper's health, as the node-engine asks it: unhealthy, naming why, when its fill pass is more than three minutes old, the catalog half's first views of its watched branches (the seat mesh-catalog's revision) have not answered, or events wait on its consumer unanswered past three minutes; it names beside the entries it could not create, the ones waiting, and those past the raise mark (another module's full entry raises that module's condition and leaves the keeper healthy).",
Run: func(map[string]any) (any, error) {
return keeperHealth(status, read, jsz, func() time.Time { return time.Now().UTC() }), nil
}}
@@ -9,6 +9,7 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"slices"
"strings"
@@ -24,8 +25,10 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/bussnap"
@@ -108,7 +111,8 @@ func eventually(t *testing.T, what string, ok func() bool) {
}
// The bus module runs whole (design 25 § 12.7, "The bus module runs whole" and "The bus module's first view"; CAT-N10):
// its keeper against the real bus server, hearing manifest-accepted on its durable consumer, creating on the bus,
// its keeper against the real bus server, planning from the catalog half of every watched branch and the seat list
// (design 47 G §344, §532, §533, §537, §538) at each move of them, creating on the bus,
// saying its events on its own stream BROKER_EVENTS, keeping its state in its own buckets, and acting on a person's
// requests. One server for every case, in order, as the mesh runs them.
func TestTheBusModuleRunsWhole(t *testing.T) {
@@ -144,7 +148,7 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
// The keeper's consumer hears the indexer's verdicts and the operator channel's decided for the bus module's own
// asks, as the grants part composes it from its consumes and the seats it uses.
if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: account.QueueConsumerName(),
FilterSubjects: []string{AcceptedSubject, DecidedSubject}, AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 5}); err != nil {
FilterSubjects: []string{DecidedSubject}, AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 5}); err != nil {
t.Fatal(err)
}
// The router's own record of the asks, which the keeper reads under its own process account (process.Conn.Ask), never
@@ -166,7 +170,18 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
store := kvStore{js}
snapshots := t.TempDir()
var mu sync.Mutex
view := keeper.FirstView{Revision: 1}
// The catalog half as meshcatalogclient's branch follower holds it, branch by branch in the registrations' order, and
// the seat list's definitions; each change moves the keeper, as the follower's and the seat list's changes do.
branches := []Branch{{Name: "main", Revision: 1, Entries: map[string]catalog.CatalogEntry{}}}
order := []string{"main"}
defs := map[string]seatdef.Definition{}
moved := make(chan struct{}, 1)
move := func() {
select {
case moved <- struct{}{}:
default:
}
}
var acts []string
// The bus module's findings go through a held writer over its state findings, started as the keeper's process starts it.
writer, err := views.NewHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{
@@ -182,7 +197,13 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
newKeeper := func() *keeper.Keeper {
return &keeper.Keeper{Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, snapshots), NC: nc, Dir: snapshots, Log: t.Logf}, Store: store,
Finding: writer, Emit: streamEmitter(js, account), Say: sayer(js),
View: func() (keeper.FirstView, error) { mu.Lock(); defer mu.Unlock(); return view, nil },
Catalog: func() (keeper.CatalogView, error) {
mu.Lock()
defer mu.Unlock()
return ViewOf(branches, func(string) []string { return order })
},
Seats: func() map[string]seatdef.Definition { mu.Lock(); defer mu.Unlock(); return maps.Clone(defs) },
Again: func() error { return nil },
Act: func(what, why, cause string) error {
mu.Lock()
defer mu.Unlock()
@@ -208,15 +229,28 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
}()
return nil
}
go func() { _ = Keep(kctx, k, hearer(js, account, t.Logf), watch, t.Logf); close(done) }()
go func() { _ = Keep(kctx, k, hearer(js, account, nil, nil, t.Logf), watch, moved, t.Logf); close(done) }()
return func() { stop(); <-done }
}
accepted := func(module string, revision uint64, entries ...catalogue.BusStateEntry) {
body, _ := json.Marshal(map[string]any{"v": 1, "module": module, "digest": fmt.Sprintf("sha256:%d", revision),
"judge": "j", "revision": revision, "bus-state": entries})
if _, err := js.Publish(ctx, AcceptedSubject, body); err != nil {
t.Fatal(err)
// onBranch puts a module's entry in a branch's newest revision (none: the module leaves it), and moves the keeper.
onBranch := func(branch, module string, revision int64, manifest map[string]any) {
mu.Lock()
for i := range branches {
if branches[i].Name == branch {
branches[i].Revision = revision
if manifest == nil {
delete(branches[i].Entries, module)
} else {
manifest["module"] = module
branches[i].Entries[module] = catalog.CatalogEntry{Module: module, Verdict: "accepted", ManifestDigest: fmt.Sprintf("sha256:%d", revision), Manifest: manifest}
}
}
}
mu.Unlock()
move()
}
accepted := func(module string, revision uint64, entries ...catalogue.BusStateEntry) {
onBranch("main", module, int64(revision), map[string]any{"bus-state": entries})
}
// saidOn reads what one stream holds of the bus module's events, by its subjects' prefix: its own stream (its
// history), or EVENTS on the seat mesh-broker's subjects (what every consumer of mesh-broker.<event> hears).
@@ -264,11 +298,13 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
// one guard never waits on the indexer.
t.Run("the fill pass runs while the first view is asked", func(t *testing.T) {
down := newKeeper()
down.View = func() (keeper.FirstView, error) { return keeper.FirstView{}, errors.New("no responders") }
down.Catalog = func() (keeper.CatalogView, error) {
return keeper.CatalogView{}, errors.New("the catalog half has no first view of main yet")
}
stopDown := run(down)
eventually(t, "a fill pass with no first view", func() bool { return !down.Status().Pass.IsZero() && down.Status().FirstView.Answered.IsZero() })
stopDown()
// And while its consumer of manifest-accepted is bound again on its backoff (review round 2, running F1).
// And while its consumer is bound again on its backoff (review round 2, running F1).
was := FillEvery
FillEvery = 20 * time.Millisecond
defer func() { FillEvery = was }()
@@ -276,7 +312,7 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
kctx, stopDeaf := context.WithCancel(ctx)
done := make(chan struct{})
go func() {
_ = Keep(kctx, deaf, func(context.Context, func(string, []byte) error) error { return errors.New("no consumer yet") }, nil, t.Logf)
_ = Keep(kctx, deaf, func(context.Context, func(string, []byte) error) error { return errors.New("no consumer yet") }, nil, nil, t.Logf)
close(done)
}()
eventually(t, "the first view answered", func() bool { return !deaf.Status().FirstView.Answered.IsZero() })
@@ -309,27 +345,6 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
stop = run(k)
eventually(t, "created n jobs", saidHas("created n jobs"))
})
t.Run("a replay older than the first view is not applied; a set-aside module keeps its key", func(t *testing.T) {
stop()
mu.Lock()
view = keeper.FirstView{Revision: 9, Modules: []keeper.Accepted{{Module: "p", Revision: 9, BusState: []catalogue.BusStateEntry{state}}}}
mu.Unlock()
accepted("p", 4)
k = newKeeper()
stop = run(k)
eventually(t, "created p drafts", saidHas("created p drafts"))
eventually(t, "the replay heard", func() bool {
info, _ := js.Consumer(ctx, "EVENTS", account.QueueConsumerName())
i, _ := info.Info(ctx)
return i.NumPending == 0 && i.NumAckPending == 0
})
if saidHas("orphaned p drafts")() {
t.Error("the replay at revision 4 orphaned what the first view at 9 asked")
}
if _, _, found, _ := store.Get(keeper.StateAsked, "m"); !found {
t.Error("m, missing from the first view, lost its key")
}
})
t.Run("drift is said, a growth applied and the drift cleared", func(t *testing.T) {
accepted("m", 5, state, entry("log", "stream", "log", map[string]string{"max-mib": "1", "max-age-days": "2"}))
eventually(t, "drifted m log", saidHas("drifted m log"))
@@ -554,6 +569,63 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
t.Error("the snapshot's record outlived its file")
}
})
// §532: a branch registered while the keeper runs is followed with no restart; a feature branch asking a local name with
// other settings than the default branch's is said drifted, naming it, while the default branch's are planned, and its
// new local name is planned from it.
t.Run("a branch registered while it runs is planned from, a local name with other settings drifted", func(t *testing.T) {
mu.Lock()
branches = append(branches, Branch{Name: "feature", Revision: 4, Entries: map[string]catalog.CatalogEntry{"m": {Module: "m", Verdict: "accepted",
Manifest: map[string]any{"module": "m", "bus-state": []any{state, entry("log", "stream", "log", map[string]string{"max-mib": "2"}), entry("extra", "key-value", "state", nil)}}}}})
order = []string{"main", "feature"}
mu.Unlock()
move()
eventually(t, "created m extra", saidHas("created m extra"))
eventually(t, "drifted m log", saidHas("drifted m log"))
if info, _ := held("m_log"); info.Config.MaxBytes != 4<<20 {
t.Errorf("m_log's cap is %d: the default branch's 4 MiB planned", info.Config.MaxBytes)
}
mu.Lock()
branches, order = branches[:1], []string{"main"}
mu.Unlock()
move()
eventually(t, "orphaned m extra", saidHas("orphaned m extra"))
})
// §344, §533, §537: a module of the new form claiming a seat with a verb gets a sent store, one with neither none, a seat
// whose verbs come only from an included part one, a seat including only bodies none, a foundation part none; a
// definition gaining a verb gives its holder one. §538: a module in no watched branch's newest revision loses its plans key.
t.Run("the sent stores follow the catalog half and the seat list", func(t *testing.T) {
mu.Lock()
defs["verbed"] = seatdef.Definition{Seat: "verbed", Verbs: []seatdef.Verb{{Name: "go"}}}
defs["bare"] = seatdef.Definition{Seat: "bare"}
defs["resolved"] = seatdef.Definition{Seat: "resolved", Includes: []string{"part"}, Verbs: []seatdef.Verb{{Name: "from-the-part"}}}
defs["bodies"] = seatdef.Definition{Seat: "bodies", Includes: []string{"shared-bodies"}}
mu.Unlock()
claims := func(seat string) map[string]any { return map[string]any{"claims": []any{map[string]any{"name": seat}}} }
onBranch("main", "sa", 5, claims("verbed"))
onBranch("main", "sb", 5, claims("bare"))
onBranch("main", "sc", 5, claims("resolved"))
onBranch("main", "sd", 5, claims("bodies"))
f := claims("verbed")
f["objects"] = []any{map[string]any{"name": "sent-objects", "max-mib": 256}}
onBranch("main", "se", 5, f)
eventually(t, "sa and sc's sent stores", func() bool {
_, a := held("OBJ_sa_sent-objects")
_, c := held("OBJ_sc_sent-objects")
return a && c
})
for _, none := range []string{"sb", "sd", "se"} {
if _, ok := held("OBJ_" + none + "_sent-objects"); ok {
t.Errorf("%s has a sent store", none)
}
}
mu.Lock()
defs["bodies"] = seatdef.Definition{Seat: "bodies", Includes: []string{"shared-bodies"}, Verbs: []seatdef.Verb{{Name: "new"}}}
mu.Unlock()
move()
eventually(t, "sd's sent store once its seat gains a verb", func() bool { _, ok := held("OBJ_sd_sent-objects"); return ok })
onBranch("main", "sb", 6, nil)
eventually(t, "sb's plan deleted", func() bool { _, _, found, _ := store.Get(keeper.StatePlans, "sb"); return !found })
})
t.Run("nats_first_view_again asks at once and records the hand act", func(t *testing.T) {
raw, _ := json.Marshal(keeper.Request{ID: "r3", Verb: keeper.VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"})
_, _ = store.Put(keeper.StateRequests, "r3", raw)
+1 -1
View File
@@ -193,7 +193,7 @@ func allTools() []stdio.Tool {
}
out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState)...)
out = append(out, expiryTools(LiveBus(), SaidByTheRuntime, RecordedBySeat)...)
out = append(out, brokerTools(HeldInState, func() (keeper.Limits, error) { s, err := StatusInState(); return s.Limits, err }, jsz, StreamHistory(LiveBus()))...)
out = append(out, brokerTools(HeldInState, StatusInState, jsz, StreamHistory(LiveBus()))...)
out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz))
out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...)
rs := RequestsInState()
+13 -8
View File
@@ -161,8 +161,8 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
}
}
// The bus module provides bus state (design 25 § 12.2-12.3, ADR 0595 §4-§6; CAT-N1): its keeper is a process of
// its own on its own account (own secret broker-keeper, ADR 0436 §1), creating only bus state; it hears the indexer's
// manifest-accepted, keeps what it heard in its own old state `asked` (ADR 0595 §10) and the persons' requests in
// its own on its own account (own secret broker-keeper, ADR 0436 §1), creating only bus state; it reads the catalog half
// of every watched branch and the seat list (design 47 G §433, §532, §614), keeps what it planned in its own old state `asked` (ADR 0595 §10) and the persons' requests in
// `requests`; its nine events are its own (ADR 0620 §2), on its own stream that keeps its history (ADR 0592 §6, the manifest
// validator's predefined form) at 256 MiB; the seat's four
// reading verbs are served, its four acting verbs are the keeper's own work and no other account's; its health is
@@ -194,17 +194,22 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
if len(nm.Claims) != 1 || nm.Claims[0].Name != "mesh-broker" || !slices.Equal(nm.Claims[0].Serves, append(slices.Clone(ReadingVerbs), SnapshotVerbs...)) {
t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v and its snapshot verbs %v alone", nm.Claims, ReadingVerbs, SnapshotVerbs)
}
if !slices.Equal(nm.Consumes, []string{"mesh-catalog.manifest-accepted", "mesh-conditions.findings-asked"}) {
t.Errorf("consumes %v: the seat mesh-catalog's manifest-accepted (ADR 0595 §4) and the finding writer's findings-asked (design 47 G A-17)", nm.Consumes)
// The catalog half replaces manifest-accepted and indexer_bus_state (design 47 G §433, §614), and the seat list is read
// through the seat mesh-catalog (§537).
catalogHalf := []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved",
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"}
if !slices.Equal(nm.Consumes, append(slices.Clone(catalogHalf), "mesh-conditions.findings-asked")) {
t.Errorf("consumes %v: the catalog half's and the seat list's events %v, and the finding writer's findings-asked (design 47 G A-17)", nm.Consumes, catalogHalf)
}
if AcceptedSubject != "mesh.seat.mesh-catalog.event.manifest-accepted" {
t.Errorf("the keeper's durable consumer filters on %s, not on the seat mesh-catalog's event", AcceptedSubject)
}
for _, want := range []string{"mesh-delivery-indexer.indexer_bus_state", "seat:mesh-hand-acts.hand-act"} {
for _, want := range []string{"seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source",
"seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act"} {
if !slices.Contains(nm.Invokes, want) {
t.Errorf("invokes %v lack %s", nm.Invokes, want)
}
}
if strings.Contains(string(raw), "indexer_bus_state") || strings.Contains(string(raw), "manifest-accepted") {
t.Error("the manifest still names indexer_bus_state or manifest-accepted (design 47 G §433)")
}
if !slices.Contains(nm.Uses, "operator-channel") {
t.Errorf("uses %v: the operator channel, for the warrant of recreate and drop", nm.Uses)
}
+1 -1
View File
@@ -209,7 +209,7 @@ func requestTools(rs requestStore) []stdio.Tool {
}}
}
firstView := stdio.Tool{Name: "nats_first_view_again",
Description: "Ask the bus module's keeper to ask the indexer's indexer_bus_state, its first view of every accepted manifest's bus state, at once (design 25 § 12.2): by hand, beside its own backoff, recorded as a hand act; answers what the first view answered and its revision.",
Description: "Ask the bus module's keeper to ask the catalog half's first views of every watched branch and the seat list again at once, and plan from them (design 47 G §614): by hand, recorded as a hand act; answers whether they answered and the newest revision planned from.",
Input: map[string]any{"why": str("why, in plain words: kept with the act"), "request": str("a request made before: its end is read")},
Run: func(a map[string]any) (any, error) {
if id := arg(a, "request"); id != "" {
+2
View File
@@ -8,7 +8,9 @@ require (
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
github.com/klauspost/compress v1.20.1
+169 -67
View File
@@ -17,6 +17,7 @@ import (
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// Module is the bus module's own name: the publisher of its events and its conditions.
@@ -81,8 +82,8 @@ type EventFill struct {
Cap int64 `json:"cap"`
}
// Accepted is a manifest-accepted the keeper heard, or a module of its first view: the module, its manifest's digest,
// the indexer's revision of the verdict (0 when the event does not say it yet), and its bus-state entries whole.
// Accepted is one module's entry as the catalog half's watched branches give it (design 47 G §344, §433, §532): the
// module, its manifest's digest, the catalog revision of the branch it was read from, and its bus-state entries whole.
type Accepted struct {
Module string `json:"module"`
Digest string `json:"digest"`
@@ -90,10 +91,33 @@ type Accepted struct {
BusState []catalogue.BusStateEntry `json:"bus-state"`
}
// FirstView is the indexer's answer of indexer_bus_state: every accepted manifest's bus-state entries, by module.
type FirstView struct {
Revision uint64 `json:"revision"`
Modules []Accepted `json:"modules"`
// CatalogModule is one module of the catalog half for the bus module (design 47 G §344, §532, §533, §547): what its
// watched branches' newest revisions ask of the bus, the seats they claim, whether one declares an event of its own,
// whether it is a foundation part (it declares an old `objects` section `sent-objects`), and the caps of the old
// sections it declares, in bytes.
type CatalogModule struct {
Accepted
Claims []string `json:"claims,omitempty"`
EmitsOwn bool `json:"emits-own,omitempty"`
Foundation bool `json:"foundation,omitempty"`
OldBytes int64 `json:"old-bytes,omitempty"`
}
// Drift is a local name a later watched branch asks with other settings than the branch it is planned from (design 47
// G §532): said as the bus module's drifted, naming the branch, and planned from nothing of that branch.
type Drift struct {
Module string `json:"module"`
Local string `json:"local-name"`
Branch string `json:"branch"`
// From is the branch whose settings are planned.
From string `json:"from"`
}
// CatalogView is the catalog half as the bus module plans from it: every module in a watched branch's newest revision,
// and the drifts between branches.
type CatalogView struct {
Modules []CatalogModule
Drifts []Drift
}
// Planned is the value of a module's key in StatePlans: the entries as planned, their digest and the revision of the verdict.
@@ -123,7 +147,9 @@ type Asked struct {
Revision uint64 `json:"revision"`
Entries []catalogue.BusStateEntry `json:"entries"`
Made map[string]Made `json:"made,omitempty"`
At time.Time `json:"at"`
// Drifts are the local names a later watched branch asks with other settings, as last said (design 47 G §532).
Drifts map[string]Drift `json:"drifts,omitempty"`
At time.Time `json:"at"`
}
// Made is one entry as the keeper made it: its plan and what stands of it now.
@@ -151,9 +177,13 @@ type Status struct {
Pass time.Time `json:"pass"`
// Limits are the bus's storage limits as last read (design 47 G §547): what `list` shows against the sum of the caps.
Limits Limits `json:"limits,omitempty"`
// FirstView is the indexer's first view: when asked, when answered and at which revision, or why not.
// OldBytes is the sum of the caps of every old section the catalog graph's entries declare, in bytes, as last planned
// (design 47 G §547, §591): counted in the sum of the bus's caps beside the entries made.
OldBytes int64 `json:"old-sections-bytes,omitempty"`
// FirstView is the catalog half's first views (design 47 G §614): when last planned from, when answered and at which
// revision, or what it waits for.
FirstView FirstViewStatus `json:"first-view"`
// Heard is when the last manifest-accepted was heard, and Unanswered since when one has waited unhandled.
// Heard is when the catalog half last moved, and Unanswered since when an event has waited unhandled.
Heard time.Time `json:"heard,omitempty"`
Unanswered time.Time `json:"unanswered,omitempty"`
// Unsaid are the events the bus did not take, kept with their id and the subjects still to say them on, and
@@ -191,10 +221,11 @@ type FirstViewStatus struct {
Next time.Time `json:"next,omitempty"`
}
// The first view's owner and verb (design 25 § 12.2).
// The first view's seat and verb: the catalog half of each watched branch, read through the seat mesh-catalog's
// `revision` (design 47 G §433, §614).
const (
FirstViewOwner = "mesh-delivery-indexer"
FirstViewVerb = "indexer_bus_state"
FirstViewOwner = "mesh-catalog"
FirstViewVerb = "revision"
)
// Bus is the bus as the keeper's own account reaches it.
@@ -257,7 +288,7 @@ func (k *Keeper) capRoom(ctx context.Context, held *Asked, m *Made) error {
}
all := k.Held()
all[held.Module] = *held
sum := CapSum(all, held.Module, m.Plan.Local)
sum := CapSum(all, held.Module, m.Plan.Local) + k.Status().OldBytes
for _, o := range m.Plan.Objects {
sum += o.Settings.MaxBytes
}
@@ -297,8 +328,12 @@ type Keeper struct {
// Emit says one of the bus module's events (Saying); Say raises or clears a condition through the condition keeper.
Emit func(s *Saying) error
Say func(oc.Condition) error
// View asks the indexer's indexer_bus_state.
View func() (FirstView, error)
// Catalog reads the catalog half of every watched branch as it stands (design 47 G §433, §532): an error while a
// branch has no first view yet, when the keeper plans nothing. Seats are the seat list's definitions, each resolved
// with the parts it includes (§537). Again asks every first view of the catalog half again, by hand.
Catalog func() (CatalogView, error)
Seats func() map[string]seatdef.Definition
Again func() error
// Act records a hand act through the seat mesh-hand-acts; Warrant reads the router's record of an ask.
Act func(what, why, cause string) error
Warrant func(askID string) (*oc.Ask, *oc.Warrant, error)
@@ -417,81 +452,148 @@ func (k *Keeper) statusWritten(s Status, err error) {
}
}
// AskFirstView asks the indexer's first view once and applies it (design 25 § 12.2): every module it answers is
// applied as an accepted verdict, a module the keeper holds and the view leaves out is kept (a set-aside manifest is
// missing from it, never removed), and then check runs over what is held. It answers the error and when to ask again,
// on the backoff from 10 s doubling to 5 min; the keeper's health is unhealthy meanwhile, naming the verb.
func (k *Keeper) AskFirstView(wait time.Duration) (time.Duration, error) {
// PlanCatalog plans from the catalog half as it stands (design 47 G §344, §433, §532, §537, §538): each module in a
// watched branch's newest revision is applied (Heard), a module of the new form that sends gets its sent store, a drift
// between branches is said once, and a module in no watched branch's newest revision has its plans key deleted, its
// entries then orphans. Before every watched branch has its first view it plans nothing, and says what it waits for in
// its status; the first plan checks what is held.
func (k *Keeper) PlanCatalog() error {
at := k.now()
k.setStatus(func(s *Status) { s.FirstView.Asked = at })
view, err := k.View()
view, err := k.Catalog()
if err != nil {
next := wait * 2
if wait <= 0 {
next = 10 * time.Second
}
if next > 5*time.Minute {
next = 5 * time.Minute
}
k.setStatus(func(s *Status) {
s.FirstView.Error = fmt.Sprintf("%s.%s did not answer: %v", FirstViewOwner, FirstViewVerb, err)
s.FirstView.Next = at.Add(next)
})
return next, err
k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Error = at, err.Error() })
return err
}
k.mu.Lock()
first := !k.viewed
k.viewed = true
k.mu.Unlock()
k.setStatus(func(s *Status) {
s.FirstView.Answered, s.FirstView.Revision, s.FirstView.Error, s.FirstView.Next = k.now(), view.Revision, "", time.Time{}
})
for _, a := range view.Modules {
k.Heard(a)
var revision uint64
var old int64
for _, m := range view.Modules {
revision = max(revision, m.Revision)
old += m.OldBytes
}
k.CheckAll()
return 0, nil
k.setStatus(func(s *Status) {
s.FirstView.Asked, s.FirstView.Revision, s.FirstView.Error, s.FirstView.Next = at, revision, "", time.Time{}
if s.FirstView.Answered.IsZero() {
s.FirstView.Answered = at
}
s.OldBytes = old
})
var defs map[string]seatdef.Definition
if k.Seats != nil {
defs = k.Seats()
}
listed := map[string]bool{}
for _, m := range view.Modules {
listed[m.Module] = true
k.Heard(m.Accepted)
if !m.Foundation && Sends(m.Claims, m.EmitsOwn, defs) {
k.Sends(m.Module)
}
}
k.drifts(view.Drifts)
gone := map[string]bool{}
for module := range k.Held() {
gone[module] = !listed[module]
}
if keys, err := k.Store.Keys(StatePlans); err == nil {
for _, module := range keys {
gone[module] = !listed[module]
}
}
for _, module := range sortedKeys(gone) {
if !gone[module] {
continue
}
if _, ok := k.planned(module); ok {
k.Heard(Accepted{Module: module, Revision: revision})
}
if err := k.Store.Delete(StatePlans, module); err != nil {
k.logf("the plan of %s cannot be removed from its own %s: %v", module, StatePlans, err)
}
}
if first {
k.CheckAll()
}
return nil
}
// Heard applies one accepted verdict (manifest-accepted, or a module of the first view): applied only when newer than
// what the keeper holds for that module by the indexer's revision alone, so a replay never takes an entry back (a
// verdict that says no revision is applied when its digest differs). Each entry asked is made where the bus lacks it;
// each entry made that is no longer asked is an orphan. An empty verdict ("this module asks nothing now") ends the
// module's key once nothing it made is held.
// drifts says each drift once (design 47 G §532): kept in the module's key, said again only when the branch or the
// settings planned change.
func (k *Keeper) drifts(ds []Drift) {
by := map[string]map[string]Drift{}
for _, d := range ds {
if by[d.Module] == nil {
by[d.Module] = map[string]Drift{}
}
by[d.Module][d.Local] = d
}
for module, a := range k.Held() {
if by[module] == nil && len(a.Drifts) > 0 {
by[module] = map[string]Drift{}
}
}
for _, module := range sortedKeys(by) {
held, _, found := k.asked(module)
if !found {
held = Asked{V: 1, Module: module, At: k.now()}
}
now := by[module]
changed := len(now) != len(held.Drifts)
for local, d := range now {
if held.Drifts[local] != d {
changed = true
var p Plan
for _, e := range held.Entries {
if e.Name == local {
p, _ = PlanOf(module, e)
}
}
p.Module, p.Local = module, local
k.emit(EventDrifted, p, held.Revision, fmt.Sprintf("the branch %s asks %s of %s with other settings than the branch %s, whose are planned: nothing of %s is planned", d.Branch, local, module, d.From, d.Branch), nil, nil)
}
}
if !changed {
continue
}
held.Drifts = nil
if len(now) > 0 {
held.Drifts = now
}
if found || len(now) > 0 {
k.write(held)
}
}
}
// Heard applies one module's entry of the catalog half: it plans again only when its bus-state entries differ from those
// it last planned (design 47 G §538), never on the manifest's digest, which changes with any field. Each entry asked is
// made where the bus lacks it; each entry made that is no longer asked is an orphan. An entry that asks nothing ("this
// module asks nothing now") ends the module's key once nothing it made is held.
func (k *Keeper) Heard(a Accepted) {
k.setStatus(func(s *Status) { s.Heard = k.now() })
if !k.Viewed() {
k.logf("manifest-accepted of %s heard before the first view: kept on the consumer until it is answered", a.Module)
k.logf("the entry of %s is heard before the catalog half's first views: nothing is planned until they are in", a.Module)
return
}
held, _, found := k.asked(a.Module)
if found && a.Revision != 0 && a.Revision <= held.Revision {
k.logf("manifest-accepted of %s at revision %d is not newer than the %d held: not applied", a.Module, a.Revision, held.Revision)
return
}
if found && a.Revision == 0 && a.Digest != "" && a.Digest == held.Digest {
return
}
if !found {
held = Asked{V: 1, Module: a.Module}
}
if held.Made == nil {
held.Made = map[string]Made{}
}
// It plans again only when its bus-state entries differ from those it last planned, never on the manifest's digest
// (design 47 G §538).
digest := EntriesDigest(a.BusState)
if p, ok := k.planned(a.Module); ok && found && p.Digest == digest {
held.Digest, held.At = a.Digest, k.now()
if a.Revision > held.Revision {
held.Revision = a.Revision
if held.Digest != a.Digest || held.Revision != a.Revision {
held.Digest, held.Revision, held.At = a.Digest, a.Revision, k.now()
k.write(held)
}
k.write(held)
return
}
held.Digest, held.Entries, held.At = a.Digest, a.BusState, k.now()
if a.Revision > held.Revision {
held.Revision = a.Revision
}
held.Digest, held.Entries, held.Revision, held.At = a.Digest, a.BusState, a.Revision, k.now()
defer k.writePlan(a.Module, digest, held.Revision, a.BusState)
asked := map[string]bool{}
for _, e := range a.BusState {
@@ -512,10 +614,10 @@ func (k *Keeper) Heard(a Accepted) {
}
m.Orphaned = true
held.Made[local] = m
k.emit(EventOrphaned, m.Plan, held.Revision, "no accepted manifest of "+a.Module+" asks for it now", nil, nil)
k.emit(EventOrphaned, m.Plan, held.Revision, "no watched branch's newest revision of "+a.Module+" asks for it now", nil, nil)
k.raise(conditionOrphaned, a.Module, local, occ.SeverityWarning, "Kept data no manifest asks for",
"the operator's drop of it, or a manifest asking for it again",
fmt.Sprintf("%s of %s is on the bus and no accepted manifest asks for it: it goes only by the bus module's drop, on your warrant (nats_drop), or a manifest asking for it again keeps it", BusName(a.Module, local), a.Module))
fmt.Sprintf("%s of %s is on the bus and no watched branch's newest revision asks for it: it goes only by the bus module's drop, on your warrant (nats_drop), or a manifest asking for it again keeps it", BusName(a.Module, local), a.Module))
}
k.write(held)
}
@@ -895,7 +997,7 @@ func (k *Keeper) asked(module string) (Asked, uint64, bool) {
// write keeps a module's key, or removes it when its manifest asks for nothing and nothing it made is held: a key goes
// only so, never because a module is missing from a first view.
func (k *Keeper) write(a Asked) {
if len(a.Entries) == 0 && len(a.Made) == 0 {
if len(a.Entries) == 0 && len(a.Made) == 0 && len(a.Drifts) == 0 {
if err := k.Store.Delete(StateAsked, a.Module); err != nil {
k.logf("the key of %s cannot be removed from its own %s: %v", a.Module, StateAsked, err)
}
+85 -24
View File
@@ -302,12 +302,12 @@ func (f *findingLog) Clear(_ context.Context, scope, id, kind, _ string) error {
return nil
}
func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) {
func newKeeper(view CatalogView) (*Keeper, *fakeBus, *memStore, *said) {
bus, store, out := newFakeBus(), newMemStore(), &said{}
at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)
k := &Keeper{Bus: bus, Store: store, Finding: &findingLog{}, Emit: out.emit, Say: out.say, Files: &memFiles{size: map[string]int64{}},
View: func() (FirstView, error) { return view, nil },
Now: func() time.Time { return at }}
Catalog: func() (CatalogView, error) { return view, nil },
Now: func() time.Time { return at }}
return k, bus, store, out
}
@@ -445,7 +445,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
if len(bus.created) != 0 {
t.Fatalf("created %v before the first view", bus.created)
}
if _, err := k.AskFirstView(0); err != nil {
if err := k.PlanCatalog(); err != nil {
t.Fatal(err)
}
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
@@ -492,11 +492,66 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
t.Errorf("the sent store was orphaned or removed: %+v %v", held.Made[SentLocal], bus.deleted)
}
}},
{"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}})
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil})
if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
t.Errorf("held %+v: the replay at 4 took back what 9 asked", a)
// §344, §533, §537: a module of the new form claiming a seat whose resolved definition has a verb or an event, or
// declaring an event of its own, gets a sent store; a seat including only bodies, and a foundation part, none; a
// definition gaining a verb gives its holder one at the next plan.
{"the catalog half and the seat list plan the sent stores", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
defs := map[string]seatdef.Definition{
"verbed": {Seat: "verbed", Verbs: []seatdef.Verb{{Name: "go"}}},
"resolved": {Seat: "resolved", Includes: []string{"part"}, Verbs: []seatdef.Verb{{Name: "from-the-part"}}},
"bodies": {Seat: "bodies", Includes: []string{"shared-bodies"}},
}
k.Seats = func() map[string]seatdef.Definition { return defs }
k.Catalog = func() (CatalogView, error) {
return CatalogView{Modules: []CatalogModule{
{Accepted: Accepted{Module: "a"}, Claims: []string{"verbed"}},
{Accepted: Accepted{Module: "b"}, Claims: []string{"resolved"}},
{Accepted: Accepted{Module: "c"}, Claims: []string{"bodies"}},
{Accepted: Accepted{Module: "d"}, Claims: []string{"verbed"}, Foundation: true},
{Accepted: Accepted{Module: "e"}, EmitsOwn: true},
}}, nil
}
_ = k.PlanCatalog()
if want := "[OBJ_a_sent-objects OBJ_b_sent-objects OBJ_e_sent-objects]"; fmt.Sprint(bus.created) != want {
t.Fatalf("created %v, want %s", bus.created, want)
}
defs["bodies"] = seatdef.Definition{Seat: "bodies", Includes: []string{"shared-bodies"}, Verbs: []seatdef.Verb{{Name: "new"}}}
_ = k.PlanCatalog()
if want := "[OBJ_a_sent-objects OBJ_b_sent-objects OBJ_e_sent-objects OBJ_c_sent-objects]"; fmt.Sprint(bus.created) != want {
t.Errorf("created %v, want %s: the holder of a definition gaining a verb gets its store", bus.created, want)
}
}},
// §532: a later branch asking a local name with other settings is said drifted once, naming the branch; §547: the
// old sections' caps count in the sum.
{"a drift between branches is said once, naming the branch", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
view := CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}}, OldBytes: 8 * MiB}},
Drifts: []Drift{{Module: "m", Local: "drafts", Branch: "feature", From: "main"}}}
k.Catalog = func() (CatalogView, error) { return view, nil }
_ = k.PlanCatalog()
_ = k.PlanCatalog()
if fmt.Sprint(out.events) != "[created m drafts drifted m drafts]" || !strings.Contains(out.bodies[1].Why, "the branch feature asks drafts") {
t.Errorf("said %v (%+v): created from main, drifted once naming feature", out.events, out.bodies)
}
if k.Status().OldBytes != 8*MiB {
t.Errorf("old sections %d: 8 MiB counted", k.Status().OldBytes)
}
}},
// §538: a module in no watched branch's newest revision has its plans key deleted, its entries then orphans.
{"a module no watched branch lists loses its plan, its entries orphans", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Catalog = func() (CatalogView, error) {
return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil
}
_ = k.PlanCatalog()
if _, ok := k.planned("m"); !ok {
t.Fatal("m's plan is not kept")
}
k.Catalog = func() (CatalogView, error) { return CatalogView{}, nil }
_ = k.PlanCatalog()
if _, ok := k.planned("m"); ok {
t.Error("m's plan is kept while no watched branch lists it")
}
if a := k.Held()["m"]; !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 {
t.Errorf("held %+v, deleted %v: drafts an orphan, kept on the bus", a, bus.deleted)
}
}},
{"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
@@ -641,12 +696,14 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
}},
{"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil }
if _, err := k.AskFirstView(0); err != nil {
t.Fatal(err)
k.Catalog = func() (CatalogView, error) {
return CatalogView{}, errors.New("the catalog half has no first view of main yet")
}
if err := k.PlanCatalog(); err == nil {
t.Fatal("planned while a watched branch has no first view")
}
if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
t.Errorf("held %+v: kept as asked, never orphaned by a first view", a)
t.Errorf("held %+v: kept as asked, never orphaned while a first view waits", a)
}
bus.refuse["KV_n_x"] = errors.New("no")
k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
@@ -687,19 +744,21 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") {
t.Errorf("health %+v: names the verdicts waiting", h)
}
k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") }
k.Catalog = func() (CatalogView, error) {
return CatalogView{}, errors.New("the catalog half has no first view of main yet")
}
k.mu.Lock()
k.status.FirstView.Answered = time.Time{}
k.mu.Unlock()
if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second {
t.Fatalf("asked again in %s (%v): on the backoff", next, err)
if err := k.PlanCatalog(); err == nil {
t.Fatal("planned with no first view")
}
// The first view unanswered is unhealthy only past the bound since the start (review round 1: health.go:41).
if h := HealthOf(k.Status(), nil, 0, at); !h.Healthy {
t.Errorf("health %+v: healthy within the bound while the first view is asked", h)
}
k.FillPass()
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") {
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "mesh-catalog.revision, has not answered: the catalog half has no first view of main yet") {
t.Errorf("health %+v: unhealthy past the bound, naming the verb", h)
}
// What waits on its consumer unread is never skipped silently (review round 1: health.go:47).
@@ -1157,24 +1216,26 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
store := k.Store.(*memStore)
var acts []string
k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
k.View = func() (FirstView, error) {
return FirstView{Revision: 8, Modules: []Accepted{{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}, nil
k.Catalog = func() (CatalogView, error) {
return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil
}
again := 0
k.Again = func() error { again++; return nil }
raw, _ := json.Marshal(Request{ID: "r9", Verb: VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"})
_, _ = store.Put(StateRequests, "r9", raw)
k.ActOnRequests()
raw, _, _, _ = store.Get(StateRequests, "r9")
var got Request
_ = json.Unmarshal(raw, &got)
if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts OBJ_m_drafts-objects]" ||
len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked mesh-delivery-indexer.indexer_bus_state again by hand") {
if got.Outcome != "the first views answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts OBJ_m_drafts-objects]" || again != 1 ||
len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked the catalog half's first views (mesh-catalog.revision) again by hand") {
t.Errorf("ended %+v, created %v, recorded %v", got, bus.created, acts)
}
}},
} {
t.Run(row.name, func(t *testing.T) {
k, bus, _, out := newKeeper(FirstView{Revision: 1})
if _, err := k.AskFirstView(0); err != nil {
k, bus, _, out := newKeeper(CatalogView{})
if err := k.PlanCatalog(); err != nil {
t.Fatal(err)
}
row.run(t, k, bus, out)
@@ -1197,7 +1258,7 @@ func TestSends(t *testing.T) {
}{
{"a seat with a verb", []string{"verbed"}, false, true},
{"a seat with an event", []string{"evented"}, false, true},
{"a seat that includes parts", []string{"included"}, false, true},
{"a seat that only includes parts (§537: resolved by the seat list, including alone counts for nothing)", []string{"included"}, false, false},
{"a seat with neither", []string{"bare"}, false, false},
{"a seat the list does not hold", []string{"unknown"}, false, false},
{"an event of its own", nil, true, true},
+4 -5
View File
@@ -1,8 +1,7 @@
// Package keeper is the bus module's keeper (novox/hq design 25 § 12.2-12.3, ADR 0595 §4-§6): it creates the bus state
// every accepted manifest asks for in `requires`, compares what the bus holds with what was asked, finds orphans and
// reads the fill of what refuses when full, and says each on the bus module's own events. It reads nothing of another
// part's state: what to create comes on the indexer's manifest-accepted, and its first view from the indexer's tool
// indexer_bus_state. Everything it touches it reaches through the interfaces in keeper.go; this file is pure.
// part's state: what to create comes from the catalog half of each watched branch (design 47 G §433, §532). Everything it touches it reaches through the interfaces in keeper.go; this file is pure.
package keeper
import (
@@ -55,14 +54,14 @@ const StateObjectsCapMiB = 16
// Sends says whether a module sends (design 47 G §344): it claims a seat whose definition has a verb or an event, or it
// declares an event of its own (emitsOwn). claims are the seats its catalog graph entry claims, defs the seat list's
// definitions by seat; a claimed seat the list does not hold counts for nothing. A definition that includes parts takes
// their verbs and events whole (seatdef.Resolve), so one counts as having some.
// definitions by seat, each as the seat list answers that one seat, resolved with the parts it includes; a claimed seat
// the list does not hold counts for nothing, and including parts alone counts for nothing (design 47 G §537).
func Sends(claims []string, emitsOwn bool, defs map[string]seatdef.Definition) bool {
if emitsOwn {
return true
}
for _, c := range claims {
if d, ok := defs[c]; ok && (len(d.Verbs) > 0 || len(d.Events) > 0 || len(d.Includes) > 0) {
if d, ok := defs[c]; ok && (len(d.Verbs) > 0 || len(d.Events) > 0) {
return true
}
}
+10 -4
View File
@@ -24,7 +24,7 @@ const (
VerbApply = "apply"
VerbRecreate = "recreate"
VerbDrop = "drop"
// VerbFirstViewAgain asks the indexer's first view at once, by hand (design 25 § 12.2), beside the backoff.
// VerbFirstViewAgain asks the catalog half's first views again at once, by hand (design 47 G §614).
VerbFirstViewAgain = "first-view-again"
)
@@ -175,13 +175,19 @@ func (k *Keeper) ActOnRequests() {
func (k *Keeper) act(r *Request) (string, error) {
if r.Verb == VerbFirstViewAgain {
_, err := k.AskFirstView(0)
err := fmt.Errorf("the keeper follows no catalog half")
if k.Again != nil {
err = k.Again()
}
if err == nil {
err = k.PlanCatalog()
}
s := k.Status().FirstView
k.record(r, fmt.Sprintf("asked %s.%s again by hand: answered %v at revision %d", s.Owner, s.Verb, err == nil, s.Revision))
k.record(r, fmt.Sprintf("asked the catalog half's first views (%s.%s) again by hand: answered %v at revision %d", s.Owner, s.Verb, err == nil, s.Revision))
if err != nil {
return "asked, not answered", err
}
return fmt.Sprintf("the first view answered at revision %d", s.Revision), nil
return fmt.Sprintf("the first views answered at revision %d", s.Revision), nil
}
if !k.Viewed() {
return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant)
+13 -3
View File
@@ -62,10 +62,17 @@
],
"invokes": [
"seat:mesh-hand-acts.hand-act",
"mesh-delivery-indexer.indexer_bus_state"
"seat:mesh-catalog.revision",
"seat:mesh-catalog.registrations",
"seat:mesh-delivery.source",
"seat:mesh-catalog.seat-definitions"
],
"consumes": [
"mesh-catalog.manifest-accepted",
"mesh-catalog.catalog-revision",
"mesh-catalog.registration-changed",
"mesh-delivery.source-moved",
"mesh-catalog.seat-definition-changed",
"mesh-catalog.seat-definition-removed",
"mesh-conditions.findings-asked"
],
"uses": [
@@ -325,7 +332,10 @@
"asks": true,
"keeps": true,
"invokes": [
"mesh-delivery-indexer.indexer_bus_state",
"seat:mesh-catalog.revision",
"seat:mesh-catalog.registrations",
"seat:mesh-delivery.source",
"seat:mesh-catalog.seat-definitions",
"seat:mesh-hand-acts.hand-act"
]
},