package keeper import ( "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "sort" "strings" "sync" "time" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/membership" "git.novox.be/novox/mesh-sdk/go/seatdef" ) // Module is the bus module's own name: the publisher of its events and its conditions. const Module = "nats" // The keeper's own states (old `state` sections, ADR 0595 §10): what it last heard per module, and the persons' // requests its tools hand it. const ( StateAsked = "asked" StateRequests = "requests" // StatePlans holds, per module, the bus-state entries as last planned and their digest (design 47 G §538): what a // verdict's entries are compared with, never the manifest's digest, which changes with any field. StatePlans = "plans" // KeyStatus is the keeper's own status in StateAsked, read by its health tool: a key no module can have, since a // module's name holds no underscore. KeyStatus = "_keeper" ) // The seat mesh-broker's events (ADR 0595 §6): on the seat's subjects, and kept on the bus module's own stream (ADR 0620 §2). const ( EventCreated = "created" EventRefused = "refused" EventDrifted = "drifted" EventDriftCleared = "drift-cleared" EventFull = "full" EventFullCleared = "full-cleared" EventOrphaned = "orphaned" EventRecreated = "recreated" EventDropped = "dropped" ) // Event is the body of every bus module event, in the field names of the seat mesh-broker's definition (its message // `said`, mesh-contracts go/seats/meshbroker Said): the entry (module, local name, storage type, template, options, // configuration), its bus name, the settings asked and held by bus object, its fill, the snapshot a recreate or drop // took, and why (refused, drifted). Beside the definition's fields: binding, the names a module receives (a ledger's // history, state, indexes and snapshot, which one bus name cannot carry), and revision, the indexer's revision of the // verdict it acted on; both asked of the definition. // // Written here until this module takes mesh-contracts' generated type; the swap changes no byte of the body. type Event struct { V int `json:"v"` Module string `json:"module"` Local string `json:"local-name"` Storage string `json:"storage-type,omitempty"` Template string `json:"template,omitempty"` Options map[string]json.RawMessage `json:"options,omitempty"` Configuration map[string]json.RawMessage `json:"configuration,omitempty"` BusName string `json:"bus-name"` Binding membership.BusBinding `json:"binding"` Asked map[string]Settings `json:"asked,omitempty"` Held map[string]Settings `json:"held,omitempty"` Fill *EventFill `json:"fill,omitempty"` Snapshot string `json:"snapshot,omitempty"` Why string `json:"why,omitempty"` Revision uint64 `json:"revision"` } // EventFill is an entry's fill as an event says it: its messages, its bytes and its cap. type EventFill struct { Count uint64 `json:"count"` Bytes uint64 `json:"bytes"` Cap int64 `json:"cap"` } // Accepted is one module's entry as the catalog half's watched branches give it (design 47 G §344, §433, §532): the // module, its manifest's digest, the catalog revision of the branch it was read from, and its bus-state entries whole. type Accepted struct { Module string `json:"module"` Digest string `json:"digest"` Revision uint64 `json:"revision"` BusState []catalogue.BusStateEntry `json:"bus-state"` } // CatalogModule is one module of the catalog half for the bus module (design 47 G §344, §532, §533, §547): what its // watched branches' newest revisions ask of the bus, the seats they claim, whether one declares an event of its own, // whether it is a foundation part (it declares an old `objects` section `sent-objects`), and the caps of the old // sections it declares, in bytes. type CatalogModule struct { Accepted Claims []string `json:"claims,omitempty"` EmitsOwn bool `json:"emits-own,omitempty"` Foundation bool `json:"foundation,omitempty"` OldBytes int64 `json:"old-bytes,omitempty"` } // Drift is a local name a later watched branch asks with other settings than the branch it is planned from (design 47 // G §532): said as the bus module's drifted, naming the branch, and planned from nothing of that branch. type Drift struct { Module string `json:"module"` Local string `json:"local-name"` Branch string `json:"branch"` // From is the branch whose settings are planned. From string `json:"from"` } // CatalogView is the catalog half as the bus module plans from it: every module in a watched branch's newest revision, // and the drifts between branches. type CatalogView struct { Modules []CatalogModule Drifts []Drift } // Planned is the value of a module's key in StatePlans: the entries as planned, their digest and the revision of the verdict. type Planned struct { V int `json:"v"` Module string `json:"module"` Digest string `json:"digest"` Revision uint64 `json:"revision"` Entries []catalogue.BusStateEntry `json:"entries"` At time.Time `json:"at"` } // EntriesDigest is the digest of bus-state entries alone (design 47 G §538): equal entries have an equal digest whatever // else a manifest changed. func EntriesDigest(entries []catalogue.BusStateEntry) string { raw, _ := json.Marshal(entries) sum := sha256.Sum256(raw) return "sha256:" + hex.EncodeToString(sum[:]) } // Asked is what the keeper holds of one module, the value of its key in StateAsked: what its newest accepted manifest // asks for, and every entry the keeper made for it and the bus holds, by local name. type Asked struct { V int `json:"v"` Module string `json:"module"` Digest string `json:"digest,omitempty"` Revision uint64 `json:"revision"` Entries []catalogue.BusStateEntry `json:"entries"` Made map[string]Made `json:"made,omitempty"` // Drifts are the local names a later watched branch asks with other settings, as last said (design 47 G §532). Drifts map[string]Drift `json:"drifts,omitempty"` At time.Time `json:"at"` } // Made is one entry as the keeper made it: its plan and what stands of it now. type Made struct { Plan Plan `json:"plan"` // Created says every object of it is on the bus and its `created` was said. Created bool `json:"created"` // Refused is why it could not be made, Waiting why it waits to be made again (a refusal the bus may lift, such as a // grant not composed yet). Refused string `json:"refused,omitempty"` Waiting string `json:"waiting,omitempty"` // Drift is how what the bus holds differs from what was asked, in a way that does not grow. Drift string `json:"drift,omitempty"` // Full is whether its fill is past the raise mark, Fill the last fill read, in percent of its cap. Full bool `json:"full,omitempty"` Fill int64 `json:"fill,omitempty"` // Orphaned says no manifest asks for it any more: it goes only by drop. Orphaned bool `json:"orphaned,omitempty"` } // Status is the keeper's own state, KeyStatus: what its health tool reads. type Status struct { Started time.Time `json:"started"` // Pass is when the last fill pass ended. Pass time.Time `json:"pass"` // Limits are the bus's storage limits as last read (design 47 G §547): what `list` shows against the sum of the caps. Limits Limits `json:"limits,omitempty"` // OldBytes is the sum of the caps of every old section the catalog graph's entries declare, in bytes, as last planned // (design 47 G §547, §591): counted in the sum of the bus's caps beside the entries made. OldBytes int64 `json:"old-sections-bytes,omitempty"` // FirstView is the catalog half's first views (design 47 G §614): when last planned from, when answered and at which // revision, or what it waits for. FirstView FirstViewStatus `json:"first-view"` // Heard is when the catalog half last moved, and Unanswered since when an event has waited unhandled. Heard time.Time `json:"heard,omitempty"` Unanswered time.Time `json:"unanswered,omitempty"` // Unsaid are the events the bus did not take, kept with their id and the subjects still to say them on, and // UnsaidConditions the conditions the condition keeper did not take, by key: each said again at every fill pass, and // named by the keeper's health until it is (design 25 § 12.2: a failed say is never only logged). Unsaid []Saying `json:"unsaid,omitempty"` UnsaidConditions map[string]oc.Condition `json:"unsaid-conditions,omitempty"` } // Saying is one event of the seat mesh-broker as the keeper says it: its name, its one event id (the same on every // subject and every try, so the bus keeps it once), its body, and Left, the subjects it is still to be said on (nil: // every one). Emit says it on Left, and leaves in Left what the bus did not take. type Saying struct { Event string `json:"event"` ID string `json:"id"` Body Event `json:"body"` Left []string `json:"left,omitempty"` // Said are the subjects this try said it on, in order: the seat's subject first. Said []string `json:"said,omitempty"` Error string `json:"error,omitempty"` At time.Time `json:"at"` // Restored, when set, is the body of the event snapshot-restored, said on its own subject (request and act) and not on // the entry events' ones. Restored *SnapshotRestored `json:"restored,omitempty"` } // FirstViewStatus is what the keeper holds of its first view. type FirstViewStatus struct { Owner string `json:"owner"` Verb string `json:"verb"` Asked time.Time `json:"asked,omitempty"` Answered time.Time `json:"answered,omitempty"` Revision uint64 `json:"revision"` Error string `json:"error,omitempty"` Next time.Time `json:"next,omitempty"` } // The first view's seat and verb: the catalog half of each watched branch, read through the seat mesh-catalog's // `revision` (design 47 G §433, §614). const ( FirstViewOwner = "mesh-catalog" FirstViewVerb = "revision" ) // Bus is the bus as the keeper's own account reaches it. type Bus interface { // Held reads the object's stream: its settings, whether it is there, and its fill (bytes and messages). Held(ctx context.Context, o Object) (Settings, Fill, bool, error) Create(ctx context.Context, o Object) error // Update brings a held object to o's settings: only a difference that grows (Grows), or a recreate's rebuild. Update(ctx context.Context, o Object) error Delete(ctx context.Context, o Object) error // Snapshot takes a snapshot of the streams and says where it is kept. Snapshot(ctx context.Context, streams []string) (string, error) // Restricted says whether any of the streams is one ADR 0436 §3 lets no other account hear (its metadata says it), so // a snapshot of them is never handed out (design 47 G §681). Restricted(ctx context.Context, streams []string) (bool, error) // SnapshotStreams are the streams a snapshot file holds; Live says whether a stream of that name exists. SnapshotStreams(path string) ([]string, error) Live(ctx context.Context, name string) (bool, error) // Restore puts every stream of the snapshot file back under its own name, and DeleteStream removes one by its name (the // keeper's own partial stream of a restore that did not end). Restore(ctx context.Context, path string) error DeleteStream(ctx context.Context, name string) error // Limits reads the bus's storage limits: the keeper's account's and the server's (design 47 G §547). Limits(ctx context.Context) (Limits, error) } // Limits are the bus's storage limits in bytes; 0 is no limit (an unlimited account, or a server that sets none). type Limits struct { AccountBytes int64 `json:"account-bytes,omitempty"` ServerBytes int64 `json:"server-bytes,omitempty"` } // CapSum is the sum, in bytes, of the caps of every object the entries made hold, but for the entry of skipModule and // skipLocal (design 47 G §547): what the bus's limits are read against. An entry never made, or refused, holds none. func CapSum(held map[string]Asked, skipModule, skipLocal string) int64 { var sum int64 for module, a := range held { for local, m := range a.Made { if (module == skipModule && local == skipLocal) || !(m.Created || m.Orphaned) { continue } for _, o := range m.Plan.Objects { sum += o.Settings.MaxBytes } } } return sum } // capRoom refuses a plan that adds or raises a cap when the sum of the bus's caps would pass the account's limit, // naming both (design 47 G §547); caps already made stay as they are. With no account limit nothing is refused. func (k *Keeper) capRoom(ctx context.Context, held *Asked, m *Made) error { lim, err := k.Bus.Limits(ctx) if err != nil { return fmt.Errorf("%w: the bus's storage limits cannot be read: %v", ErrRetry, err) } k.setStatus(func(s *Status) { s.Limits = lim }) if lim.AccountBytes <= 0 { return nil } all := k.Held() all[held.Module] = *held sum := CapSum(all, held.Module, m.Plan.Local) + k.Status().OldBytes for _, o := range m.Plan.Objects { sum += o.Settings.MaxBytes } if sum > lim.AccountBytes { return fmt.Errorf("the plan takes the sum of the bus's caps to %d MiB, past the account's storage limit of %d MiB", sum/MiB, lim.AccountBytes/MiB) } return nil } // Fill is how much one object holds: its bytes and messages. type Fill struct { Bytes uint64 `json:"bytes"` Messages uint64 `json:"messages"` } // ErrRetry marks a refusal the bus may lift without a change of the manifest (a create the keeper's account is not yet // granted): the entry waits and is made again on the backoff, never said refused. var ErrRetry = errors.New("refused for now") // Store is the keeper's own state. type Store interface { Get(state, key string) ([]byte, uint64, bool, error) Put(state, key string, value []byte) (uint64, error) Delete(state, key string) error Keys(state string) ([]string, error) // Create writes a key only when it has none: ErrExists when it has one (a compare-and-set at revision 0). Create(state, key string, value []byte) (uint64, error) } // ErrExists is a Create of a key that has a value. var ErrExists = errors.New("the key has a value already") // Keeper is the bus module's keeper. type Keeper struct { Bus Bus Store Store // Emit says one of the bus module's events (Saying); Say raises or clears a condition through the condition keeper. Emit func(s *Saying) error Say func(oc.Condition) error // Catalog reads the catalog half of every watched branch as it stands (design 47 G §433, §532): an error while a // branch has no first view yet, when the keeper plans nothing. Seats are the seat list's definitions, each resolved // with the parts it includes (§537). Again asks every first view of the catalog half again, by hand. Catalog func() (CatalogView, error) Seats func() map[string]seatdef.Definition Again func() error // Act records a hand act through the seat mesh-hand-acts; Warrant reads the router's record of an ask. Act func(what, why, cause string) error Warrant func(askID string) (*oc.Ask, *oc.Warrant, error) Now func() time.Time Logf func(string, ...any) // Raise and Clear are the fill marks in percent (ADR 0595 §6): settings of the bus module, 75 and 70 by default. Raise, Clear int64 // Files are the snapshot files (the disk when nil); SnapshotsCap the cap of their directory in bytes and // SnapshotKeepDays how long a snapshot is kept: the settings `snapshots-cap` (20 GiB) and `snapshot-keep-days` (90) (design 47 G §681). Files Files // Finding is the writer of the finding snapshots-filling. Finding Findings SnapshotsCap int64 SnapshotKeepDays int mu sync.Mutex status Status viewed bool // unwritten says the condition that the status cannot be written, with words unsaid, is raised. unwritten bool // filling is whether the snapshots directory was last judged past 80 % of its cap, fillJudged whether it is to be judged afresh (the keeper's start). filling, fillUnjudged bool // restoring is the snapshot restore that runs, at most one. restoring *restoreRun } func (k *Keeper) now() time.Time { if k.Now == nil { return time.Now().UTC() } return k.Now().UTC() } func (k *Keeper) logf(format string, a ...any) { if k.Logf != nil { k.Logf(format, a...) } } func (k *Keeper) marks() (int64, int64) { raise, clear := k.Raise, k.Clear if raise <= 0 { raise = 75 } if clear <= 0 || clear >= raise { clear = raise - 5 } return raise, clear } // Viewed is whether the first view was answered: before it the keeper creates and orphans nothing. func (k *Keeper) Viewed() bool { k.mu.Lock() defer k.mu.Unlock() return k.viewed } // Status is the keeper's status now. func (k *Keeper) Status() Status { k.mu.Lock() defer k.mu.Unlock() return k.status } func (k *Keeper) setStatus(f func(*Status)) { k.mu.Lock() if k.status.Started.IsZero() { k.status.Started = k.now() } k.status.FirstView.Owner, k.status.FirstView.Verb = FirstViewOwner, FirstViewVerb f(&k.status) s := k.status k.mu.Unlock() if k.Store != nil { raw, _ := json.Marshal(s) _, err := k.Store.Put(StateAsked, KeyStatus, raw) if err != nil { k.logf("the keeper's status cannot be written to its own %s: %v", StateAsked, err) } k.statusWritten(s, err) } } // statusWritten raises, once, the keeper's own condition when its status cannot be written while it holds words the bus // did not take: those are then kept only in memory, and a restart loses them (review round 2, seams note 3 and health // note 4: named, not only a log). The next write that stands clears it. func (k *Keeper) statusWritten(s Status, err error) { if k.Say == nil { return } n := len(s.Unsaid) + len(s.UnsaidConditions) k.mu.Lock() raised := k.unwritten switch { case err != nil && n > 0 && !raised: k.unwritten = true case err == nil && raised: k.unwritten = false default: k.mu.Unlock() return } k.mu.Unlock() c := oc.Condition{Key: ConditionKey(conditionUnwritten, Module, "keeper"), Subject: oc.ConditionSubject{Scope: "module", ID: Module}} if err != nil { needs := "a look at why the keeper cannot write its own state " + StateAsked c.State, c.Severity, c.Headline, c.Needs = occ.StateRaised, occ.SeverityWarning, "The bus module may lose words it could not say", needs c.Summary = fmt.Sprintf("the keeper holds %d word(s) the bus did not take and cannot write its status (%v): they are kept only in memory, and a restart of the keeper loses them", n, err) c.Explanation = occ.NeedsYou + needs + ". " + c.Summary c.Resolved = "cleared by the bus module once its status is written again" } else { c.State, c.Resolved = occ.StateCleared, "the keeper's status is written again, with every word it still has to say" } if e := k.Say(c); e != nil { k.logf("the condition %s (%s) could not be said: %v", c.Key, c.State, e) } } // PlanCatalog plans from the catalog half as it stands (design 47 G §344, §433, §532, §537, §538): each module in a // watched branch's newest revision is applied (Heard), a module of the new form that sends gets its sent store, a drift // between branches is said once, and a module in no watched branch's newest revision has its plans key deleted, its // entries then orphans. Before every watched branch has its first view it plans nothing, and says what it waits for in // its status; the first plan checks what is held. func (k *Keeper) PlanCatalog() error { at := k.now() view, err := k.Catalog() if err != nil { k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Error = at, err.Error() }) return err } k.mu.Lock() first := !k.viewed k.viewed = true k.mu.Unlock() var revision uint64 var old int64 for _, m := range view.Modules { revision = max(revision, m.Revision) old += m.OldBytes } k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Revision, s.FirstView.Error, s.FirstView.Next = at, revision, "", time.Time{} if s.FirstView.Answered.IsZero() { s.FirstView.Answered = at } s.OldBytes = old }) var defs map[string]seatdef.Definition if k.Seats != nil { defs = k.Seats() } listed := map[string]bool{} for _, m := range view.Modules { listed[m.Module] = true k.Heard(m.Accepted) if !m.Foundation && Sends(m.Claims, m.EmitsOwn, defs) { k.Sends(m.Module) } } k.drifts(view.Drifts) gone := map[string]bool{} for module := range k.Held() { gone[module] = !listed[module] } if keys, err := k.Store.Keys(StatePlans); err == nil { for _, module := range keys { gone[module] = !listed[module] } } for _, module := range sortedKeys(gone) { if !gone[module] { continue } if _, ok := k.planned(module); ok { k.Heard(Accepted{Module: module, Revision: revision}) } if err := k.Store.Delete(StatePlans, module); err != nil { k.logf("the plan of %s cannot be removed from its own %s: %v", module, StatePlans, err) } } if first { k.CheckAll() } return nil } // drifts says each drift once (design 47 G §532): kept in the module's key, said again only when the branch or the // settings planned change. func (k *Keeper) drifts(ds []Drift) { by := map[string]map[string]Drift{} for _, d := range ds { if by[d.Module] == nil { by[d.Module] = map[string]Drift{} } by[d.Module][d.Local] = d } for module, a := range k.Held() { if by[module] == nil && len(a.Drifts) > 0 { by[module] = map[string]Drift{} } } for _, module := range sortedKeys(by) { held, _, found := k.asked(module) if !found { held = Asked{V: 1, Module: module, At: k.now()} } now := by[module] changed := len(now) != len(held.Drifts) for local, d := range now { if held.Drifts[local] != d { changed = true var p Plan for _, e := range held.Entries { if e.Name == local { p, _ = PlanOf(module, e) } } p.Module, p.Local = module, local k.emit(EventDrifted, p, held.Revision, fmt.Sprintf("the branch %s asks %s of %s with other settings than the branch %s, whose are planned: nothing of %s is planned", d.Branch, local, module, d.From, d.Branch), nil, nil) } } if !changed { continue } held.Drifts = nil if len(now) > 0 { held.Drifts = now } if found || len(now) > 0 { k.write(held) } } } // Heard applies one module's entry of the catalog half: it plans again only when its bus-state entries differ from those // it last planned (design 47 G §538), never on the manifest's digest, which changes with any field. Each entry asked is // made where the bus lacks it; each entry made that is no longer asked is an orphan. An entry that asks nothing ("this // module asks nothing now") ends the module's key once nothing it made is held. func (k *Keeper) Heard(a Accepted) { k.setStatus(func(s *Status) { s.Heard = k.now() }) if !k.Viewed() { k.logf("the entry of %s is heard before the catalog half's first views: nothing is planned until they are in", a.Module) return } held, _, found := k.asked(a.Module) if !found { held = Asked{V: 1, Module: a.Module} } if held.Made == nil { held.Made = map[string]Made{} } digest := EntriesDigest(a.BusState) if p, ok := k.planned(a.Module); ok && found && p.Digest == digest { if held.Digest != a.Digest || held.Revision != a.Revision { held.Digest, held.Revision, held.At = a.Digest, a.Revision, k.now() k.write(held) } return } held.Digest, held.Entries, held.Revision, held.At = a.Digest, a.BusState, a.Revision, k.now() defer k.writePlan(a.Module, digest, held.Revision, a.BusState) asked := map[string]bool{} for _, e := range a.BusState { asked[e.Name] = true k.makeEntry(&held, e) } for _, local := range sortedKeys(held.Made) { m := held.Made[local] if asked[local] || m.Orphaned || local == SentLocal { // The sent store is no manifest's ask: it is made by Sends and goes only by drop (§59, §344). continue } if !m.Created { // Never made: nothing on the bus to orphan. delete(held.Made, local) k.clear(conditionRefused, a.Module, local) continue } m.Orphaned = true held.Made[local] = m k.emit(EventOrphaned, m.Plan, held.Revision, "no watched branch's newest revision of "+a.Module+" asks for it now", nil, nil) k.raise(conditionOrphaned, a.Module, local, occ.SeverityWarning, "Kept data no manifest asks for", "the operator's drop of it, or a manifest asking for it again", fmt.Sprintf("%s of %s is on the bus and no watched branch's newest revision asks for it: it goes only by the bus module's drop, on your warrant (nats_drop), or a manifest asking for it again keeps it", BusName(a.Module, local), a.Module)) } k.write(held) } // Sends makes module's sent store where the bus lacks it (§59, §344): called when the catalog half and the seat list // say the module serves a verb or emits an event, and again on the change that gives a holder its first verb or event. // A module that no longer sends keeps its store: an object in it expires by age, and the store goes only by drop. func (k *Keeper) Sends(module string) { held, _, found := k.asked(module) if !found { held = Asked{V: 1, Module: module, At: k.now()} } if held.Made == nil { held.Made = map[string]Made{} } m := held.Made[SentLocal] m.Plan, m.Orphaned = SentPlanOf(module), false k.ensure(&held, &m) held.Made[SentLocal] = m k.write(held) } // makeEntry makes one asked entry where the bus lacks it, compares it where it holds it, and says what it did. func (k *Keeper) makeEntry(held *Asked, e catalogue.BusStateEntry) { module := held.Module plan, err := PlanOf(module, e) m := held.Made[e.Name] if err != nil { if m.Refused != err.Error() { k.emit(EventRefused, Plan{Module: module, Local: e.Name, Storage: e.Storage, Template: e.Template}, held.Revision, err.Error(), nil, nil) k.raise(conditionRefused, module, e.Name, occ.SeverityWarning, "The bus module refused to create kept data", "a manifest that asks for it in a form the bus module makes", fmt.Sprintf("%s of %s was not created: %v. The module gets no binding for it and fails at its start, naming it", e.Name, module, err)) } m.Refused, m.Waiting = err.Error(), "" held.Made[e.Name] = m return } wasOrphan := m.Orphaned m.Plan, m.Orphaned = plan, false if wasOrphan { // Asked again: kept, and its binding said again with created (design 25 § 12.3). m.Created = false k.clear(conditionOrphaned, module, e.Name) } k.ensure(held, &m) held.Made[e.Name] = m } // ensure brings one planned entry onto the bus: each object the bus lacks created, a held one compared. A difference // that only grows is applied; any other is a drift, said and never fixed quietly. func (k *Keeper) ensure(held *Asked, m *Made) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() asked, holds := map[string]Settings{}, map[string]Settings{} var drifts []string // madeLater is an object newly made for an entry that was made whole before (a state's `-objects`, design 47 G // §58): its binding gained a name, so `created` is said again. madeLater := false for _, o := range m.Plan.Objects { asked[o.Name] = o.Settings s, _, found, err := k.Bus.Held(ctx, o) if err != nil { m.Waiting = fmt.Sprintf("%s cannot be read: %v", o.Stream(), err) return } if !found { if err := k.capRoom(ctx, held, m); err != nil { if errors.Is(err, ErrRetry) { m.Waiting = err.Error() return } m.Refused, m.Waiting = fmt.Sprintf("%s: %v", o.Stream(), err), "" k.emit(EventRefused, m.Plan, held.Revision, m.Refused, asked, nil) k.raise(conditionRefused, held.Module, m.Plan.Local, occ.SeverityWarning, "The bus module refused to create kept data", "a smaller cap, or more storage for the bus's account", fmt.Sprintf("%s of %s was not created: %s. The module gets no binding for it and fails at its start, naming it", m.Plan.Local, held.Module, m.Refused)) return } if err := k.Bus.Create(ctx, o); err != nil { if errors.Is(err, ErrRetry) { m.Waiting = fmt.Sprintf("%s cannot be created yet: %v", o.Stream(), err) return } m.Refused, m.Waiting = fmt.Sprintf("%s: %v", o.Stream(), err), "" k.emit(EventRefused, m.Plan, held.Revision, m.Refused, asked, nil) k.raise(conditionRefused, held.Module, m.Plan.Local, occ.SeverityWarning, "The bus module refused to create kept data", "a look at why the bus refused it", fmt.Sprintf("%s of %s was not created: %s. The module gets no binding for it and fails at its start, naming it", m.Plan.Local, held.Module, m.Refused)) return } holds[o.Name] = o.Settings madeLater = madeLater || m.Created continue } holds[o.Name] = s if Differs(o.Settings, s) == "" { continue } if Grows(o.Settings, s) { if o.Settings.MaxBytes > s.MaxBytes { if err := k.capRoom(ctx, held, m); err != nil { m.Waiting = err.Error() return } } if err := k.Bus.Update(ctx, o); err != nil { m.Waiting = fmt.Sprintf("%s's larger settings cannot be applied yet: %v", o.Stream(), err) return } holds[o.Name] = o.Settings continue } drifts = append(drifts, o.Stream()+": "+Differs(o.Settings, s)) } wasRefused := m.Refused != "" m.Waiting, m.Refused = "", "" if wasRefused { k.clear(conditionRefused, held.Module, m.Plan.Local) } if !m.Created || madeLater { // Marked created only once `created` is said whole: the grants part binds on it, so one the bus did not take is // said again on the backoff (Waiting), never marked made and forgotten. if err := k.sayNow(EventCreated, m.Plan, held.Revision, "", asked, holds); err != nil { m.Waiting = fmt.Sprintf("its created could not be said yet: %v", err) return } m.Created = true } drift := strings.Join(drifts, "; ") switch { case drift != "" && drift != m.Drift: m.Drift = drift k.emit(EventDrifted, m.Plan, held.Revision, drift, asked, holds) k.raise(conditionDrifted, held.Module, m.Plan.Local, occ.SeverityWarning, "Kept data differs from what was asked", "a decision: change the manifest back, or the bus module's recreate on your warrant", fmt.Sprintf("%s of %s differs from what its manifest asks, in a way that does not only grow: %s", m.Plan.Local, held.Module, drift)) case drift == "" && m.Drift != "": m.Drift = "" k.emit(EventDriftCleared, m.Plan, held.Revision, "", asked, holds) k.clear(conditionDrifted, held.Module, m.Plan.Local) } } // CheckAll compares every entry the keeper holds with the bus, making what is missing (design 25 § 12.2's check at // start) and making again what waits; it answers what it found, one line per entry. func (k *Keeper) CheckAll() []CheckLine { var out []CheckLine keys, err := k.Store.Keys(StateAsked) if err != nil { return []CheckLine{{Why: "the keeper's own " + StateAsked + " cannot be read: " + err.Error()}} } for _, module := range keys { if module == KeyStatus { continue } held, _, found := k.asked(module) if !found { continue } for _, local := range sortedKeys(held.Made) { m := held.Made[local] if m.Orphaned || m.Plan.Local == "" { out = append(out, lineOf(module, local, m)) continue } for _, e := range held.Entries { if e.Name == local { k.makeEntry(&held, e) } } out = append(out, lineOf(module, local, held.Made[local])) } k.write(held) } return out } // CheckLine is one entry as check answers it. type CheckLine struct { Module string `json:"module,omitempty"` Local string `json:"local,omitempty"` Bus string `json:"bus,omitempty"` Created bool `json:"created"` Refused string `json:"refused,omitempty"` Waiting string `json:"waiting,omitempty"` Drift string `json:"drift,omitempty"` Orphaned bool `json:"orphaned,omitempty"` Full bool `json:"full,omitempty"` Fill int64 `json:"fill-percent,omitempty"` Why string `json:"why,omitempty"` } func lineOf(module, local string, m Made) CheckLine { return CheckLine{Module: module, Local: local, Bus: BusName(module, local), Created: m.Created, Refused: m.Refused, Waiting: m.Waiting, Drift: m.Drift, Orphaned: m.Orphaned, Full: m.Full, Fill: m.Fill} } // Waiting is every entry that waits to be made again, ` : `: what the backoff retries. func (k *Keeper) Waiting() []string { var out []string keys, _ := k.Store.Keys(StateAsked) for _, module := range keys { held, _, found := k.asked(module) if !found { continue } for _, local := range sortedKeys(held.Made) { if w := held.Made[local].Waiting; w != "" { out = append(out, module+" "+local+": "+w) } } } return out } // The kinds of condition the keeper raises, each naming the module and the entry (design 25 § 12.3). const ( conditionFull = "full" conditionDrifted = "drifted" conditionOrphaned = "orphaned" conditionRefused = "refused" // conditionUnwritten is the keeper's own: its status, with words unsaid, cannot be written. conditionUnwritten = "status-unwritten" ) // ConditionKey is the key of one of the keeper's conditions: `nats.._`. func ConditionKey(kind, module, local string) string { return Module + "." + kind + "." + BusName(module, local) } func (k *Keeper) raise(kind, module, local, severity, headline, needs, explanation string) { if k.Say == nil { return } c := oc.Condition{Key: ConditionKey(kind, module, local), Subject: oc.ConditionSubject{Scope: "module", ID: module}, State: occ.StateRaised, Severity: severity, Headline: headline, Needs: needs, Summary: explanation, Explanation: occ.NeedsYou + needs + ". " + explanation, Resolved: "cleared by the bus module once it no longer holds"} k.sayCondition(c) } func (k *Keeper) clear(kind, module, local string) { if k.Say == nil { return } c := oc.Condition{Key: ConditionKey(kind, module, local), Subject: oc.ConditionSubject{Scope: "module", ID: module}, State: occ.StateCleared} k.sayCondition(c) } // sayCondition says c; one the condition keeper did not take is kept in the keeper's status, replacing an older word on // its key, and said again at the next fill pass. func (k *Keeper) sayCondition(c oc.Condition) { err := k.Say(c) k.setStatus(func(s *Status) { if err == nil { delete(s.UnsaidConditions, c.Key) return } if s.UnsaidConditions == nil { s.UnsaidConditions = map[string]oc.Condition{} } s.UnsaidConditions[c.Key] = c }) if err != nil { k.logf("the condition %s (%s) could not be said, said again at the next fill pass: %v", c.Key, c.State, err) } } // SayUnsaid says again every event and condition the bus did not take (the fill pass calls it): an event on the // subjects it is still to be said on, with its one id. func (k *Keeper) SayUnsaid() { s := k.Status() var still []Saying for _, u := range s.Unsaid { u := u if err := k.Emit(&u); err != nil { u.Error = err.Error() still = append(still, u) k.logf("the event %s of %s %s (%s) could not be said again: %v", u.Event, u.Body.Module, u.Body.Local, u.ID, err) } } for _, key := range sortedKeys(s.UnsaidConditions) { k.sayCondition(s.UnsaidConditions[key]) } k.setStatus(func(st *Status) { // Kept: what was not said now, and what was added meanwhile. added := st.Unsaid[min(len(s.Unsaid), len(st.Unsaid)):] st.Unsaid = append(still, added...) }) } // emit says one event; one the bus did not take whole is kept in the keeper's status and said again, on the subjects // left, at the next fill pass. func (k *Keeper) emit(event string, p Plan, revision uint64, why string, asked, held map[string]Settings) { if k.Emit == nil { return } s := k.saying(event, p, revision, why, asked, held) if err := k.Emit(&s); err != nil { s.Error = err.Error() k.setStatus(func(st *Status) { st.Unsaid = append(st.Unsaid, s) }) k.logf("the event %s of %s %s could not be said, said again at the next fill pass: %v", event, p.Module, p.Local, err) } } // sayNow says one event and answers an error when the bus took none of it, so its caller says it again itself. One said // on the seat's subject and not on its history is said: what is left is kept with its one id and said again at the next // fill pass on the subjects left, never said again with a new id (review round 2, health note 3). func (k *Keeper) sayNow(event string, p Plan, revision uint64, why string, asked, held map[string]Settings) error { if k.Emit == nil { return nil } s := k.saying(event, p, revision, why, asked, held) err := k.Emit(&s) if err == nil { return nil } if len(s.Said) > 0 { s.Error = err.Error() k.setStatus(func(st *Status) { st.Unsaid = append(st.Unsaid, s) }) k.logf("the event %s of %s %s was said on %v and not on %v: the rest said again at the next fill pass: %v", event, p.Module, p.Local, s.Said, s.Left, err) return nil } k.logf("the event %s of %s %s could not be said: %v", event, p.Module, p.Local, err) return err } func (k *Keeper) saying(event string, p Plan, revision uint64, why string, asked, held map[string]Settings) Saying { body := Event{V: 1, Module: p.Module, Local: p.Local, Storage: p.Storage, Template: p.Template, Options: p.Entry.Options, Configuration: p.Entry.Configuration, BusName: BusName(p.Module, p.Local), Binding: p.Binding, Asked: asked, Held: held, Why: why, Revision: revision} if event == EventDropped { // A dropped entry's names only: the binding to remove. body.Binding.Types = nil } return Saying{Event: event, ID: EventID(), Body: body, At: k.now()} } // EventID is a new event id: sixteen random bytes in hex. func EventID() string { var b [16]byte _, _ = rand.Read(b[:]) return hex.EncodeToString(b[:]) } func (k *Keeper) planned(module string) (Planned, bool) { raw, _, found, err := k.Store.Get(StatePlans, module) if err != nil || !found { return Planned{}, false } var p Planned return p, json.Unmarshal(raw, &p) == nil } // writePlan keeps the entries a module was last planned for and their digest (§538). func (k *Keeper) writePlan(module, digest string, revision uint64, entries []catalogue.BusStateEntry) { raw, _ := json.Marshal(Planned{V: 1, Module: module, Digest: digest, Revision: revision, Entries: entries, At: k.now()}) if _, err := k.Store.Put(StatePlans, module, raw); err != nil { k.logf("the plan of %s cannot be written to its own %s: %v", module, StatePlans, err) } } func (k *Keeper) asked(module string) (Asked, uint64, bool) { raw, rev, found, err := k.Store.Get(StateAsked, module) if err != nil || !found { return Asked{}, 0, false } var a Asked if json.Unmarshal(raw, &a) != nil { return Asked{}, 0, false } return a, rev, true } // write keeps a module's key, or removes it when its manifest asks for nothing and nothing it made is held: a key goes // only so, never because a module is missing from a first view. func (k *Keeper) write(a Asked) { if len(a.Entries) == 0 && len(a.Made) == 0 && len(a.Drifts) == 0 { if err := k.Store.Delete(StateAsked, a.Module); err != nil { k.logf("the key of %s cannot be removed from its own %s: %v", a.Module, StateAsked, err) } _ = k.Store.Delete(StatePlans, a.Module) return } raw, _ := json.Marshal(a) if _, err := k.Store.Put(StateAsked, a.Module, raw); err != nil { k.logf("the key of %s cannot be written to its own %s: %v", a.Module, StateAsked, err) } } // Held is every module the keeper holds, by name: what list, show and orphans answer from. func (k *Keeper) Held() map[string]Asked { out := map[string]Asked{} keys, _ := k.Store.Keys(StateAsked) for _, module := range keys { if module == KeyStatus { continue } if a, _, found := k.asked(module); found { out[module] = a } } return out } func sortedKeys[V any](m map[string]V) []string { out := make([]string, 0, len(m)) for key := range m { out = append(out, key) } sort.Strings(out) return out }