From 4fff214eeb8e4a92c313ef220197ff165b8dce21 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:35:16 +0100 Subject: [PATCH] Give the condition keeper the controller's condition bodies, the findings layout and the take-over key, so readers change only the seat they hear (issue 1952 S2-S4, ADR 0564 clauses 2, 3, 7-9) --- go/health/condition.go | 288 +++++++++++++++++++ go/health/condition_test.go | 145 ++++++++++ go/health/findings.go | 341 +++++++++++++++++++++++ go/health/health_test.go | 55 ++-- go/health/testdata/condition-events.json | 222 +++++++++++++++ 5 files changed, 1022 insertions(+), 29 deletions(-) create mode 100644 go/health/condition.go create mode 100644 go/health/condition_test.go create mode 100644 go/health/findings.go create mode 100644 go/health/testdata/condition-events.json diff --git a/go/health/condition.go b/go/health/condition.go new file mode 100644 index 0000000..987cf1c --- /dev/null +++ b/go/health/condition.go @@ -0,0 +1,288 @@ +package health + +// **A condition and its events, as the controller says them today** (novox/hq ADR 0564 clause 3, issue 1952): the body +// of `condition-raised`, `condition-changed` and `condition-cleared`, ported field for field from mesh-controller +// internal/conditions (condition.go, events.go, plain.go at 0fbb5ef3), so the condition keeper says them unchanged under +// the seat mesh-conditions and a reader (the router, the grants part, the healers) changes only the seat it hears. +// +// A condition is a durable fact about something the mesh owns that is wrong: raised when an observer's finding says so, +// kept with since-when, evidence and who can resolve it, said on the bus as it changes, and cleared when the finding +// goes, never by hand (ADR 0231 §14, ADR 0564 clause 2). + +import ( + "regexp" + "sort" + "strings" + "time" +) + +// Severity is how soon the operator is needed: two levels, no more. +type Severity string + +const ( + // Urgent needs the operator now. + Urgent Severity = "urgent" + // Warning needs the operator when they can. + Warning Severity = "warning" +) + +// The scopes a condition's key starts with: what kind of thing is wrong. +const ( + ScopeMachine = "machine" + ScopePlan = "plan" + ScopeCall = "call" + ScopeBuild = "build" + ScopeMerge = "merge" + ScopeProvider = "provider" + ScopeSeat = "seat" + ScopeBus = "bus" + ScopeCore = "core" + ScopeProbe = "probe" + ScopeMesh = "mesh" + // ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239). + ScopeDelivery = "delivery" + // ScopeModule is a module on a machine, by `.` (novox/hq ADR 0240). + ScopeModule = "module" +) + +// Scopes is every scope, in the order a person reads them. +var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider, + ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule} + +// Who resolves a condition. +const ( + // ResolverSelf clears on observation: the signal returns, the probe passes. + ResolverSelf = "self" + // ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy. + ResolverOperator = "operator" + // ResolverAgent is work handed to an agent. + ResolverAgent = "agent" +) + +// ResolverHealer is the resolver of a condition a registered healer works on. +func ResolverHealer(name string) string { return "healer:" + name } + +// Subject is what the condition is about: its scope, its id within the scope, and the machine it concerns when there is +// one. +type Subject struct { + Scope string `json:"scope"` + ID string `json:"id"` + Machine string `json:"machine,omitempty"` + // Also are the other machines it concerns: a consumer's, for a provider failing it. + Also []string `json:"also,omitempty"` +} + +// Evidence is one observation, as it was said. +type Evidence struct { + At time.Time `json:"at"` + Said string `json:"said"` +} + +// Tried is one healer's try at a condition, as the condition carries it in `tried`: when, what it did, what came of it, +// and which healer (`healer H3`), never a person. The keeper writes it from the healers' attempts (ADR 0564 clause 7). +type Tried struct { + At time.Time `json:"at"` + What string `json:"what"` + Outcome string `json:"outcome"` + By string `json:"by"` +} + +// Silence is a person saying they know: no messages until it ends. The condition stays open. +type Silence struct { + Until time.Time `json:"until"` + By string `json:"by"` + Why string `json:"why"` + Since time.Time `json:"since"` +} + +// Gap is a stretch in which a fault was cleared, between its clearing and its reopening. +type Gap struct { + Cleared time.Time `json:"cleared"` + Reopened time.Time `json:"reopened"` +} + +// Action is one answer the operator may give from a notification: a label, and the seat verb the operator channel calls +// with these arguments when it is chosen (novox/hq ADR 0253). Level is how much proof its answer needs (ADR 0234 §8, +// ADR 0259). +type Action struct { + Label string `json:"label"` + Verb string `json:"verb"` + Machine string `json:"machine,omitempty"` + Arguments map[string]string `json:"arguments,omitempty"` + Level string `json:"level,omitempty"` +} + +// The assurance levels an action's answer needs (novox/hq ADR 0234 §8). +const ( + LevelAcknowledge = "acknowledge" + LevelApprove = "approve" +) + +// The bounds a condition keeps, as the controller keeps them. +const ( + // KeptEvidence is how many observations a condition keeps, newest first. + KeptEvidence = 10 + // KeptTried is how many tries a condition keeps, newest last. + KeptTried = 10 + // KeptGaps is how many gaps a condition keeps. + KeptGaps = 8 + // MaxSilence is the longest a condition may be silenced at once. + MaxSilence = 7 * 24 * time.Hour + // ReopenWithin is how soon after it cleared a condition raised again is the same one again, its count increased, + // rather than news. + ReopenWithin = 10 * time.Minute +) + +// Condition is one open condition, as the keeper keeps it and its events carry it: the controller's body, field for +// field (ADR 0564 clause 3). The keeper writes Epoch 0 (Limits 10). +type Condition struct { + Key string `json:"key"` + Kind string `json:"kind"` + Subject Subject `json:"subject"` + Severity Severity `json:"severity"` + Summary string `json:"summary"` + Headline string `json:"headline"` + Explanation string `json:"explanation"` + Resolved string `json:"resolved"` + Needs string `json:"needs"` + Actions []Action `json:"actions"` + Evidence []Evidence `json:"evidence"` + // Source is the observer's row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`. + Source string `json:"source"` + Raised time.Time `json:"raised"` + LastObserved time.Time `json:"last-observed"` + // First and Gaps are the fault's earlier life: a reopening within ReopenWithin is the same fault. + First time.Time `json:"first,omitzero"` + Gaps []Gap `json:"gaps,omitempty"` + Observations int `json:"observations"` + Count int `json:"count"` + Tried []Tried `json:"tried,omitempty"` + Resolver string `json:"resolver"` + // Silenced is null when no silence is in force: said, not left out. + Silenced *Silence `json:"silenced"` + Epoch uint64 `json:"epoch"` + // LooksUnread says why Observations, LastObserved and Evidence are not known; the controller's alone. + LooksUnread string `json:"looks-unread,omitempty"` +} + +// SilencedAt says whether a person's silence is in force at a moment. +func (c Condition) SilencedAt(now time.Time) bool { + return c.Silenced != nil && now.Before(c.Silenced.Until) +} + +// Escalated says a healer tried and its budget is spent: the operator resolves it now. +func (c Condition) Escalated() bool { return c.Resolver == ResolverOperator && len(c.Tried) > 0 } + +// Began is when this fault began as the mesh knows it: its first raising, a reopening within ReopenWithin being the +// same fault again. +func (c Condition) Began() time.Time { + if !c.First.IsZero() && c.First.Before(c.Raised) { + return c.First + } + return c.Raised +} + +// ShowKeeper is the verb that shows more about a condition the keeper says (ADR 0564 Limits 9); a condition carried from +// the controller's bucket keeps the controller's (ShowController) until the lease write. +func ShowKeeper(key string) string { return "mesh-conditions.conditions key=" + key } + +// ShowController is the controller's verb that shows more about a condition it raised. +func ShowController(key string) string { return "mesh-controller.conditions key=" + key } + +// SilenceAction is the action that stops a condition's messages for a week, as the keeper offers it (ADR 0564 Limits 9: +// `mesh-conditions.silence`). An acknowledgement (ADR 0234 §8); its cause marks it as an answer. +func SilenceAction(key string) Action { + return Action{Label: "Silence for a week", Verb: "mesh-conditions.silence", Level: LevelAcknowledge, + Arguments: map[string]string{"key": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}} +} + +// CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with. +const CauseOperatorAnswer = "operator-answer" + +// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's tokens, and a `*` or `>` +// would make one a wildcard. +var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) + +// Key composes a condition's key from its parts, `..`, the controller's rule (condition.go Key): each +// token made safe for the bus, a character the bus would refuse becoming `_`, an empty token `_`. The id may hold several +// tokens joined by dots. A finding and the condition it raises have one key. +func Key(scope, id, token string) string { + var parts []string + for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) { + p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_") + if p == "" { + p = "_" + } + parts = append(parts, p) + } + return strings.Join(parts, ".") +} + +// Order sorts conditions as `conditions` says them: urgent before warning, then oldest first, then by key. +func Order(list []Condition) { + sort.SliceStable(list, func(i, j int) bool { + if list[i].Severity != list[j].Severity { + return list[i].Severity == Urgent + } + if !list[i].Raised.Equal(list[j].Raised) { + return list[i].Raised.Before(list[j].Raised) + } + return list[i].Key < list[j].Key + }) +} + +// The events a condition's life emits, under the seat that keeps it (ADR 0564 clause 3: the controller's names). +const ( + // EventRaised: a condition was raised, new or the same fault again within ReopenWithin (Change says which). + EventRaised = "condition-raised" + // EventChanged: its severity, its resolver, its silence or its words changed; not every observation. + EventChanged = "condition-changed" + // EventCleared: the finding went; the condition is removed and the transition kept in history. + EventCleared = "condition-cleared" +) + +// Events is every event a condition's life emits. +var Events = []string{EventRaised, EventChanged, EventCleared} + +// What changed, as an event's Change and a history entry say it. +const ( + ChangeRaised = "raised" + ChangeReopened = "reopened" + ChangeSeverity = "severity" + ChangeResolver = "resolver" + ChangeSilenced = "silenced" + ChangeUnsilenced = "silence-ended" + ChangeCleared = "cleared" + // ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR 0258). + ChangeWords = "words" +) + +// Event is the body of every condition event and the shape a history entry keeps: the condition itself at the top level, +// with what happened to it beside. +type Event struct { + Condition + // Event is the event's own name, so a body read without its subject still says what it is. + Event string `json:"event"` + At time.Time `json:"at"` + // Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, words, cleared. + Change string `json:"change"` + // Was is the value before, for a severity or resolver change. + Was string `json:"was,omitempty"` + // Why says why it cleared, or why it was silenced. + Why string `json:"why,omitempty"` + // Cleared is when it cleared, on a clearing. + Cleared *time.Time `json:"cleared,omitempty"` + // Show is the verb that shows more. + Show string `json:"show"` +} + +// EventFor is the event a change is said under. +func EventFor(change string) string { + switch change { + case ChangeRaised, ChangeReopened: + return EventRaised + case ChangeCleared: + return EventCleared + } + return EventChanged +} diff --git a/go/health/condition_test.go b/go/health/condition_test.go new file mode 100644 index 0000000..e62ea04 --- /dev/null +++ b/go/health/condition_test.go @@ -0,0 +1,145 @@ +package health + +import ( + "encoding/json" + "os" + "reflect" + "strings" + "testing" + "time" +) + +// ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the +// events the live controller said unchanged. testdata/condition-events.json holds five transitions of the controller's +// condition history (mesh-controller_condition-history, whose entries are the events' bodies, events.go Event), captured +// on 2026-10-29 with `mesh-controller.conditions history=true days=1`: a raising, a reopening with gaps, a resolver and a +// severity change with tried and was, and a clearing with why. Never written by hand. +func TestTheControllersConditionEventsReadUnchanged(t *testing.T) { + raw, err := os.ReadFile("testdata/condition-events.json") + if err != nil { + t.Fatal(err) + } + var captured []json.RawMessage + if err := json.Unmarshal(raw, &captured); err != nil { + t.Fatal(err) + } + changes := map[string]bool{} + for _, body := range captured { + var e Event + if err := json.Unmarshal(body, &e); err != nil { + t.Fatalf("%s: %v", body, err) + } + changes[e.Change] = true + again, err := json.Marshal(e) + if err != nil { + t.Fatal(err) + } + var want, got map[string]any + _ = json.Unmarshal(body, &want) + _ = json.Unmarshal(again, &got) + if !reflect.DeepEqual(want, got) { + t.Errorf("the controller's %s of %s reads back as\n%s\nnot\n%s", e.Change, e.Key, again, body) + } + if EventFor(e.Change) != e.Event || e.Key != Key(e.Subject.Scope, e.Subject.ID, e.Key[strings.LastIndex(e.Key, ".")+1:]) { + t.Errorf("%s: said as %s for a %s, or its key is not its subject's", e.Key, e.Event, e.Change) + } + } + for _, c := range []string{ChangeRaised, ChangeReopened, ChangeResolver, ChangeSeverity, ChangeCleared} { + if !changes[c] { + t.Errorf("no captured %s", c) + } + } + // The keeper's own body (Limits 9, 10): epoch 0, said and not left out, silenced null, and its show and silence + // naming the new seat. + c := Condition{Key: "machine.g14.silent", Actions: []Action{SilenceAction("machine.g14.silent")}} + b, _ := json.Marshal(Event{Condition: c, Event: EventRaised, Change: ChangeRaised, Show: ShowKeeper(c.Key)}) + for _, says := range []string{`"epoch":0`, `"silenced":null`, `"show":"mesh-conditions.conditions key=machine.g14.silent"`, + `"verb":"mesh-conditions.silence"`} { + if !strings.Contains(string(b), says) { + t.Errorf("the keeper's body %s does not say %s", b, says) + } + } +} + +// ADR 0564 clauses 2, 7, 8, 9 (issue 1952 S3, S4): the layouts between the observers, the healers and the keeper, and the +// take-over key the readers follow. +func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) { + at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) + good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing", + Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"}, + Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at} + for _, c := range []struct { + name string + edit func(*Finding) + fails string + }{ + {name: "a finding keyed by the controller's key rule"}, + // The key rule is the controller's (condition.go Key): a character the bus refuses becomes `_`, so the finding + // and the condition it raises share one key. + {name: "an unsafe id", edit: func(f *Finding) { + f.Subject.ID, f.Key = "keycloak.ace.mesh mail*", "provider.keycloak.ace.mesh_mail_.failing" + }}, + {name: "a key not its parts'", edit: func(f *Finding) { f.Key = "provider.keycloak.failing" }, fails: "not"}, + {name: "an unknown scope", edit: func(f *Finding) { f.Subject.Scope = "thing" }, fails: "scope"}, + {name: "no severity", edit: func(f *Finding) { f.Severity = "" }, fails: "urgent or a warning"}, + {name: "no observer", edit: func(f *Finding) { f.Observer = "" }, fails: "observer"}, + {name: "eleven observations", edit: func(f *Finding) { f.Evidence = make([]Evidence, KeptEvidence+1) }, fails: "at most"}, + } { + f := good + if c.edit != nil { + c.edit(&f) + } + b, err := EncodeFinding(f) + if c.fails != "" { + if err == nil || !strings.Contains(err.Error(), c.fails) { + t.Errorf("%s: encoded (%v), want refused saying %q", c.name, err, c.fails) + } + continue + } + back, err := DecodeFinding(b) + if err != nil || back.Key != f.Key || back.V != FindingVersion || back.Evidence == nil { + t.Errorf("%s: read back as %+v, %v", c.name, back, err) + } + } + if FindingsBucket(SelfCheckModule) != "mesh-self-check_findings" || Bucket(HealersModule, AttemptsState) != "mesh-healers_attempts" || + Bucket(KeeperModule, TakeoverState) != "mesh-conditions_takeover" { + t.Errorf("the buckets are named %s, %s", FindingsBucket(SelfCheckModule), Bucket(HealersModule, AttemptsState)) + } + + // An attempt reaches the condition's tried as the controller wrote it: by the healer, its outcome and what it said. + if tr := (Attempt{Healer: "H3", Act: "asserted the grants", Outcome: OutcomeEscalated, Said: "budget spent", Begun: at}).Tried(); tr != + (Tried{At: at, What: "asserted the grants", Outcome: "escalated: budget spent", By: "healer H3"}) { + t.Errorf("an attempt is tried as %+v", tr) + } + + // The take-over key (clause 9): absent follows the controller; the keeper's names who chose; anything else refused. + for _, c := range []struct { + raw string + found bool + seat, events string + fails bool + }{ + {found: false, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"}, + {raw: `{"v":1,"follow":"controller","by":"the operator"}`, found: true, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"}, + {raw: `{"v":1,"follow":"keeper","by":"the operator","ask":"ask-1"}`, found: true, seat: "mesh-conditions", events: "mesh-self-check.checked"}, + {raw: `{"v":1,"follow":"keeper"}`, found: true, fails: true}, + {raw: `{"v":1,"follow":"both","by":"x"}`, found: true, fails: true}, + {raw: `{"v":2,"follow":"keeper","by":"x"}`, found: true, fails: true}, + } { + got, err := DecodeTakeover([]byte(c.raw), c.found) + if c.fails != (err != nil) { + t.Errorf("%q: read as %+v, %v", c.raw, got, err) + continue + } + if c.fails { + continue + } + seat, event := got.SelfCheckEvent() + if got.Seat() != c.seat || seat+"."+event != c.events { + t.Errorf("%q follows %s and %s.%s", c.raw, got.Seat(), seat, event) + } + if b, err := EncodeTakeover(got); err != nil || !strings.Contains(string(b), `"v":1`) { + t.Errorf("%q encodes as %s, %v", c.raw, b, err) + } + } +} diff --git a/go/health/findings.go b/go/health/findings.go new file mode 100644 index 0000000..393a52f --- /dev/null +++ b/go/health/findings.go @@ -0,0 +1,341 @@ +package health + +// **What the observers write and the keeper reads** (novox/hq ADR 0564 clauses 2, 6, 7, 8 and 9; issue 1952): the +// layout of a finding, the self-check's `checked`, the healers' `healer-acted`, an act begun in `attempts` and one the +// healers would have taken, and the take-over key the readers follow. The keeper is the one writer of conditions; an +// observer writes findings in its own bucket and never another's (clause 2). + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +// The modules holding the four health seats, and the states between them, as `_` buckets. +const ( + // KeeperModule is the condition keeper, holder of mesh-conditions. + KeeperModule = "mesh-conditions" + // SelfCheckModule holds mesh-self-check; HealersModule mesh-healers; WatchmanModule mesh-watchdogs. + SelfCheckModule = "mesh-self-check" + HealersModule = "mesh-healers" + WatchmanModule = "mesh-delivery-watchman" + + // FindingsState is every observer's own state of findings, one key per finding (clause 2). + FindingsState = "findings" + // ConditionsState is the keeper's open conditions, one key each, without their looks (clause 3). + ConditionsState = "conditions" + // TakeoverState is the keeper's take-over, one key TakeoverKey (clause 9). + TakeoverState = "takeover" + // TakeoverKey is the take-over state's one key. + TakeoverKey = "switch" + // AttemptsState is the healers' acts begun, each created before it is made (clause 7); WouldState the acts they + // would have taken while the lease names a controller (clause 8), which the keeper never reads. + AttemptsState = "attempts" + WouldState = "would" +) + +// Bucket is a module's state as the bus names it, `_`. +func Bucket(module, state string) string { return module + "_" + state } + +// FindingsBucket is an observer's bucket of findings. +func FindingsBucket(observer string) string { return Bucket(observer, FindingsState) } + +// FindingVersion is the version of a finding's layout. +const FindingVersion = 1 + +// Finding is one thing an observer sees wrong now, kept as the key Key in its bucket `findings` until it no longer sees +// it (clause 2): written only when what it says changes (ADR 0289 §1, §4), deleted when it goes. A key written raises or +// changes the condition of the same key; a key deleted clears it. The keeper makes the condition from it: its words, +// severity, subject and source unchanged, its evidence kept newest first. +type Finding struct { + V int `json:"v"` + // Key is Key(Subject.Scope, Subject.ID, Token or Kind): the condition's key. + Key string `json:"key"` + Kind string `json:"kind"` + // Token is the key's last part when it is not Kind (`silent` for a machine). + Token string `json:"token,omitempty"` + Subject Subject `json:"subject"` + Severity Severity `json:"severity"` + // Summary is one line in the mesh's words; Headline, Explanation, Resolved and Needs the plain words of ADR 0253 + // and Actions the answers a notification offers. + Summary string `json:"summary"` + Headline string `json:"headline"` + Explanation string `json:"explanation"` + Resolved string `json:"resolved"` + Needs string `json:"needs,omitempty"` + Actions []Action `json:"actions,omitempty"` + // Evidence is what the observer saw, newest first, at most KeptEvidence. Detail goes here, never in Summary. + Evidence []Evidence `json:"evidence"` + // Source is the row, probe or event that saw it: `S1`, `D6`, `apply-failing`. + Source string `json:"source"` + // Resolver is who resolves it, ResolverSelf when empty. + Resolver string `json:"resolver,omitempty"` + // Observer is the module that wrote it, and Run what of its own saw it last (a self-check run's id, empty for a + // watch): the condition's looks are its finding's and its observer's last run (clause 2). + Observer string `json:"observer"` + Run string `json:"run,omitempty"` + At time.Time `json:"at"` +} + +// FindingKey is the key of a finding about scope and id of kind (the token): the condition's key rule. +func FindingKey(scope, id, token string) string { return Key(scope, id, token) } + +func (f Finding) token() string { + if f.Token != "" { + return f.Token + } + return f.Kind +} + +// Check refuses a finding the keeper could not make a condition of, naming why. +func (f Finding) Check() error { + known := false + for _, s := range Scopes { + known = known || s == f.Subject.Scope + } + switch { + case !known: + return fmt.Errorf("health: a finding's scope is one of %s, not %q", strings.Join(Scopes, ", "), f.Subject.Scope) + case strings.TrimSpace(f.Subject.ID) == "": + return fmt.Errorf("health: a %s finding names what it is about", f.Subject.Scope) + case strings.TrimSpace(f.Kind) == "": + return fmt.Errorf("health: the finding %s has no kind", f.Key) + case f.Key != Key(f.Subject.Scope, f.Subject.ID, f.token()): + return fmt.Errorf("health: the finding's key %q is not %q, its scope, id and kind", f.Key, + Key(f.Subject.Scope, f.Subject.ID, f.token())) + case f.Severity != Urgent && f.Severity != Warning: + return fmt.Errorf("health: the finding %s is urgent or a warning, not %q", f.Key, f.Severity) + case strings.TrimSpace(f.Summary) == "": + return fmt.Errorf("health: the finding %s says nothing", f.Key) + case strings.TrimSpace(f.Source) == "": + return fmt.Errorf("health: the finding %s does not say what saw it", f.Key) + case strings.TrimSpace(f.Observer) == "": + return fmt.Errorf("health: the finding %s does not name its observer", f.Key) + case len(f.Evidence) > KeptEvidence: + return fmt.Errorf("health: the finding %s keeps %d observations, at most %d", f.Key, len(f.Evidence), KeptEvidence) + } + return nil +} + +// EncodeFinding is f as JSON at FindingVersion; it refuses what Check refuses. +func EncodeFinding(f Finding) ([]byte, error) { + f.V = FindingVersion + if f.Evidence == nil { + f.Evidence = []Evidence{} + } + if err := f.Check(); err != nil { + return nil, err + } + return json.Marshal(f) +} + +// DecodeFinding reads a finding's value; it refuses another version and what Check refuses. +func DecodeFinding(data []byte) (Finding, error) { + var f Finding + if err := json.Unmarshal(data, &f); err != nil { + return Finding{}, fmt.Errorf("health: a finding is not readable: %w", err) + } + if f.V != FindingVersion { + return Finding{}, fmt.Errorf("health: a finding of version %d, and this reads %d", f.V, FindingVersion) + } + if err := f.Check(); err != nil { + return Finding{}, err + } + return f, nil +} + +// EventChecked is the self-check's event, ending every run (clause 6), with the body of the controller's +// `doctor-heartbeat`, which mesh-watcher hears in its place. +const EventChecked = "checked" + +// The verdicts of a probe in a run. +const ( + VerdictPassed = "passed" + VerdictFailed = "failed" + VerdictFailedToRun = "failed-to-run" + VerdictDeferred = "deferred" +) + +// ProbeVerdict is one probe's outcome in a run (mesh-controller doctor.go probeVerdict). +type ProbeVerdict struct { + ID string `json:"id"` + Verdict string `json:"verdict"` + Found []string `json:"found,omitempty"` + // Unconfirmed are findings one look can be wrong about, seen by this run and not the one before. + Unconfirmed []string `json:"unconfirmed,omitempty"` + Error string `json:"error,omitempty"` + Took string `json:"took,omitempty"` + // FoundCount and More are an overview's, never a checked event's. + FoundCount int `json:"found-count,omitempty"` + More string `json:"more,omitempty"` +} + +// CheckedCounts are a run's verdicts, counted. +type CheckedCounts struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + FailedToRun int `json:"failed-to-run"` + Deferred int `json:"deferred"` +} + +// Checked is one run of the self-check and the body of `checked`, the controller's doctorRun (doctor.go:230) unchanged. +type Checked struct { + Run string `json:"run"` + At time.Time `json:"at"` + Started time.Time `json:"started"` + Took string `json:"took"` + IntervalSeconds int `json:"interval-seconds"` + Counts CheckedCounts `json:"counts"` + Probes []ProbeVerdict `json:"probes"` + // Controller is the machine that ran it (the name kept for mesh-watcher, which reads it), and Why what started it. + Controller string `json:"controller"` + Why string `json:"why"` + Unsaid int `json:"unsaid,omitempty"` +} + +// EventHealerActed is the healers' event, one per act (clause 7). +const EventHealerActed = "healer-acted" + +// The outcomes of an act. +const ( + // OutcomeActing is an act begun and not yet finished: counted against the budget and the brake from its begin. + OutcomeActing = "acting" + // OutcomeActed is an act that did what it does; whether it repaired anything is the condition's clearing to say. + OutcomeActed = "acted" + // OutcomeFailed is an act that could not be done. + OutcomeFailed = "failed" + // OutcomeEscalated is a budget spent, said: the condition handed to the operator. + OutcomeEscalated = "escalated" +) + +// HealerActed is the body of `healer-acted`, the controller's (healers.go:345) unchanged; mesh-healers leaves Epoch out +// (ADR 0564 Limits 10). +type HealerActed struct { + Event string `json:"event"` + At time.Time `json:"at"` + Healer string `json:"healer"` + By string `json:"by"` + Condition string `json:"condition"` + Kind string `json:"kind"` + Act string `json:"act"` + Outcome string `json:"outcome"` + Said string `json:"said"` + Budget string `json:"budget"` + Epoch uint64 `json:"epoch,omitempty"` + Show string `json:"show"` +} + +// Attempt is one act of a healer as kept in the healers' state `attempts` (clause 7): created before it is made, so a +// holder that dies mid-act has spent it and a second instance cannot begin the same act, and finished after. The key is +// the healers' own; the keeper reads every value and writes the condition named its `tried`, resolver and escalation. +type Attempt struct { + Healer string `json:"healer"` + Condition string `json:"condition"` + Kind string `json:"kind"` + // BudgetKey is what the budget counts by: the condition's key unless the healer says another. + BudgetKey string `json:"budget-key"` + Act string `json:"act"` + Outcome string `json:"outcome"` + Said string `json:"said,omitempty"` + Begun time.Time `json:"begun"` + Finished *time.Time `json:"finished,omitempty"` +} + +// Tried is the attempt as the condition carries it in `tried`. +func (a Attempt) Tried() Tried { + outcome := a.Outcome + if a.Said != "" { + outcome += ": " + a.Said + } + return Tried{At: a.Begun, What: a.Act, Outcome: outcome, By: "healer " + a.Healer} +} + +// Would is an act mesh-healers would have taken while the controller's lease names a controller, kept in `would`: it +// counts for no budget, settle or brake and reaches no condition (clause 8). +type Would struct { + Healer string `json:"healer"` + Condition string `json:"condition"` + Kind string `json:"kind"` + Act string `json:"act"` + At time.Time `json:"at"` + // Why says why it was not taken: the lease names a controller, which heals. + Why string `json:"why"` +} + +// Which source the readers of conditions follow (clause 9). +const ( + FollowController = "controller" + FollowKeeper = "keeper" +) + +// TakeoverVersion is the version of the take-over key's value. +const TakeoverVersion = 1 + +// Takeover is the value of the keeper's key `takeover.switch` (clause 9): which source the router, the grants part and +// mesh-watcher hear, the controller's condition events and `doctor-heartbeat`, or the keeper's events and the +// self-check's `checked`. Written by the keeper alone, on the operator's warrant for a take-over and by `hand-back`. An +// absent key is FollowController. +type Takeover struct { + V int `json:"v"` + Follow string `json:"follow"` + At time.Time `json:"at"` + // By is who chose (the operator, as the warrant proves), and Why why; Ask the approve ask it was chosen on. + By string `json:"by"` + Why string `json:"why,omitempty"` + Ask string `json:"ask,omitempty"` +} + +// EncodeTakeover is t as JSON at TakeoverVersion; it refuses a source the readers could not follow. +func EncodeTakeover(t Takeover) ([]byte, error) { + t.V = TakeoverVersion + if err := t.check(); err != nil { + return nil, err + } + return json.Marshal(t) +} + +// DecodeTakeover reads the key as a getter answered it: an absent key follows the controller. +func DecodeTakeover(data []byte, found bool) (Takeover, error) { + if !found || len(data) == 0 { + return Takeover{V: TakeoverVersion, Follow: FollowController}, nil + } + var t Takeover + if err := json.Unmarshal(data, &t); err != nil { + return Takeover{}, fmt.Errorf("health: the take-over key is not readable: %w", err) + } + if t.V != TakeoverVersion { + return Takeover{}, fmt.Errorf("health: the take-over key is of version %d, and this reads %d", t.V, TakeoverVersion) + } + if err := t.check(); err != nil { + return Takeover{}, err + } + return t, nil +} + +func (t Takeover) check() error { + if t.Follow != FollowController && t.Follow != FollowKeeper { + return fmt.Errorf("health: the take-over follows %q, not %s or %s", t.Follow, FollowController, FollowKeeper) + } + if t.Follow == FollowKeeper && strings.TrimSpace(t.By) == "" { + return fmt.Errorf("health: a take-over by the keeper names who chose it") + } + return nil +} + +// Seat is the seat a reader hears condition events on while it follows t: mesh-controller or mesh-conditions. +func (t Takeover) Seat() string { + if t.Follow == FollowKeeper { + return "mesh-conditions" + } + return "mesh-controller" +} + +// SelfCheckEvent is the seat and event a reader of the self-check's run hears while it follows t: the controller's +// `doctor-heartbeat`, or mesh-self-check's `checked`. +func (t Takeover) SelfCheckEvent() (seat, event string) { + if t.Follow == FollowKeeper { + return "mesh-self-check", EventChecked + } + return "mesh-controller", "doctor-heartbeat" +} diff --git a/go/health/health_test.go b/go/health/health_test.go index b6a3531..06310f2 100644 --- a/go/health/health_test.go +++ b/go/health/health_test.go @@ -52,15 +52,6 @@ func TestAWaitIsReadWithWhatItNames(t *testing.T) { } } -func TestAnAnswerWithoutStateIsReadFromHealthy(t *testing.T) { - for raw, want := range map[string]string{`{"healthy":true,"why":"ok"}`: StateHealthy, `{"healthy":false,"why":"down"}`: StateUnhealthy} { - got, err := Read([]byte(raw)) - if err != nil || got.State != want { - t.Fatalf("%s read as %+v, %v", raw, got, err) - } - } -} - func TestAWaitThatCannotBeReadIsUnhealthyAndSaysWhy(t *testing.T) { for name, c := range map[string]struct { w []Wait @@ -88,13 +79,6 @@ func TestAWaitThatCannotBeReadIsUnhealthyAndSaysWhy(t *testing.T) { } } -func TestAnUnknownStateIsUnhealthy(t *testing.T) { - got, err := Read([]byte(`{"state":"fine","healthy":true}`)) - if err != nil || got.State != StateUnhealthy || got.Healthy { - t.Fatalf("read as %+v, %v", got, err) - } -} - func TestWaitsBesideAFaultAreSaidAndExcuseNothing(t *testing.T) { got := roundTrip(t, Unhealthy("the share is unreachable", password, Wait{Part: "p", What: "w"})) if got.State != StateUnhealthy || len(got.Waits) != 1 { @@ -111,12 +95,6 @@ func TestWaitingWithNothingIsUnhealthy(t *testing.T) { } } -func TestNotAnAnswerIsAnError(t *testing.T) { - if _, err := Read([]byte(`"yes"`)); err == nil { - t.Fatal("a string read as an answer") - } -} - func TestEveryFormOfPathOrAddressIsRefused(t *testing.T) { for _, words := range []string{ "the source (/mnt/games)", `the source "/mnt/games"`, "the folder ~/games", "the folder ./games", @@ -136,15 +114,34 @@ func TestEveryFormOfPathOrAddressIsRefused(t *testing.T) { } } -func TestHealthyStateWithHealthyFalseIsUnhealthy(t *testing.T) { - got, err := Read([]byte(`{"state":"healthy","healthy":false,"why":"x"}`)) - if err != nil || got.State != StateUnhealthy || !strings.Contains(got.Why, "contradict") { - t.Fatalf("read as %+v, %v", got, err) - } -} - func TestWaitingChecksItsWaits(t *testing.T) { if a := Waiting("it waits", Wait{Part: "p", What: "w"}); a.State != StateUnhealthy || !strings.Contains(a.Why, "neither") { t.Fatalf("Waiting with a malformed wait is %+v", a) } } + +// A raw answer is read by what it says: without state from healthy; an unknown state, or healthy with healthy false, as +// unhealthy; not an answer at all as an error. +func TestARawAnswerIsReadByWhatItSays(t *testing.T) { + for _, c := range []struct { + raw, state, says string + fails bool + }{ + {raw: `{"healthy":true,"why":"ok"}`, state: StateHealthy}, + {raw: `{"healthy":false,"why":"down"}`, state: StateUnhealthy}, + {raw: `{"state":"fine","healthy":true}`, state: StateUnhealthy}, + {raw: `{"state":"healthy","healthy":false,"why":"x"}`, state: StateUnhealthy, says: "contradict"}, + {raw: `"yes"`, fails: true}, + } { + got, err := Read([]byte(c.raw)) + if c.fails { + if err == nil { + t.Errorf("%s read as an answer", c.raw) + } + continue + } + if err != nil || got.State != c.state || (c.state == StateUnhealthy && got.Healthy) || !strings.Contains(got.Why, c.says) { + t.Errorf("%s read as %+v, %v", c.raw, got, err) + } + } +} diff --git a/go/health/testdata/condition-events.json b/go/health/testdata/condition-events.json new file mode 100644 index 0000000..2b25e7e --- /dev/null +++ b/go/health/testdata/condition-events.json @@ -0,0 +1,222 @@ +[ + { + "actions": null, + "at": "2026-10-29T08:41:18.10610281Z", + "change": "raised", + "count": 1, + "epoch": 395230, + "event": "condition-raised", + "evidence": [ + { + "at": "2026-10-29T08:41:18.10610281Z", + "said": "silent for 2m0s" + } + ], + "explanation": "Nothing for you to do. The mesh has not heard from g14 for several minutes. A machine asleep or away is normal.", + "headline": "g14 is not answering", + "key": "machine.g14.silent", + "kind": "silent", + "last-observed": "2026-10-29T08:41:18.10610281Z", + "needs": "", + "observations": 1, + "raised": "2026-10-29T08:41:18.10610281Z", + "resolved": "g14 answers again", + "resolver": "self", + "severity": "warning", + "show": "mesh-controller.conditions key=machine.g14.silent", + "silenced": null, + "source": "S1", + "subject": { + "id": "g14", + "machine": "g14", + "scope": "machine" + }, + "summary": "g14 has not been heard from since 2026-10-29 08:38 UTC (bound 3m0s)" + }, + { + "actions": null, + "at": "2026-10-28T20:59:53.719121257Z", + "change": "reopened", + "count": 2, + "epoch": 387175, + "event": "condition-raised", + "evidence": [ + { + "at": "2026-10-28T20:59:53.719121257Z", + "said": "70 pending ([70 on EVENTS]), last taken 2026-10-28T20:57:25Z" + } + ], + "explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.", + "first": "2026-10-28T20:54:53.510073517Z", + "gaps": [ + { + "cleared": "2026-10-28T20:56:52.499154879Z", + "reopened": "2026-10-28T20:59:53.719121257Z" + } + ], + "headline": "The controller stopped listening", + "key": "core.controller.deaf", + "kind": "controller-deaf", + "last-observed": "2026-10-28T20:59:53.719121257Z", + "needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.", + "observations": 1, + "raised": "2026-10-28T20:59:53.719121257Z", + "resolved": "The controller listens again", + "resolver": "self", + "severity": "urgent", + "show": "mesh-controller.conditions key=core.controller.deaf", + "silenced": null, + "source": "S4", + "subject": { + "id": "controller", + "machine": "novox", + "scope": "core" + }, + "summary": "the controller's event loop has taken nothing for 2m0s while its consumers hold 70 message(s): reports, builds and merges are not being acted on" + }, + { + "actions": null, + "at": "2026-10-29T18:31:48.308721256Z", + "change": "resolver", + "count": 1, + "epoch": 401364, + "event": "condition-changed", + "evidence": [ + { + "at": "2026-10-29T18:31:33.979579505Z", + "said": "no answer for node-files from g14 to the bus's discovery" + } + ], + "explanation": "Nothing for you to do. Whatever serves node-files on g14 does not answer, so its tools do nothing there. A machine asleep is normal.", + "headline": "Node-files on g14 is not answering", + "key": "seat.node-files.g14.silent", + "kind": "holder-silent", + "last-observed": "2026-10-29T18:31:33.979579Z", + "needs": "", + "observations": 1, + "raised": "2026-10-29T18:31:33.979579505Z", + "resolved": "Node-files on g14 answers again", + "resolver": "healer:H3", + "severity": "warning", + "show": "mesh-controller.conditions key=seat.node-files.g14.silent", + "silenced": null, + "source": "D3", + "subject": { + "id": "node-files.g14", + "machine": "g14", + "scope": "seat" + }, + "summary": "node-files's holder on g14 does not answer the bus: its verbs reach nothing there", + "tried": [ + { + "at": "2026-10-29T18:31:48.308721256Z", + "by": "healer H3", + "outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)", + "what": "asserted the bus's streams, consumers and seat workers again" + } + ], + "was": "self", + "why": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)" + }, + { + "actions": null, + "at": "2026-10-28T20:56:51.686413523Z", + "change": "severity", + "count": 1, + "epoch": 387175, + "event": "condition-changed", + "evidence": [ + { + "at": "2026-10-28T20:55:13.087322533Z", + "said": "no answer for mesh-placement from ace to the bus's discovery" + }, + { + "at": "2026-10-28T20:50:11.256396519Z", + "said": "no answer for mesh-placement from ace to the bus's discovery" + } + ], + "explanation": "Needs you: the mesh tried to repair this and could not; read the details to decide what to do. Whatever serves mesh-placement on ace does not answer, so its tools do nothing there. A machine asleep is normal.", + "headline": "Mesh-placement on ace is not answering", + "key": "seat.mesh-placement.ace.silent", + "kind": "holder-silent", + "last-observed": "2026-10-28T20:55:13.087322Z", + "needs": "the mesh tried to repair this and could not; read the details to decide what to do.", + "observations": 2, + "raised": "2026-10-28T20:50:11.256396519Z", + "resolved": "Mesh-placement on ace answers again", + "resolver": "operator", + "severity": "urgent", + "show": "mesh-controller.conditions key=seat.mesh-placement.ace.silent", + "silenced": null, + "source": "D3", + "subject": { + "id": "mesh-placement.ace", + "machine": "ace", + "scope": "seat" + }, + "summary": "mesh-placement's holder on ace does not answer the bus: its verbs reach nothing there", + "tried": [ + { + "at": "2026-10-28T20:50:22.435049179Z", + "by": "healer H3", + "outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)", + "what": "asserted the bus's streams, consumers and seat workers again" + }, + { + "at": "2026-10-28T20:56:51.686413523Z", + "by": "healer H3", + "outcome": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)", + "what": "handed to the operator: nothing in the mesh will repair it now" + } + ], + "was": "warning", + "why": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)" + }, + { + "actions": null, + "at": "2026-10-28T21:18:48.630162912Z", + "change": "cleared", + "cleared": "2026-10-28T21:18:48.630162912Z", + "count": 2, + "epoch": 387463, + "event": "condition-cleared", + "evidence": [ + { + "at": "2026-10-28T21:18:16.589232939Z", + "said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z" + }, + { + "at": "2026-10-28T21:17:46.778840937Z", + "said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z" + } + ], + "explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.", + "first": "2026-10-28T21:13:16.626696927Z", + "gaps": [ + { + "cleared": "2026-10-28T21:13:47.735189608Z", + "reopened": "2026-10-28T21:17:46.778840937Z" + } + ], + "headline": "The controller stopped listening", + "key": "core.controller.deaf", + "kind": "controller-deaf", + "last-observed": "2026-10-28T21:18:16.589232Z", + "needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.", + "observations": 2, + "raised": "2026-10-28T21:17:46.778840937Z", + "resolved": "The controller listens again", + "resolver": "self", + "severity": "urgent", + "show": "mesh-controller.conditions key=core.controller.deaf", + "silenced": null, + "source": "S4", + "subject": { + "id": "controller", + "machine": "novox", + "scope": "core" + }, + "summary": "the controller's event loop has taken nothing for 3m0s while its consumers hold 82 message(s): reports, builds and merges are not being acted on", + "why": "S4 no longer observes it" + } +]