diff --git a/go/health/condition.go b/go/health/condition.go deleted file mode 100644 index e5076b4..0000000 --- a/go/health/condition.go +++ /dev/null @@ -1,284 +0,0 @@ -package health - -// **A condition and its events, as the controller says them today** (novox/hq ADR 0564 clause 3, issue 1952): the body -// of `condition-raised`, `condition-changed` and `condition-cleared`, ported field for field from mesh-controller -// internal/conditions (condition.go, events.go, plain.go at 0fbb5ef3), so the condition keeper says them unchanged under -// the seat mesh-conditions and a reader (the router, the grants part, the healers) changes only the seat it hears. -// -// A condition is a durable fact about something the mesh owns that is wrong: raised when an observer's finding says so, -// kept with since-when, evidence and who can resolve it, said on the bus as it changes, and cleared when the finding -// goes, never by hand (ADR 0231 §14, ADR 0564 clause 2). - -import ( - "regexp" - "sort" - "strings" - "time" -) - -// Severity is how soon the operator is needed: two levels, no more. -type Severity string - -const ( - // Urgent needs the operator now. - Urgent Severity = "urgent" - // Warning needs the operator when they can. - Warning Severity = "warning" -) - -// The scopes a condition's key starts with: what kind of thing is wrong. -const ( - ScopeMachine = "machine" - ScopePlan = "plan" - ScopeCall = "call" - ScopeBuild = "build" - ScopeMerge = "merge" - ScopeProvider = "provider" - ScopeSeat = "seat" - ScopeBus = "bus" - ScopeCore = "core" - ScopeProbe = "probe" - ScopeMesh = "mesh" - // ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239). - ScopeDelivery = "delivery" - // ScopeModule is a module on a machine, by `.` (novox/hq ADR 0240). - ScopeModule = "module" -) - -// Scopes is every scope, in the order a person reads them. -var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider, - ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule} - -// Who resolves a condition. -const ( - // ResolverSelf clears on observation: the signal returns, the probe passes. - ResolverSelf = "self" - // ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy. - ResolverOperator = "operator" - // ResolverAgent is work handed to an agent. - ResolverAgent = "agent" -) - -// ResolverHealer is the resolver of a condition a registered healer works on. -func ResolverHealer(name string) string { return "healer:" + name } - -// Subject is what the condition is about: its scope, its id within the scope, and the machine it concerns when there is -// one. -type Subject struct { - Scope string `json:"scope"` - ID string `json:"id"` - Machine string `json:"machine,omitempty"` - // Also are the other machines it concerns: a consumer's, for a provider failing it. - Also []string `json:"also,omitempty"` -} - -// Evidence is one observation, as it was said. -type Evidence struct { - At time.Time `json:"at"` - Said string `json:"said"` -} - -// Tried is one healer's try at a condition, as the condition carries it in `tried`: when, what it did, what came of it, -// and which healer (`healer H3`), never a person. The keeper writes it from the healers' attempts (ADR 0564 clause 7). -type Tried struct { - At time.Time `json:"at"` - What string `json:"what"` - Outcome string `json:"outcome"` - By string `json:"by"` -} - -// Silence is a person saying they know: no messages until it ends. The condition stays open. -type Silence struct { - Until time.Time `json:"until"` - By string `json:"by"` - Why string `json:"why"` - Since time.Time `json:"since"` -} - -// Gap is a stretch in which a fault was cleared, between its clearing and its reopening. -type Gap struct { - Cleared time.Time `json:"cleared"` - Reopened time.Time `json:"reopened"` -} - -// Action is one answer the operator may give from a notification: a label, and the seat verb the operator channel calls -// with these arguments when it is chosen (novox/hq ADR 0253). Level is how much proof its answer needs (ADR 0234 §8, -// ADR 0259). -type Action struct { - Label string `json:"label"` - Verb string `json:"verb"` - Machine string `json:"machine,omitempty"` - Arguments map[string]string `json:"arguments,omitempty"` - Level string `json:"level,omitempty"` -} - -// The assurance levels an action's answer needs (novox/hq ADR 0234 §8). -const ( - LevelAcknowledge = "acknowledge" - LevelApprove = "approve" -) - -// The bounds a condition keeps, as the controller keeps them. -const ( - // KeptEvidence is how many observations a condition keeps, newest first. - KeptEvidence = 10 - // KeptTried is how many tries a condition keeps, newest last. - KeptTried = 10 - // KeptGaps is how many gaps a condition keeps. - KeptGaps = 8 - // MaxSilence is the longest a condition may be silenced at once. - MaxSilence = 7 * 24 * time.Hour - // ReopenWithin is how soon after it cleared a condition raised again is the same one again, its count increased, - // rather than news. - ReopenWithin = 10 * time.Minute -) - -// Condition is one open condition, as the keeper keeps it and its events carry it: the controller's body, field for -// field (ADR 0564 clause 3). The keeper writes Epoch 0 (Limits 10). -type Condition struct { - Key string `json:"key"` - Kind string `json:"kind"` - Subject Subject `json:"subject"` - Severity Severity `json:"severity"` - Summary string `json:"summary"` - Headline string `json:"headline"` - Explanation string `json:"explanation"` - Resolved string `json:"resolved"` - Needs string `json:"needs"` - Actions []Action `json:"actions"` - Evidence []Evidence `json:"evidence"` - // Source is the observer's row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`. - Source string `json:"source"` - Raised time.Time `json:"raised"` - LastObserved time.Time `json:"last-observed"` - // First and Gaps are the fault's earlier life: a reopening within ReopenWithin is the same fault. - First time.Time `json:"first,omitzero"` - Gaps []Gap `json:"gaps,omitempty"` - Observations int `json:"observations"` - Count int `json:"count"` - Tried []Tried `json:"tried,omitempty"` - Resolver string `json:"resolver"` - // Silenced is null when no silence is in force: said, not left out. - Silenced *Silence `json:"silenced"` - Epoch uint64 `json:"epoch"` - // LooksUnread says why Observations, LastObserved and Evidence are not known; the controller's alone. - LooksUnread string `json:"looks-unread,omitempty"` -} - -// SilencedAt says whether a person's silence is in force at a moment. -func (c Condition) SilencedAt(now time.Time) bool { - return c.Silenced != nil && now.Before(c.Silenced.Until) -} - -// Escalated says a healer tried and its budget is spent: the operator resolves it now. -func (c Condition) Escalated() bool { return c.Resolver == ResolverOperator && len(c.Tried) > 0 } - -// Began is when this fault began as the mesh knows it: its first raising, a reopening within ReopenWithin being the -// same fault again. -func (c Condition) Began() time.Time { - if !c.First.IsZero() && c.First.Before(c.Raised) { - return c.First - } - return c.Raised -} - -// ShowKeeper is the verb that shows more about a condition the keeper says (ADR 0564 Limits 9). -func ShowKeeper(key string) string { return "mesh-conditions.conditions key=" + key } - -// SilenceAction is the action that stops a condition's messages for a week, as the keeper offers it (ADR 0564 Limits 9: -// `mesh-conditions.silence`). An acknowledgement (ADR 0234 §8); its cause marks it as an answer. -func SilenceAction(key string) Action { - return Action{Label: "Silence for a week", Verb: "mesh-conditions.silence", Level: LevelAcknowledge, - Arguments: map[string]string{"key": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}} -} - -// CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with. -const CauseOperatorAnswer = "operator-answer" - -// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's tokens, and a `*` or `>` -// would make one a wildcard. -var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) - -// Key composes a condition's key from its parts, `..`, the controller's rule (condition.go Key): each -// token made safe for the bus, a character the bus would refuse becoming `_`, an empty token `_`. The id may hold several -// tokens joined by dots. A finding and the condition it raises have one key. -func Key(scope, id, token string) string { - var parts []string - for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) { - p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_") - if p == "" { - p = "_" - } - parts = append(parts, p) - } - return strings.Join(parts, ".") -} - -// Order sorts conditions as `conditions` says them: urgent before warning, then oldest first, then by key. -func Order(list []Condition) { - sort.SliceStable(list, func(i, j int) bool { - if list[i].Severity != list[j].Severity { - return list[i].Severity == Urgent - } - if !list[i].Raised.Equal(list[j].Raised) { - return list[i].Raised.Before(list[j].Raised) - } - return list[i].Key < list[j].Key - }) -} - -// The events a condition's life emits, under the seat that keeps it (ADR 0564 clause 3: the controller's names). -const ( - // EventRaised: a condition was raised, new or the same fault again within ReopenWithin (Change says which). - EventRaised = "condition-raised" - // EventChanged: its severity, its resolver, its silence or its words changed; not every observation. - EventChanged = "condition-changed" - // EventCleared: the finding went; the condition is removed and the transition kept in history. - EventCleared = "condition-cleared" -) - -// Events is every event a condition's life emits. -var Events = []string{EventRaised, EventChanged, EventCleared} - -// What changed, as an event's Change and a history entry say it. -const ( - ChangeRaised = "raised" - ChangeReopened = "reopened" - ChangeSeverity = "severity" - ChangeResolver = "resolver" - ChangeSilenced = "silenced" - ChangeUnsilenced = "silence-ended" - ChangeCleared = "cleared" - // ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR 0258). - ChangeWords = "words" -) - -// Event is the body of every condition event and the shape a history entry keeps: the condition itself at the top level, -// with what happened to it beside. -type Event struct { - Condition - // Event is the event's own name, so a body read without its subject still says what it is. - Event string `json:"event"` - At time.Time `json:"at"` - // Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, words, cleared. - Change string `json:"change"` - // Was is the value before, for a severity or resolver change. - Was string `json:"was,omitempty"` - // Why says why it cleared, or why it was silenced. - Why string `json:"why,omitempty"` - // Cleared is when it cleared, on a clearing. - Cleared *time.Time `json:"cleared,omitempty"` - // Show is the verb that shows more. - Show string `json:"show"` -} - -// EventFor is the event a change is said under. -func EventFor(change string) string { - switch change { - case ChangeRaised, ChangeReopened: - return EventRaised - case ChangeCleared: - return EventCleared - } - return EventChanged -} diff --git a/go/health/condition_test.go b/go/health/condition_test.go deleted file mode 100644 index 05269d9..0000000 --- a/go/health/condition_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package health - -import ( - "encoding/json" - "os" - "reflect" - "strings" - "testing" - "time" -) - -// ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the -// events the live controller said unchanged. testdata/condition-events.json holds five transitions of the controller's -// condition history (mesh-controller_condition-history, whose entries are the events' bodies, events.go Event), captured -// on 2026-10-29 with `mesh-controller.conditions history=true days=1`: a raising, a reopening with gaps, a resolver and a -// severity change with tried and was, and a clearing with why. Never written by hand. -func TestTheControllersConditionEventsReadUnchanged(t *testing.T) { - raw, err := os.ReadFile("testdata/condition-events.json") - if err != nil { - t.Fatal(err) - } - var captured []json.RawMessage - if err := json.Unmarshal(raw, &captured); err != nil { - t.Fatal(err) - } - changes := map[string]bool{} - for _, body := range captured { - var e Event - if err := json.Unmarshal(body, &e); err != nil { - t.Fatalf("%s: %v", body, err) - } - changes[e.Change] = true - again, err := json.Marshal(e) - if err != nil { - t.Fatal(err) - } - var want, got map[string]any - _ = json.Unmarshal(body, &want) - _ = json.Unmarshal(again, &got) - if !reflect.DeepEqual(want, got) { - t.Errorf("the controller's %s of %s reads back as\n%s\nnot\n%s", e.Change, e.Key, again, body) - } - if EventFor(e.Change) != e.Event || e.Key != Key(e.Subject.Scope, e.Subject.ID, e.Key[strings.LastIndex(e.Key, ".")+1:]) { - t.Errorf("%s: said as %s for a %s, or its key is not its subject's", e.Key, e.Event, e.Change) - } - } - for _, c := range []string{ChangeRaised, ChangeReopened, ChangeResolver, ChangeSeverity, ChangeCleared} { - if !changes[c] { - t.Errorf("no captured %s", c) - } - } - // The keeper's own body (Limits 9, 10): epoch 0, said and not left out, silenced null, and its show and silence - // naming the new seat. - c := Condition{Key: "machine.g14.silent", Actions: []Action{SilenceAction("machine.g14.silent")}} - b, _ := json.Marshal(Event{Condition: c, Event: EventRaised, Change: ChangeRaised, Show: ShowKeeper(c.Key)}) - for _, says := range []string{`"epoch":0`, `"silenced":null`, `"show":"mesh-conditions.conditions key=machine.g14.silent"`, - `"verb":"mesh-conditions.silence"`} { - if !strings.Contains(string(b), says) { - t.Errorf("the keeper's body %s does not say %s", b, says) - } - } -} - -// ADR 0564 clauses 2, 7, 8, 9 (issue 1952 S3, S4): the layouts between the observers, the healers and the keeper, and the -// take-over key the readers follow. -func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) { - at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) - good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Ref: "failing", - Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"}, - Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at} - for _, c := range []struct { - name string - edit func(*Finding) - fails string - }{ - {name: "a finding keyed by the controller's key rule"}, - // The key rule is the controller's (condition.go Key): a character the bus refuses becomes `_`, so the finding - // and the condition it raises share one key. - {name: "an unsafe id", edit: func(f *Finding) { - f.Subject.ID, f.Key = "keycloak.ace.mesh mail*", "provider.keycloak.ace.mesh_mail_.failing" - }}, - {name: "a key not its parts'", edit: func(f *Finding) { f.Key = "provider.keycloak.failing" }, fails: "not"}, - {name: "an unknown scope", edit: func(f *Finding) { f.Subject.Scope = "thing" }, fails: "scope"}, - {name: "no severity", edit: func(f *Finding) { f.Severity = "" }, fails: "urgent or a warning"}, - {name: "no observer", edit: func(f *Finding) { f.Observer = "" }, fails: "observer"}, - {name: "eleven observations", edit: func(f *Finding) { f.Evidence = make([]Evidence, KeptEvidence+1) }, fails: "at most"}, - } { - f := good - if c.edit != nil { - c.edit(&f) - } - b, err := EncodeFinding(f) - if c.fails != "" { - if err == nil || !strings.Contains(err.Error(), c.fails) { - t.Errorf("%s: encoded (%v), want refused saying %q", c.name, err, c.fails) - } - continue - } - back, err := DecodeFinding(b) - if err != nil || back.Key != f.Key || back.V != FindingVersion || back.Evidence == nil { - t.Errorf("%s: read back as %+v, %v", c.name, back, err) - } - // The tool runner refuses a state value with a field named like a credential (mesh-tools bus/state.go - // credentialEndings), so a finding names none, or no observer could write it (issue 1952). - if field := credentialNamed(t, b); field != "" { - t.Errorf("%s: the finding carries the field %q, which the runtime refuses in state: %s", c.name, field, b) - } - } - // A finding written under the old name `token` is still read, its key's last part kept. - if old, err := DecodeFinding([]byte(`{"v":1,"key":"provider.keycloak.ace.mesh-mail.failing","kind":"provider-failing",` + - `"token":"failing","subject":{"scope":"provider","id":"keycloak.ace.mesh-mail"},"severity":"warning","summary":"s",` + - `"source":"S8","observer":"mesh-delivery-watchman"}`)); err != nil || old.Ref != "failing" { - t.Errorf("a finding under the old name read as %+v, %v", old, err) - } - if FindingsBucket(SelfCheckModule) != "mesh-self-check_findings" || Bucket(HealersModule, AttemptsState) != "mesh-healers_attempts" || - KeeperSeat != KeeperModule || SelfCheckSeat != SelfCheckModule { - t.Errorf("the buckets are named %s, %s", FindingsBucket(SelfCheckModule), Bucket(HealersModule, AttemptsState)) - } - - // An attempt reaches the condition's tried as the controller wrote it: by the healer, its outcome and what it said. - if tr := (Attempt{Healer: "H3", Act: "asserted the grants", Outcome: OutcomeEscalated, Said: "budget spent", Begun: at}).Tried(); tr != - (Tried{At: at, What: "asserted the grants", Outcome: "escalated: budget spent", By: "healer H3"}) { - t.Errorf("an attempt is tried as %+v", tr) - } -} - -// credentialNamed is the first field of a JSON value whose name ends as mesh-tools' credentialEndings do, or "". -func credentialNamed(t *testing.T, raw []byte) string { - t.Helper() - var v any - if err := json.Unmarshal(raw, &v); err != nil { - t.Fatal(err) - } - var walk func(any) string - walk = func(v any) string { - switch x := v.(type) { - case map[string]any: - for k, inner := range x { - norm := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(k)) - for _, end := range []string{"password", "passwd", "secret", "token", "credential", "credentials", - "authorization", "apikey", "privatekey", "accesskey", "cookie"} { - if strings.HasSuffix(norm, end) { - return k - } - } - if found := walk(inner); found != "" { - return found - } - } - case []any: - for _, inner := range x { - if found := walk(inner); found != "" { - return found - } - } - } - return "" - } - return walk(v) -} diff --git a/go/health/findings.go b/go/health/findings.go deleted file mode 100644 index 2ae46c3..0000000 --- a/go/health/findings.go +++ /dev/null @@ -1,277 +0,0 @@ -package health - -// **What the observers write and the keeper reads** (novox/hq ADR 0564 clauses 2, 6, 7, 8 and 9; issue 1952): the -// layout of a finding, the self-check's `checked`, the healers' `healer-acted`, an act begun in `attempts` and one the -// healers would have taken, and the take-over key the readers follow. The keeper is the one writer of conditions; an -// observer writes findings in its own bucket and never another's (clause 2). - -import ( - "encoding/json" - "fmt" - "strings" - "time" -) - -// The modules holding the four health seats, and the states between them, as `_` buckets. -const ( - // KeeperModule is the condition keeper, holder of mesh-conditions. - KeeperModule = "mesh-conditions" - // SelfCheckModule holds mesh-self-check; HealersModule mesh-healers; WatchmanModule mesh-watchdogs. - SelfCheckModule = "mesh-self-check" - HealersModule = "mesh-healers" - WatchmanModule = "mesh-delivery-watchman" - - // FindingsState is every observer's own state of findings, one key per finding (clause 2). - FindingsState = "findings" - // ConditionsState is the keeper's open conditions, one key each, without their looks (clause 3). - ConditionsState = "conditions" - // AttemptsState is the healers' acts begun, each created before it is made (clause 7); WouldState the acts they - // would have taken while the lease names a controller (clause 8), which the keeper never reads. - AttemptsState = "attempts" - WouldState = "would" -) - -// Bucket is a module's state as the bus names it, `_`. -func Bucket(module, state string) string { return module + "_" + state } - -// FindingsBucket is an observer's bucket of findings. -func FindingsBucket(observer string) string { return Bucket(observer, FindingsState) } - -// FindingVersion is the version of a finding's layout. -const FindingVersion = 1 - -// Finding is one thing an observer sees wrong now, kept as the key Key in its bucket `findings` until it no longer sees -// it (clause 2): written only when what it says changes (ADR 0289 §1, §4), deleted when it goes. A key written raises or -// changes the condition of the same key; a key deleted clears it. The keeper makes the condition from it: its words, -// severity, subject and source unchanged, its evidence kept newest first. -type Finding struct { - V int `json:"v"` - // Key is Key(Subject.Scope, Subject.ID, Ref or Kind): the condition's key. - Key string `json:"key"` - Kind string `json:"kind"` - // Ref is the key's last part when it is not Kind (`silent` for a machine). It is not called `token`: the tool - // runner refuses any state field named like a credential, so a finding with one could not be written (issue 1952). - Ref string `json:"ref,omitempty"` - Subject Subject `json:"subject"` - Severity Severity `json:"severity"` - // Summary is one line in the mesh's words; Headline, Explanation, Resolved and Needs the plain words of ADR 0253 - // and Actions the answers a notification offers. - Summary string `json:"summary"` - Headline string `json:"headline"` - Explanation string `json:"explanation"` - Resolved string `json:"resolved"` - Needs string `json:"needs,omitempty"` - Actions []Action `json:"actions,omitempty"` - // Evidence is what the observer saw, newest first, at most KeptEvidence. Detail goes here, never in Summary. - Evidence []Evidence `json:"evidence"` - // Source is the row, probe or event that saw it: `S1`, `D6`, `apply-failing`. - Source string `json:"source"` - // Resolver is who resolves it, ResolverSelf when empty. - Resolver string `json:"resolver,omitempty"` - // Observer is the module that wrote it, and Run what of its own saw it last (a self-check run's id, empty for a - // watch): the condition's looks are its finding's and its observer's last run (clause 2). - Observer string `json:"observer"` - Run string `json:"run,omitempty"` - At time.Time `json:"at"` -} - -// FindingKey is the key of a finding about scope and id of kind (the token): the condition's key rule. -func FindingKey(scope, id, token string) string { return Key(scope, id, token) } - -func (f Finding) token() string { - if f.Ref != "" { - return f.Ref - } - return f.Kind -} - -// Check refuses a finding the keeper could not make a condition of, naming why. -func (f Finding) Check() error { - known := false - for _, s := range Scopes { - known = known || s == f.Subject.Scope - } - switch { - case !known: - return fmt.Errorf("health: a finding's scope is one of %s, not %q", strings.Join(Scopes, ", "), f.Subject.Scope) - case strings.TrimSpace(f.Subject.ID) == "": - return fmt.Errorf("health: a %s finding names what it is about", f.Subject.Scope) - case strings.TrimSpace(f.Kind) == "": - return fmt.Errorf("health: the finding %s has no kind", f.Key) - case f.Key != Key(f.Subject.Scope, f.Subject.ID, f.token()): - return fmt.Errorf("health: the finding's key %q is not %q, its scope, id and kind", f.Key, - Key(f.Subject.Scope, f.Subject.ID, f.token())) - case f.Severity != Urgent && f.Severity != Warning: - return fmt.Errorf("health: the finding %s is urgent or a warning, not %q", f.Key, f.Severity) - case strings.TrimSpace(f.Summary) == "": - return fmt.Errorf("health: the finding %s says nothing", f.Key) - case strings.TrimSpace(f.Source) == "": - return fmt.Errorf("health: the finding %s does not say what saw it", f.Key) - case strings.TrimSpace(f.Observer) == "": - return fmt.Errorf("health: the finding %s does not name its observer", f.Key) - case len(f.Evidence) > KeptEvidence: - return fmt.Errorf("health: the finding %s keeps %d observations, at most %d", f.Key, len(f.Evidence), KeptEvidence) - } - return nil -} - -// EncodeFinding is f as JSON at FindingVersion; it refuses what Check refuses. -func EncodeFinding(f Finding) ([]byte, error) { - f.V = FindingVersion - if f.Evidence == nil { - f.Evidence = []Evidence{} - } - if err := f.Check(); err != nil { - return nil, err - } - return json.Marshal(f) -} - -// DecodeFinding reads a finding's value; it refuses another version and what Check refuses. -func DecodeFinding(data []byte) (Finding, error) { - var f Finding - if err := json.Unmarshal(data, &f); err != nil { - return Finding{}, fmt.Errorf("health: a finding is not readable: %w", err) - } - if f.Ref == "" { - // A finding written before the field was named `ref` (issue 1952) is read by its old name. - var old struct { - Token string `json:"token"` - } - _ = json.Unmarshal(data, &old) - f.Ref = old.Token - } - if f.V != FindingVersion { - return Finding{}, fmt.Errorf("health: a finding of version %d, and this reads %d", f.V, FindingVersion) - } - if err := f.Check(); err != nil { - return Finding{}, err - } - return f, nil -} - -// EventChecked is the self-check's event, ending every run (clause 6), with the body of the controller's -// `doctor-heartbeat`, which mesh-watcher hears in its place. -const EventChecked = "checked" - -// The verdicts of a probe in a run. -const ( - VerdictPassed = "passed" - VerdictFailed = "failed" - VerdictFailedToRun = "failed-to-run" - VerdictDeferred = "deferred" -) - -// ProbeVerdict is one probe's outcome in a run (mesh-controller doctor.go probeVerdict). -type ProbeVerdict struct { - ID string `json:"id"` - Verdict string `json:"verdict"` - Found []string `json:"found,omitempty"` - // Unconfirmed are findings one look can be wrong about, seen by this run and not the one before. - Unconfirmed []string `json:"unconfirmed,omitempty"` - Error string `json:"error,omitempty"` - Took string `json:"took,omitempty"` - // FoundCount and More are an overview's, never a checked event's. - FoundCount int `json:"found-count,omitempty"` - More string `json:"more,omitempty"` -} - -// CheckedCounts are a run's verdicts, counted. -type CheckedCounts struct { - Passed int `json:"passed"` - Failed int `json:"failed"` - FailedToRun int `json:"failed-to-run"` - Deferred int `json:"deferred"` -} - -// Checked is one run of the self-check and the body of `checked`, the controller's doctorRun (doctor.go:230) unchanged. -type Checked struct { - Run string `json:"run"` - At time.Time `json:"at"` - Started time.Time `json:"started"` - Took string `json:"took"` - IntervalSeconds int `json:"interval-seconds"` - Counts CheckedCounts `json:"counts"` - Probes []ProbeVerdict `json:"probes"` - // Controller is the machine that ran it (the name kept for mesh-watcher, which reads it), and Why what started it. - Controller string `json:"controller"` - Why string `json:"why"` - Unsaid int `json:"unsaid,omitempty"` -} - -// EventHealerActed is the healers' event, one per act (clause 7). -const EventHealerActed = "healer-acted" - -// The outcomes of an act. -const ( - // OutcomeActing is an act begun and not yet finished: counted against the budget and the brake from its begin. - OutcomeActing = "acting" - // OutcomeActed is an act that did what it does; whether it repaired anything is the condition's clearing to say. - OutcomeActed = "acted" - // OutcomeFailed is an act that could not be done. - OutcomeFailed = "failed" - // OutcomeEscalated is a budget spent, said: the condition handed to the operator. - OutcomeEscalated = "escalated" -) - -// HealerActed is the body of `healer-acted`, the controller's (healers.go:345) unchanged; mesh-healers leaves Epoch out -// (ADR 0564 Limits 10). -type HealerActed struct { - Event string `json:"event"` - At time.Time `json:"at"` - Healer string `json:"healer"` - By string `json:"by"` - Condition string `json:"condition"` - Kind string `json:"kind"` - Act string `json:"act"` - Outcome string `json:"outcome"` - Said string `json:"said"` - Budget string `json:"budget"` - Epoch uint64 `json:"epoch,omitempty"` - Show string `json:"show"` -} - -// Attempt is one act of a healer as kept in the healers' state `attempts` (clause 7): created before it is made, so a -// holder that dies mid-act has spent it and a second instance cannot begin the same act, and finished after. The key is -// the healers' own; the keeper reads every value and writes the condition named its `tried`, resolver and escalation. -type Attempt struct { - Healer string `json:"healer"` - Condition string `json:"condition"` - Kind string `json:"kind"` - // BudgetKey is what the budget counts by: the condition's key unless the healer says another. - BudgetKey string `json:"budget-key"` - Act string `json:"act"` - Outcome string `json:"outcome"` - Said string `json:"said,omitempty"` - Begun time.Time `json:"begun"` - Finished *time.Time `json:"finished,omitempty"` -} - -// Tried is the attempt as the condition carries it in `tried`. -func (a Attempt) Tried() Tried { - outcome := a.Outcome - if a.Said != "" { - outcome += ": " + a.Said - } - return Tried{At: a.Begun, What: a.Act, Outcome: outcome, By: "healer " + a.Healer} -} - -// Would is an act mesh-healers would have taken while the controller's lease names a controller, kept in `would`: it -// counts for no budget, settle or brake and reaches no condition (clause 8). -type Would struct { - Healer string `json:"healer"` - Condition string `json:"condition"` - Kind string `json:"kind"` - Act string `json:"act"` - At time.Time `json:"at"` - // Why says why it was not taken: the lease names a controller, which heals. - Why string `json:"why"` -} - -// The readers of conditions (the router, the grants part and mesh-watcher) hear the keeper's condition events on -// KeeperSeat and the self-check's run as `checked` on SelfCheckSeat: the controller's events and the take-over key that -// chose between the two sources ended with it (novox/hq ADR 0590 clause 3). -const ( - KeeperSeat = "mesh-conditions" - SelfCheckSeat = "mesh-self-check" -) diff --git a/go/health/testdata/condition-events.json b/go/health/testdata/condition-events.json deleted file mode 100644 index 2b25e7e..0000000 --- a/go/health/testdata/condition-events.json +++ /dev/null @@ -1,222 +0,0 @@ -[ - { - "actions": null, - "at": "2026-10-29T08:41:18.10610281Z", - "change": "raised", - "count": 1, - "epoch": 395230, - "event": "condition-raised", - "evidence": [ - { - "at": "2026-10-29T08:41:18.10610281Z", - "said": "silent for 2m0s" - } - ], - "explanation": "Nothing for you to do. The mesh has not heard from g14 for several minutes. A machine asleep or away is normal.", - "headline": "g14 is not answering", - "key": "machine.g14.silent", - "kind": "silent", - "last-observed": "2026-10-29T08:41:18.10610281Z", - "needs": "", - "observations": 1, - "raised": "2026-10-29T08:41:18.10610281Z", - "resolved": "g14 answers again", - "resolver": "self", - "severity": "warning", - "show": "mesh-controller.conditions key=machine.g14.silent", - "silenced": null, - "source": "S1", - "subject": { - "id": "g14", - "machine": "g14", - "scope": "machine" - }, - "summary": "g14 has not been heard from since 2026-10-29 08:38 UTC (bound 3m0s)" - }, - { - "actions": null, - "at": "2026-10-28T20:59:53.719121257Z", - "change": "reopened", - "count": 2, - "epoch": 387175, - "event": "condition-raised", - "evidence": [ - { - "at": "2026-10-28T20:59:53.719121257Z", - "said": "70 pending ([70 on EVENTS]), last taken 2026-10-28T20:57:25Z" - } - ], - "explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.", - "first": "2026-10-28T20:54:53.510073517Z", - "gaps": [ - { - "cleared": "2026-10-28T20:56:52.499154879Z", - "reopened": "2026-10-28T20:59:53.719121257Z" - } - ], - "headline": "The controller stopped listening", - "key": "core.controller.deaf", - "kind": "controller-deaf", - "last-observed": "2026-10-28T20:59:53.719121257Z", - "needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.", - "observations": 1, - "raised": "2026-10-28T20:59:53.719121257Z", - "resolved": "The controller listens again", - "resolver": "self", - "severity": "urgent", - "show": "mesh-controller.conditions key=core.controller.deaf", - "silenced": null, - "source": "S4", - "subject": { - "id": "controller", - "machine": "novox", - "scope": "core" - }, - "summary": "the controller's event loop has taken nothing for 2m0s while its consumers hold 70 message(s): reports, builds and merges are not being acted on" - }, - { - "actions": null, - "at": "2026-10-29T18:31:48.308721256Z", - "change": "resolver", - "count": 1, - "epoch": 401364, - "event": "condition-changed", - "evidence": [ - { - "at": "2026-10-29T18:31:33.979579505Z", - "said": "no answer for node-files from g14 to the bus's discovery" - } - ], - "explanation": "Nothing for you to do. Whatever serves node-files on g14 does not answer, so its tools do nothing there. A machine asleep is normal.", - "headline": "Node-files on g14 is not answering", - "key": "seat.node-files.g14.silent", - "kind": "holder-silent", - "last-observed": "2026-10-29T18:31:33.979579Z", - "needs": "", - "observations": 1, - "raised": "2026-10-29T18:31:33.979579505Z", - "resolved": "Node-files on g14 answers again", - "resolver": "healer:H3", - "severity": "warning", - "show": "mesh-controller.conditions key=seat.node-files.g14.silent", - "silenced": null, - "source": "D3", - "subject": { - "id": "node-files.g14", - "machine": "g14", - "scope": "seat" - }, - "summary": "node-files's holder on g14 does not answer the bus: its verbs reach nothing there", - "tried": [ - { - "at": "2026-10-29T18:31:48.308721256Z", - "by": "healer H3", - "outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)", - "what": "asserted the bus's streams, consumers and seat workers again" - } - ], - "was": "self", - "why": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)" - }, - { - "actions": null, - "at": "2026-10-28T20:56:51.686413523Z", - "change": "severity", - "count": 1, - "epoch": 387175, - "event": "condition-changed", - "evidence": [ - { - "at": "2026-10-28T20:55:13.087322533Z", - "said": "no answer for mesh-placement from ace to the bus's discovery" - }, - { - "at": "2026-10-28T20:50:11.256396519Z", - "said": "no answer for mesh-placement from ace to the bus's discovery" - } - ], - "explanation": "Needs you: the mesh tried to repair this and could not; read the details to decide what to do. Whatever serves mesh-placement on ace does not answer, so its tools do nothing there. A machine asleep is normal.", - "headline": "Mesh-placement on ace is not answering", - "key": "seat.mesh-placement.ace.silent", - "kind": "holder-silent", - "last-observed": "2026-10-28T20:55:13.087322Z", - "needs": "the mesh tried to repair this and could not; read the details to decide what to do.", - "observations": 2, - "raised": "2026-10-28T20:50:11.256396519Z", - "resolved": "Mesh-placement on ace answers again", - "resolver": "operator", - "severity": "urgent", - "show": "mesh-controller.conditions key=seat.mesh-placement.ace.silent", - "silenced": null, - "source": "D3", - "subject": { - "id": "mesh-placement.ace", - "machine": "ace", - "scope": "seat" - }, - "summary": "mesh-placement's holder on ace does not answer the bus: its verbs reach nothing there", - "tried": [ - { - "at": "2026-10-28T20:50:22.435049179Z", - "by": "healer H3", - "outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)", - "what": "asserted the bus's streams, consumers and seat workers again" - }, - { - "at": "2026-10-28T20:56:51.686413523Z", - "by": "healer H3", - "outcome": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)", - "what": "handed to the operator: nothing in the mesh will repair it now" - } - ], - "was": "warning", - "why": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)" - }, - { - "actions": null, - "at": "2026-10-28T21:18:48.630162912Z", - "change": "cleared", - "cleared": "2026-10-28T21:18:48.630162912Z", - "count": 2, - "epoch": 387463, - "event": "condition-cleared", - "evidence": [ - { - "at": "2026-10-28T21:18:16.589232939Z", - "said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z" - }, - { - "at": "2026-10-28T21:17:46.778840937Z", - "said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z" - } - ], - "explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.", - "first": "2026-10-28T21:13:16.626696927Z", - "gaps": [ - { - "cleared": "2026-10-28T21:13:47.735189608Z", - "reopened": "2026-10-28T21:17:46.778840937Z" - } - ], - "headline": "The controller stopped listening", - "key": "core.controller.deaf", - "kind": "controller-deaf", - "last-observed": "2026-10-28T21:18:16.589232Z", - "needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.", - "observations": 2, - "raised": "2026-10-28T21:17:46.778840937Z", - "resolved": "The controller listens again", - "resolver": "self", - "severity": "urgent", - "show": "mesh-controller.conditions key=core.controller.deaf", - "silenced": null, - "source": "S4", - "subject": { - "id": "controller", - "machine": "novox", - "scope": "core" - }, - "summary": "the controller's event loop has taken nothing for 3m0s while its consumers hold 82 message(s): reports, builds and merges are not being acted on", - "why": "S4 no longer observes it" - } -] diff --git a/go/testdata/seat-names.txt b/go/testdata/seat-names.txt index aad76dc..1fa53c0 100644 --- a/go/testdata/seat-names.txt +++ b/go/testdata/seat-names.txt @@ -28,8 +28,5 @@ catalogue/seat_dependencies.go: node-package-manager catalogue/seat_dependencies.go: node-service-manager catalogue/shares.go: node-mounts catalogue/shares.go: node-nfs-server -health/findings.go: mesh-conditions -health/findings.go: mesh-healers -health/findings.go: mesh-self-check internal/beside/beside.go: mesh-catalog seatlist/seatlist.go: mesh-catalog