Give the condition keeper the controller's condition bodies, the findings layout and the take-over key, so readers change only the seat they hear (issue 1952 S2-S4, ADR 0564 clauses 2, 3, 7-9)
This commit is contained in:
@@ -0,0 +1,288 @@
|
||||
package health
|
||||
|
||||
// **A condition and its events, as the controller says them today** (novox/hq ADR 0564 clause 3, issue 1952): the body
|
||||
// of `condition-raised`, `condition-changed` and `condition-cleared`, ported field for field from mesh-controller
|
||||
// internal/conditions (condition.go, events.go, plain.go at 0fbb5ef3), so the condition keeper says them unchanged under
|
||||
// the seat mesh-conditions and a reader (the router, the grants part, the healers) changes only the seat it hears.
|
||||
//
|
||||
// A condition is a durable fact about something the mesh owns that is wrong: raised when an observer's finding says so,
|
||||
// kept with since-when, evidence and who can resolve it, said on the bus as it changes, and cleared when the finding
|
||||
// goes, never by hand (ADR 0231 §14, ADR 0564 clause 2).
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Severity is how soon the operator is needed: two levels, no more.
|
||||
type Severity string
|
||||
|
||||
const (
|
||||
// Urgent needs the operator now.
|
||||
Urgent Severity = "urgent"
|
||||
// Warning needs the operator when they can.
|
||||
Warning Severity = "warning"
|
||||
)
|
||||
|
||||
// The scopes a condition's key starts with: what kind of thing is wrong.
|
||||
const (
|
||||
ScopeMachine = "machine"
|
||||
ScopePlan = "plan"
|
||||
ScopeCall = "call"
|
||||
ScopeBuild = "build"
|
||||
ScopeMerge = "merge"
|
||||
ScopeProvider = "provider"
|
||||
ScopeSeat = "seat"
|
||||
ScopeBus = "bus"
|
||||
ScopeCore = "core"
|
||||
ScopeProbe = "probe"
|
||||
ScopeMesh = "mesh"
|
||||
// ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239).
|
||||
ScopeDelivery = "delivery"
|
||||
// ScopeModule is a module on a machine, by `<module>.<machine>` (novox/hq ADR 0240).
|
||||
ScopeModule = "module"
|
||||
)
|
||||
|
||||
// Scopes is every scope, in the order a person reads them.
|
||||
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
||||
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule}
|
||||
|
||||
// Who resolves a condition.
|
||||
const (
|
||||
// ResolverSelf clears on observation: the signal returns, the probe passes.
|
||||
ResolverSelf = "self"
|
||||
// ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy.
|
||||
ResolverOperator = "operator"
|
||||
// ResolverAgent is work handed to an agent.
|
||||
ResolverAgent = "agent"
|
||||
)
|
||||
|
||||
// ResolverHealer is the resolver of a condition a registered healer works on.
|
||||
func ResolverHealer(name string) string { return "healer:" + name }
|
||||
|
||||
// Subject is what the condition is about: its scope, its id within the scope, and the machine it concerns when there is
|
||||
// one.
|
||||
type Subject struct {
|
||||
Scope string `json:"scope"`
|
||||
ID string `json:"id"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
// Also are the other machines it concerns: a consumer's, for a provider failing it.
|
||||
Also []string `json:"also,omitempty"`
|
||||
}
|
||||
|
||||
// Evidence is one observation, as it was said.
|
||||
type Evidence struct {
|
||||
At time.Time `json:"at"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
// Tried is one healer's try at a condition, as the condition carries it in `tried`: when, what it did, what came of it,
|
||||
// and which healer (`healer H3`), never a person. The keeper writes it from the healers' attempts (ADR 0564 clause 7).
|
||||
type Tried struct {
|
||||
At time.Time `json:"at"`
|
||||
What string `json:"what"`
|
||||
Outcome string `json:"outcome"`
|
||||
By string `json:"by"`
|
||||
}
|
||||
|
||||
// Silence is a person saying they know: no messages until it ends. The condition stays open.
|
||||
type Silence struct {
|
||||
Until time.Time `json:"until"`
|
||||
By string `json:"by"`
|
||||
Why string `json:"why"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// Gap is a stretch in which a fault was cleared, between its clearing and its reopening.
|
||||
type Gap struct {
|
||||
Cleared time.Time `json:"cleared"`
|
||||
Reopened time.Time `json:"reopened"`
|
||||
}
|
||||
|
||||
// Action is one answer the operator may give from a notification: a label, and the seat verb the operator channel calls
|
||||
// with these arguments when it is chosen (novox/hq ADR 0253). Level is how much proof its answer needs (ADR 0234 §8,
|
||||
// ADR 0259).
|
||||
type Action struct {
|
||||
Label string `json:"label"`
|
||||
Verb string `json:"verb"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
Arguments map[string]string `json:"arguments,omitempty"`
|
||||
Level string `json:"level,omitempty"`
|
||||
}
|
||||
|
||||
// The assurance levels an action's answer needs (novox/hq ADR 0234 §8).
|
||||
const (
|
||||
LevelAcknowledge = "acknowledge"
|
||||
LevelApprove = "approve"
|
||||
)
|
||||
|
||||
// The bounds a condition keeps, as the controller keeps them.
|
||||
const (
|
||||
// KeptEvidence is how many observations a condition keeps, newest first.
|
||||
KeptEvidence = 10
|
||||
// KeptTried is how many tries a condition keeps, newest last.
|
||||
KeptTried = 10
|
||||
// KeptGaps is how many gaps a condition keeps.
|
||||
KeptGaps = 8
|
||||
// MaxSilence is the longest a condition may be silenced at once.
|
||||
MaxSilence = 7 * 24 * time.Hour
|
||||
// ReopenWithin is how soon after it cleared a condition raised again is the same one again, its count increased,
|
||||
// rather than news.
|
||||
ReopenWithin = 10 * time.Minute
|
||||
)
|
||||
|
||||
// Condition is one open condition, as the keeper keeps it and its events carry it: the controller's body, field for
|
||||
// field (ADR 0564 clause 3). The keeper writes Epoch 0 (Limits 10).
|
||||
type Condition struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
Subject Subject `json:"subject"`
|
||||
Severity Severity `json:"severity"`
|
||||
Summary string `json:"summary"`
|
||||
Headline string `json:"headline"`
|
||||
Explanation string `json:"explanation"`
|
||||
Resolved string `json:"resolved"`
|
||||
Needs string `json:"needs"`
|
||||
Actions []Action `json:"actions"`
|
||||
Evidence []Evidence `json:"evidence"`
|
||||
// Source is the observer's row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
|
||||
Source string `json:"source"`
|
||||
Raised time.Time `json:"raised"`
|
||||
LastObserved time.Time `json:"last-observed"`
|
||||
// First and Gaps are the fault's earlier life: a reopening within ReopenWithin is the same fault.
|
||||
First time.Time `json:"first,omitzero"`
|
||||
Gaps []Gap `json:"gaps,omitempty"`
|
||||
Observations int `json:"observations"`
|
||||
Count int `json:"count"`
|
||||
Tried []Tried `json:"tried,omitempty"`
|
||||
Resolver string `json:"resolver"`
|
||||
// Silenced is null when no silence is in force: said, not left out.
|
||||
Silenced *Silence `json:"silenced"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
// LooksUnread says why Observations, LastObserved and Evidence are not known; the controller's alone.
|
||||
LooksUnread string `json:"looks-unread,omitempty"`
|
||||
}
|
||||
|
||||
// SilencedAt says whether a person's silence is in force at a moment.
|
||||
func (c Condition) SilencedAt(now time.Time) bool {
|
||||
return c.Silenced != nil && now.Before(c.Silenced.Until)
|
||||
}
|
||||
|
||||
// Escalated says a healer tried and its budget is spent: the operator resolves it now.
|
||||
func (c Condition) Escalated() bool { return c.Resolver == ResolverOperator && len(c.Tried) > 0 }
|
||||
|
||||
// Began is when this fault began as the mesh knows it: its first raising, a reopening within ReopenWithin being the
|
||||
// same fault again.
|
||||
func (c Condition) Began() time.Time {
|
||||
if !c.First.IsZero() && c.First.Before(c.Raised) {
|
||||
return c.First
|
||||
}
|
||||
return c.Raised
|
||||
}
|
||||
|
||||
// ShowKeeper is the verb that shows more about a condition the keeper says (ADR 0564 Limits 9); a condition carried from
|
||||
// the controller's bucket keeps the controller's (ShowController) until the lease write.
|
||||
func ShowKeeper(key string) string { return "mesh-conditions.conditions key=" + key }
|
||||
|
||||
// ShowController is the controller's verb that shows more about a condition it raised.
|
||||
func ShowController(key string) string { return "mesh-controller.conditions key=" + key }
|
||||
|
||||
// SilenceAction is the action that stops a condition's messages for a week, as the keeper offers it (ADR 0564 Limits 9:
|
||||
// `mesh-conditions.silence`). An acknowledgement (ADR 0234 §8); its cause marks it as an answer.
|
||||
func SilenceAction(key string) Action {
|
||||
return Action{Label: "Silence for a week", Verb: "mesh-conditions.silence", Level: LevelAcknowledge,
|
||||
Arguments: map[string]string{"key": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}}
|
||||
}
|
||||
|
||||
// CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with.
|
||||
const CauseOperatorAnswer = "operator-answer"
|
||||
|
||||
// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's tokens, and a `*` or `>`
|
||||
// would make one a wildcard.
|
||||
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
|
||||
|
||||
// Key composes a condition's key from its parts, `<scope>.<id>.<token>`, the controller's rule (condition.go Key): each
|
||||
// token made safe for the bus, a character the bus would refuse becoming `_`, an empty token `_`. The id may hold several
|
||||
// tokens joined by dots. A finding and the condition it raises have one key.
|
||||
func Key(scope, id, token string) string {
|
||||
var parts []string
|
||||
for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
|
||||
p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
|
||||
if p == "" {
|
||||
p = "_"
|
||||
}
|
||||
parts = append(parts, p)
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
// Order sorts conditions as `conditions` says them: urgent before warning, then oldest first, then by key.
|
||||
func Order(list []Condition) {
|
||||
sort.SliceStable(list, func(i, j int) bool {
|
||||
if list[i].Severity != list[j].Severity {
|
||||
return list[i].Severity == Urgent
|
||||
}
|
||||
if !list[i].Raised.Equal(list[j].Raised) {
|
||||
return list[i].Raised.Before(list[j].Raised)
|
||||
}
|
||||
return list[i].Key < list[j].Key
|
||||
})
|
||||
}
|
||||
|
||||
// The events a condition's life emits, under the seat that keeps it (ADR 0564 clause 3: the controller's names).
|
||||
const (
|
||||
// EventRaised: a condition was raised, new or the same fault again within ReopenWithin (Change says which).
|
||||
EventRaised = "condition-raised"
|
||||
// EventChanged: its severity, its resolver, its silence or its words changed; not every observation.
|
||||
EventChanged = "condition-changed"
|
||||
// EventCleared: the finding went; the condition is removed and the transition kept in history.
|
||||
EventCleared = "condition-cleared"
|
||||
)
|
||||
|
||||
// Events is every event a condition's life emits.
|
||||
var Events = []string{EventRaised, EventChanged, EventCleared}
|
||||
|
||||
// What changed, as an event's Change and a history entry say it.
|
||||
const (
|
||||
ChangeRaised = "raised"
|
||||
ChangeReopened = "reopened"
|
||||
ChangeSeverity = "severity"
|
||||
ChangeResolver = "resolver"
|
||||
ChangeSilenced = "silenced"
|
||||
ChangeUnsilenced = "silence-ended"
|
||||
ChangeCleared = "cleared"
|
||||
// ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR 0258).
|
||||
ChangeWords = "words"
|
||||
)
|
||||
|
||||
// Event is the body of every condition event and the shape a history entry keeps: the condition itself at the top level,
|
||||
// with what happened to it beside.
|
||||
type Event struct {
|
||||
Condition
|
||||
// Event is the event's own name, so a body read without its subject still says what it is.
|
||||
Event string `json:"event"`
|
||||
At time.Time `json:"at"`
|
||||
// Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, words, cleared.
|
||||
Change string `json:"change"`
|
||||
// Was is the value before, for a severity or resolver change.
|
||||
Was string `json:"was,omitempty"`
|
||||
// Why says why it cleared, or why it was silenced.
|
||||
Why string `json:"why,omitempty"`
|
||||
// Cleared is when it cleared, on a clearing.
|
||||
Cleared *time.Time `json:"cleared,omitempty"`
|
||||
// Show is the verb that shows more.
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// EventFor is the event a change is said under.
|
||||
func EventFor(change string) string {
|
||||
switch change {
|
||||
case ChangeRaised, ChangeReopened:
|
||||
return EventRaised
|
||||
case ChangeCleared:
|
||||
return EventCleared
|
||||
}
|
||||
return EventChanged
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the
|
||||
// events the live controller said unchanged. testdata/condition-events.json holds five transitions of the controller's
|
||||
// condition history (mesh-controller_condition-history, whose entries are the events' bodies, events.go Event), captured
|
||||
// on 2026-10-29 with `mesh-controller.conditions history=true days=1`: a raising, a reopening with gaps, a resolver and a
|
||||
// severity change with tried and was, and a clearing with why. Never written by hand.
|
||||
func TestTheControllersConditionEventsReadUnchanged(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/condition-events.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var captured []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &captured); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changes := map[string]bool{}
|
||||
for _, body := range captured {
|
||||
var e Event
|
||||
if err := json.Unmarshal(body, &e); err != nil {
|
||||
t.Fatalf("%s: %v", body, err)
|
||||
}
|
||||
changes[e.Change] = true
|
||||
again, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var want, got map[string]any
|
||||
_ = json.Unmarshal(body, &want)
|
||||
_ = json.Unmarshal(again, &got)
|
||||
if !reflect.DeepEqual(want, got) {
|
||||
t.Errorf("the controller's %s of %s reads back as\n%s\nnot\n%s", e.Change, e.Key, again, body)
|
||||
}
|
||||
if EventFor(e.Change) != e.Event || e.Key != Key(e.Subject.Scope, e.Subject.ID, e.Key[strings.LastIndex(e.Key, ".")+1:]) {
|
||||
t.Errorf("%s: said as %s for a %s, or its key is not its subject's", e.Key, e.Event, e.Change)
|
||||
}
|
||||
}
|
||||
for _, c := range []string{ChangeRaised, ChangeReopened, ChangeResolver, ChangeSeverity, ChangeCleared} {
|
||||
if !changes[c] {
|
||||
t.Errorf("no captured %s", c)
|
||||
}
|
||||
}
|
||||
// The keeper's own body (Limits 9, 10): epoch 0, said and not left out, silenced null, and its show and silence
|
||||
// naming the new seat.
|
||||
c := Condition{Key: "machine.g14.silent", Actions: []Action{SilenceAction("machine.g14.silent")}}
|
||||
b, _ := json.Marshal(Event{Condition: c, Event: EventRaised, Change: ChangeRaised, Show: ShowKeeper(c.Key)})
|
||||
for _, says := range []string{`"epoch":0`, `"silenced":null`, `"show":"mesh-conditions.conditions key=machine.g14.silent"`,
|
||||
`"verb":"mesh-conditions.silence"`} {
|
||||
if !strings.Contains(string(b), says) {
|
||||
t.Errorf("the keeper's body %s does not say %s", b, says)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0564 clauses 2, 7, 8, 9 (issue 1952 S3, S4): the layouts between the observers, the healers and the keeper, and the
|
||||
// take-over key the readers follow.
|
||||
func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) {
|
||||
at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
|
||||
good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing",
|
||||
Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"},
|
||||
Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
edit func(*Finding)
|
||||
fails string
|
||||
}{
|
||||
{name: "a finding keyed by the controller's key rule"},
|
||||
// The key rule is the controller's (condition.go Key): a character the bus refuses becomes `_`, so the finding
|
||||
// and the condition it raises share one key.
|
||||
{name: "an unsafe id", edit: func(f *Finding) {
|
||||
f.Subject.ID, f.Key = "keycloak.ace.mesh mail*", "provider.keycloak.ace.mesh_mail_.failing"
|
||||
}},
|
||||
{name: "a key not its parts'", edit: func(f *Finding) { f.Key = "provider.keycloak.failing" }, fails: "not"},
|
||||
{name: "an unknown scope", edit: func(f *Finding) { f.Subject.Scope = "thing" }, fails: "scope"},
|
||||
{name: "no severity", edit: func(f *Finding) { f.Severity = "" }, fails: "urgent or a warning"},
|
||||
{name: "no observer", edit: func(f *Finding) { f.Observer = "" }, fails: "observer"},
|
||||
{name: "eleven observations", edit: func(f *Finding) { f.Evidence = make([]Evidence, KeptEvidence+1) }, fails: "at most"},
|
||||
} {
|
||||
f := good
|
||||
if c.edit != nil {
|
||||
c.edit(&f)
|
||||
}
|
||||
b, err := EncodeFinding(f)
|
||||
if c.fails != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.fails) {
|
||||
t.Errorf("%s: encoded (%v), want refused saying %q", c.name, err, c.fails)
|
||||
}
|
||||
continue
|
||||
}
|
||||
back, err := DecodeFinding(b)
|
||||
if err != nil || back.Key != f.Key || back.V != FindingVersion || back.Evidence == nil {
|
||||
t.Errorf("%s: read back as %+v, %v", c.name, back, err)
|
||||
}
|
||||
}
|
||||
if FindingsBucket(SelfCheckModule) != "mesh-self-check_findings" || Bucket(HealersModule, AttemptsState) != "mesh-healers_attempts" ||
|
||||
Bucket(KeeperModule, TakeoverState) != "mesh-conditions_takeover" {
|
||||
t.Errorf("the buckets are named %s, %s", FindingsBucket(SelfCheckModule), Bucket(HealersModule, AttemptsState))
|
||||
}
|
||||
|
||||
// An attempt reaches the condition's tried as the controller wrote it: by the healer, its outcome and what it said.
|
||||
if tr := (Attempt{Healer: "H3", Act: "asserted the grants", Outcome: OutcomeEscalated, Said: "budget spent", Begun: at}).Tried(); tr !=
|
||||
(Tried{At: at, What: "asserted the grants", Outcome: "escalated: budget spent", By: "healer H3"}) {
|
||||
t.Errorf("an attempt is tried as %+v", tr)
|
||||
}
|
||||
|
||||
// The take-over key (clause 9): absent follows the controller; the keeper's names who chose; anything else refused.
|
||||
for _, c := range []struct {
|
||||
raw string
|
||||
found bool
|
||||
seat, events string
|
||||
fails bool
|
||||
}{
|
||||
{found: false, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"},
|
||||
{raw: `{"v":1,"follow":"controller","by":"the operator"}`, found: true, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"},
|
||||
{raw: `{"v":1,"follow":"keeper","by":"the operator","ask":"ask-1"}`, found: true, seat: "mesh-conditions", events: "mesh-self-check.checked"},
|
||||
{raw: `{"v":1,"follow":"keeper"}`, found: true, fails: true},
|
||||
{raw: `{"v":1,"follow":"both","by":"x"}`, found: true, fails: true},
|
||||
{raw: `{"v":2,"follow":"keeper","by":"x"}`, found: true, fails: true},
|
||||
} {
|
||||
got, err := DecodeTakeover([]byte(c.raw), c.found)
|
||||
if c.fails != (err != nil) {
|
||||
t.Errorf("%q: read as %+v, %v", c.raw, got, err)
|
||||
continue
|
||||
}
|
||||
if c.fails {
|
||||
continue
|
||||
}
|
||||
seat, event := got.SelfCheckEvent()
|
||||
if got.Seat() != c.seat || seat+"."+event != c.events {
|
||||
t.Errorf("%q follows %s and %s.%s", c.raw, got.Seat(), seat, event)
|
||||
}
|
||||
if b, err := EncodeTakeover(got); err != nil || !strings.Contains(string(b), `"v":1`) {
|
||||
t.Errorf("%q encodes as %s, %v", c.raw, b, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
package health
|
||||
|
||||
// **What the observers write and the keeper reads** (novox/hq ADR 0564 clauses 2, 6, 7, 8 and 9; issue 1952): the
|
||||
// layout of a finding, the self-check's `checked`, the healers' `healer-acted`, an act begun in `attempts` and one the
|
||||
// healers would have taken, and the take-over key the readers follow. The keeper is the one writer of conditions; an
|
||||
// observer writes findings in its own bucket and never another's (clause 2).
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The modules holding the four health seats, and the states between them, as `<module>_<state>` buckets.
|
||||
const (
|
||||
// KeeperModule is the condition keeper, holder of mesh-conditions.
|
||||
KeeperModule = "mesh-conditions"
|
||||
// SelfCheckModule holds mesh-self-check; HealersModule mesh-healers; WatchmanModule mesh-watchdogs.
|
||||
SelfCheckModule = "mesh-self-check"
|
||||
HealersModule = "mesh-healers"
|
||||
WatchmanModule = "mesh-delivery-watchman"
|
||||
|
||||
// FindingsState is every observer's own state of findings, one key per finding (clause 2).
|
||||
FindingsState = "findings"
|
||||
// ConditionsState is the keeper's open conditions, one key each, without their looks (clause 3).
|
||||
ConditionsState = "conditions"
|
||||
// TakeoverState is the keeper's take-over, one key TakeoverKey (clause 9).
|
||||
TakeoverState = "takeover"
|
||||
// TakeoverKey is the take-over state's one key.
|
||||
TakeoverKey = "switch"
|
||||
// AttemptsState is the healers' acts begun, each created before it is made (clause 7); WouldState the acts they
|
||||
// would have taken while the lease names a controller (clause 8), which the keeper never reads.
|
||||
AttemptsState = "attempts"
|
||||
WouldState = "would"
|
||||
)
|
||||
|
||||
// Bucket is a module's state as the bus names it, `<module>_<state>`.
|
||||
func Bucket(module, state string) string { return module + "_" + state }
|
||||
|
||||
// FindingsBucket is an observer's bucket of findings.
|
||||
func FindingsBucket(observer string) string { return Bucket(observer, FindingsState) }
|
||||
|
||||
// FindingVersion is the version of a finding's layout.
|
||||
const FindingVersion = 1
|
||||
|
||||
// Finding is one thing an observer sees wrong now, kept as the key Key in its bucket `findings` until it no longer sees
|
||||
// it (clause 2): written only when what it says changes (ADR 0289 §1, §4), deleted when it goes. A key written raises or
|
||||
// changes the condition of the same key; a key deleted clears it. The keeper makes the condition from it: its words,
|
||||
// severity, subject and source unchanged, its evidence kept newest first.
|
||||
type Finding struct {
|
||||
V int `json:"v"`
|
||||
// Key is Key(Subject.Scope, Subject.ID, Token or Kind): the condition's key.
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
// Token is the key's last part when it is not Kind (`silent` for a machine).
|
||||
Token string `json:"token,omitempty"`
|
||||
Subject Subject `json:"subject"`
|
||||
Severity Severity `json:"severity"`
|
||||
// Summary is one line in the mesh's words; Headline, Explanation, Resolved and Needs the plain words of ADR 0253
|
||||
// and Actions the answers a notification offers.
|
||||
Summary string `json:"summary"`
|
||||
Headline string `json:"headline"`
|
||||
Explanation string `json:"explanation"`
|
||||
Resolved string `json:"resolved"`
|
||||
Needs string `json:"needs,omitempty"`
|
||||
Actions []Action `json:"actions,omitempty"`
|
||||
// Evidence is what the observer saw, newest first, at most KeptEvidence. Detail goes here, never in Summary.
|
||||
Evidence []Evidence `json:"evidence"`
|
||||
// Source is the row, probe or event that saw it: `S1`, `D6`, `apply-failing`.
|
||||
Source string `json:"source"`
|
||||
// Resolver is who resolves it, ResolverSelf when empty.
|
||||
Resolver string `json:"resolver,omitempty"`
|
||||
// Observer is the module that wrote it, and Run what of its own saw it last (a self-check run's id, empty for a
|
||||
// watch): the condition's looks are its finding's and its observer's last run (clause 2).
|
||||
Observer string `json:"observer"`
|
||||
Run string `json:"run,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// FindingKey is the key of a finding about scope and id of kind (the token): the condition's key rule.
|
||||
func FindingKey(scope, id, token string) string { return Key(scope, id, token) }
|
||||
|
||||
func (f Finding) token() string {
|
||||
if f.Token != "" {
|
||||
return f.Token
|
||||
}
|
||||
return f.Kind
|
||||
}
|
||||
|
||||
// Check refuses a finding the keeper could not make a condition of, naming why.
|
||||
func (f Finding) Check() error {
|
||||
known := false
|
||||
for _, s := range Scopes {
|
||||
known = known || s == f.Subject.Scope
|
||||
}
|
||||
switch {
|
||||
case !known:
|
||||
return fmt.Errorf("health: a finding's scope is one of %s, not %q", strings.Join(Scopes, ", "), f.Subject.Scope)
|
||||
case strings.TrimSpace(f.Subject.ID) == "":
|
||||
return fmt.Errorf("health: a %s finding names what it is about", f.Subject.Scope)
|
||||
case strings.TrimSpace(f.Kind) == "":
|
||||
return fmt.Errorf("health: the finding %s has no kind", f.Key)
|
||||
case f.Key != Key(f.Subject.Scope, f.Subject.ID, f.token()):
|
||||
return fmt.Errorf("health: the finding's key %q is not %q, its scope, id and kind", f.Key,
|
||||
Key(f.Subject.Scope, f.Subject.ID, f.token()))
|
||||
case f.Severity != Urgent && f.Severity != Warning:
|
||||
return fmt.Errorf("health: the finding %s is urgent or a warning, not %q", f.Key, f.Severity)
|
||||
case strings.TrimSpace(f.Summary) == "":
|
||||
return fmt.Errorf("health: the finding %s says nothing", f.Key)
|
||||
case strings.TrimSpace(f.Source) == "":
|
||||
return fmt.Errorf("health: the finding %s does not say what saw it", f.Key)
|
||||
case strings.TrimSpace(f.Observer) == "":
|
||||
return fmt.Errorf("health: the finding %s does not name its observer", f.Key)
|
||||
case len(f.Evidence) > KeptEvidence:
|
||||
return fmt.Errorf("health: the finding %s keeps %d observations, at most %d", f.Key, len(f.Evidence), KeptEvidence)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeFinding is f as JSON at FindingVersion; it refuses what Check refuses.
|
||||
func EncodeFinding(f Finding) ([]byte, error) {
|
||||
f.V = FindingVersion
|
||||
if f.Evidence == nil {
|
||||
f.Evidence = []Evidence{}
|
||||
}
|
||||
if err := f.Check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(f)
|
||||
}
|
||||
|
||||
// DecodeFinding reads a finding's value; it refuses another version and what Check refuses.
|
||||
func DecodeFinding(data []byte) (Finding, error) {
|
||||
var f Finding
|
||||
if err := json.Unmarshal(data, &f); err != nil {
|
||||
return Finding{}, fmt.Errorf("health: a finding is not readable: %w", err)
|
||||
}
|
||||
if f.V != FindingVersion {
|
||||
return Finding{}, fmt.Errorf("health: a finding of version %d, and this reads %d", f.V, FindingVersion)
|
||||
}
|
||||
if err := f.Check(); err != nil {
|
||||
return Finding{}, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// EventChecked is the self-check's event, ending every run (clause 6), with the body of the controller's
|
||||
// `doctor-heartbeat`, which mesh-watcher hears in its place.
|
||||
const EventChecked = "checked"
|
||||
|
||||
// The verdicts of a probe in a run.
|
||||
const (
|
||||
VerdictPassed = "passed"
|
||||
VerdictFailed = "failed"
|
||||
VerdictFailedToRun = "failed-to-run"
|
||||
VerdictDeferred = "deferred"
|
||||
)
|
||||
|
||||
// ProbeVerdict is one probe's outcome in a run (mesh-controller doctor.go probeVerdict).
|
||||
type ProbeVerdict struct {
|
||||
ID string `json:"id"`
|
||||
Verdict string `json:"verdict"`
|
||||
Found []string `json:"found,omitempty"`
|
||||
// Unconfirmed are findings one look can be wrong about, seen by this run and not the one before.
|
||||
Unconfirmed []string `json:"unconfirmed,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// FoundCount and More are an overview's, never a checked event's.
|
||||
FoundCount int `json:"found-count,omitempty"`
|
||||
More string `json:"more,omitempty"`
|
||||
}
|
||||
|
||||
// CheckedCounts are a run's verdicts, counted.
|
||||
type CheckedCounts struct {
|
||||
Passed int `json:"passed"`
|
||||
Failed int `json:"failed"`
|
||||
FailedToRun int `json:"failed-to-run"`
|
||||
Deferred int `json:"deferred"`
|
||||
}
|
||||
|
||||
// Checked is one run of the self-check and the body of `checked`, the controller's doctorRun (doctor.go:230) unchanged.
|
||||
type Checked struct {
|
||||
Run string `json:"run"`
|
||||
At time.Time `json:"at"`
|
||||
Started time.Time `json:"started"`
|
||||
Took string `json:"took"`
|
||||
IntervalSeconds int `json:"interval-seconds"`
|
||||
Counts CheckedCounts `json:"counts"`
|
||||
Probes []ProbeVerdict `json:"probes"`
|
||||
// Controller is the machine that ran it (the name kept for mesh-watcher, which reads it), and Why what started it.
|
||||
Controller string `json:"controller"`
|
||||
Why string `json:"why"`
|
||||
Unsaid int `json:"unsaid,omitempty"`
|
||||
}
|
||||
|
||||
// EventHealerActed is the healers' event, one per act (clause 7).
|
||||
const EventHealerActed = "healer-acted"
|
||||
|
||||
// The outcomes of an act.
|
||||
const (
|
||||
// OutcomeActing is an act begun and not yet finished: counted against the budget and the brake from its begin.
|
||||
OutcomeActing = "acting"
|
||||
// OutcomeActed is an act that did what it does; whether it repaired anything is the condition's clearing to say.
|
||||
OutcomeActed = "acted"
|
||||
// OutcomeFailed is an act that could not be done.
|
||||
OutcomeFailed = "failed"
|
||||
// OutcomeEscalated is a budget spent, said: the condition handed to the operator.
|
||||
OutcomeEscalated = "escalated"
|
||||
)
|
||||
|
||||
// HealerActed is the body of `healer-acted`, the controller's (healers.go:345) unchanged; mesh-healers leaves Epoch out
|
||||
// (ADR 0564 Limits 10).
|
||||
type HealerActed struct {
|
||||
Event string `json:"event"`
|
||||
At time.Time `json:"at"`
|
||||
Healer string `json:"healer"`
|
||||
By string `json:"by"`
|
||||
Condition string `json:"condition"`
|
||||
Kind string `json:"kind"`
|
||||
Act string `json:"act"`
|
||||
Outcome string `json:"outcome"`
|
||||
Said string `json:"said"`
|
||||
Budget string `json:"budget"`
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
Show string `json:"show"`
|
||||
}
|
||||
|
||||
// Attempt is one act of a healer as kept in the healers' state `attempts` (clause 7): created before it is made, so a
|
||||
// holder that dies mid-act has spent it and a second instance cannot begin the same act, and finished after. The key is
|
||||
// the healers' own; the keeper reads every value and writes the condition named its `tried`, resolver and escalation.
|
||||
type Attempt struct {
|
||||
Healer string `json:"healer"`
|
||||
Condition string `json:"condition"`
|
||||
Kind string `json:"kind"`
|
||||
// BudgetKey is what the budget counts by: the condition's key unless the healer says another.
|
||||
BudgetKey string `json:"budget-key"`
|
||||
Act string `json:"act"`
|
||||
Outcome string `json:"outcome"`
|
||||
Said string `json:"said,omitempty"`
|
||||
Begun time.Time `json:"begun"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
}
|
||||
|
||||
// Tried is the attempt as the condition carries it in `tried`.
|
||||
func (a Attempt) Tried() Tried {
|
||||
outcome := a.Outcome
|
||||
if a.Said != "" {
|
||||
outcome += ": " + a.Said
|
||||
}
|
||||
return Tried{At: a.Begun, What: a.Act, Outcome: outcome, By: "healer " + a.Healer}
|
||||
}
|
||||
|
||||
// Would is an act mesh-healers would have taken while the controller's lease names a controller, kept in `would`: it
|
||||
// counts for no budget, settle or brake and reaches no condition (clause 8).
|
||||
type Would struct {
|
||||
Healer string `json:"healer"`
|
||||
Condition string `json:"condition"`
|
||||
Kind string `json:"kind"`
|
||||
Act string `json:"act"`
|
||||
At time.Time `json:"at"`
|
||||
// Why says why it was not taken: the lease names a controller, which heals.
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// Which source the readers of conditions follow (clause 9).
|
||||
const (
|
||||
FollowController = "controller"
|
||||
FollowKeeper = "keeper"
|
||||
)
|
||||
|
||||
// TakeoverVersion is the version of the take-over key's value.
|
||||
const TakeoverVersion = 1
|
||||
|
||||
// Takeover is the value of the keeper's key `takeover.switch` (clause 9): which source the router, the grants part and
|
||||
// mesh-watcher hear, the controller's condition events and `doctor-heartbeat`, or the keeper's events and the
|
||||
// self-check's `checked`. Written by the keeper alone, on the operator's warrant for a take-over and by `hand-back`. An
|
||||
// absent key is FollowController.
|
||||
type Takeover struct {
|
||||
V int `json:"v"`
|
||||
Follow string `json:"follow"`
|
||||
At time.Time `json:"at"`
|
||||
// By is who chose (the operator, as the warrant proves), and Why why; Ask the approve ask it was chosen on.
|
||||
By string `json:"by"`
|
||||
Why string `json:"why,omitempty"`
|
||||
Ask string `json:"ask,omitempty"`
|
||||
}
|
||||
|
||||
// EncodeTakeover is t as JSON at TakeoverVersion; it refuses a source the readers could not follow.
|
||||
func EncodeTakeover(t Takeover) ([]byte, error) {
|
||||
t.V = TakeoverVersion
|
||||
if err := t.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(t)
|
||||
}
|
||||
|
||||
// DecodeTakeover reads the key as a getter answered it: an absent key follows the controller.
|
||||
func DecodeTakeover(data []byte, found bool) (Takeover, error) {
|
||||
if !found || len(data) == 0 {
|
||||
return Takeover{V: TakeoverVersion, Follow: FollowController}, nil
|
||||
}
|
||||
var t Takeover
|
||||
if err := json.Unmarshal(data, &t); err != nil {
|
||||
return Takeover{}, fmt.Errorf("health: the take-over key is not readable: %w", err)
|
||||
}
|
||||
if t.V != TakeoverVersion {
|
||||
return Takeover{}, fmt.Errorf("health: the take-over key is of version %d, and this reads %d", t.V, TakeoverVersion)
|
||||
}
|
||||
if err := t.check(); err != nil {
|
||||
return Takeover{}, err
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (t Takeover) check() error {
|
||||
if t.Follow != FollowController && t.Follow != FollowKeeper {
|
||||
return fmt.Errorf("health: the take-over follows %q, not %s or %s", t.Follow, FollowController, FollowKeeper)
|
||||
}
|
||||
if t.Follow == FollowKeeper && strings.TrimSpace(t.By) == "" {
|
||||
return fmt.Errorf("health: a take-over by the keeper names who chose it")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Seat is the seat a reader hears condition events on while it follows t: mesh-controller or mesh-conditions.
|
||||
func (t Takeover) Seat() string {
|
||||
if t.Follow == FollowKeeper {
|
||||
return "mesh-conditions"
|
||||
}
|
||||
return "mesh-controller"
|
||||
}
|
||||
|
||||
// SelfCheckEvent is the seat and event a reader of the self-check's run hears while it follows t: the controller's
|
||||
// `doctor-heartbeat`, or mesh-self-check's `checked`.
|
||||
func (t Takeover) SelfCheckEvent() (seat, event string) {
|
||||
if t.Follow == FollowKeeper {
|
||||
return "mesh-self-check", EventChecked
|
||||
}
|
||||
return "mesh-controller", "doctor-heartbeat"
|
||||
}
|
||||
+26
-29
@@ -52,15 +52,6 @@ func TestAWaitIsReadWithWhatItNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerWithoutStateIsReadFromHealthy(t *testing.T) {
|
||||
for raw, want := range map[string]string{`{"healthy":true,"why":"ok"}`: StateHealthy, `{"healthy":false,"why":"down"}`: StateUnhealthy} {
|
||||
got, err := Read([]byte(raw))
|
||||
if err != nil || got.State != want {
|
||||
t.Fatalf("%s read as %+v, %v", raw, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitThatCannotBeReadIsUnhealthyAndSaysWhy(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
w []Wait
|
||||
@@ -88,13 +79,6 @@ func TestAWaitThatCannotBeReadIsUnhealthyAndSaysWhy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownStateIsUnhealthy(t *testing.T) {
|
||||
got, err := Read([]byte(`{"state":"fine","healthy":true}`))
|
||||
if err != nil || got.State != StateUnhealthy || got.Healthy {
|
||||
t.Fatalf("read as %+v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitsBesideAFaultAreSaidAndExcuseNothing(t *testing.T) {
|
||||
got := roundTrip(t, Unhealthy("the share is unreachable", password, Wait{Part: "p", What: "w"}))
|
||||
if got.State != StateUnhealthy || len(got.Waits) != 1 {
|
||||
@@ -111,12 +95,6 @@ func TestWaitingWithNothingIsUnhealthy(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotAnAnswerIsAnError(t *testing.T) {
|
||||
if _, err := Read([]byte(`"yes"`)); err == nil {
|
||||
t.Fatal("a string read as an answer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryFormOfPathOrAddressIsRefused(t *testing.T) {
|
||||
for _, words := range []string{
|
||||
"the source (/mnt/games)", `the source "/mnt/games"`, "the folder ~/games", "the folder ./games",
|
||||
@@ -136,15 +114,34 @@ func TestEveryFormOfPathOrAddressIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthyStateWithHealthyFalseIsUnhealthy(t *testing.T) {
|
||||
got, err := Read([]byte(`{"state":"healthy","healthy":false,"why":"x"}`))
|
||||
if err != nil || got.State != StateUnhealthy || !strings.Contains(got.Why, "contradict") {
|
||||
t.Fatalf("read as %+v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitingChecksItsWaits(t *testing.T) {
|
||||
if a := Waiting("it waits", Wait{Part: "p", What: "w"}); a.State != StateUnhealthy || !strings.Contains(a.Why, "neither") {
|
||||
t.Fatalf("Waiting with a malformed wait is %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// A raw answer is read by what it says: without state from healthy; an unknown state, or healthy with healthy false, as
|
||||
// unhealthy; not an answer at all as an error.
|
||||
func TestARawAnswerIsReadByWhatItSays(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
raw, state, says string
|
||||
fails bool
|
||||
}{
|
||||
{raw: `{"healthy":true,"why":"ok"}`, state: StateHealthy},
|
||||
{raw: `{"healthy":false,"why":"down"}`, state: StateUnhealthy},
|
||||
{raw: `{"state":"fine","healthy":true}`, state: StateUnhealthy},
|
||||
{raw: `{"state":"healthy","healthy":false,"why":"x"}`, state: StateUnhealthy, says: "contradict"},
|
||||
{raw: `"yes"`, fails: true},
|
||||
} {
|
||||
got, err := Read([]byte(c.raw))
|
||||
if c.fails {
|
||||
if err == nil {
|
||||
t.Errorf("%s read as an answer", c.raw)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || got.State != c.state || (c.state == StateUnhealthy && got.Healthy) || !strings.Contains(got.Why, c.says) {
|
||||
t.Errorf("%s read as %+v, %v", c.raw, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+222
@@ -0,0 +1,222 @@
|
||||
[
|
||||
{
|
||||
"actions": null,
|
||||
"at": "2026-10-29T08:41:18.10610281Z",
|
||||
"change": "raised",
|
||||
"count": 1,
|
||||
"epoch": 395230,
|
||||
"event": "condition-raised",
|
||||
"evidence": [
|
||||
{
|
||||
"at": "2026-10-29T08:41:18.10610281Z",
|
||||
"said": "silent for 2m0s"
|
||||
}
|
||||
],
|
||||
"explanation": "Nothing for you to do. The mesh has not heard from g14 for several minutes. A machine asleep or away is normal.",
|
||||
"headline": "g14 is not answering",
|
||||
"key": "machine.g14.silent",
|
||||
"kind": "silent",
|
||||
"last-observed": "2026-10-29T08:41:18.10610281Z",
|
||||
"needs": "",
|
||||
"observations": 1,
|
||||
"raised": "2026-10-29T08:41:18.10610281Z",
|
||||
"resolved": "g14 answers again",
|
||||
"resolver": "self",
|
||||
"severity": "warning",
|
||||
"show": "mesh-controller.conditions key=machine.g14.silent",
|
||||
"silenced": null,
|
||||
"source": "S1",
|
||||
"subject": {
|
||||
"id": "g14",
|
||||
"machine": "g14",
|
||||
"scope": "machine"
|
||||
},
|
||||
"summary": "g14 has not been heard from since 2026-10-29 08:38 UTC (bound 3m0s)"
|
||||
},
|
||||
{
|
||||
"actions": null,
|
||||
"at": "2026-10-28T20:59:53.719121257Z",
|
||||
"change": "reopened",
|
||||
"count": 2,
|
||||
"epoch": 387175,
|
||||
"event": "condition-raised",
|
||||
"evidence": [
|
||||
{
|
||||
"at": "2026-10-28T20:59:53.719121257Z",
|
||||
"said": "70 pending ([70 on EVENTS]), last taken 2026-10-28T20:57:25Z"
|
||||
}
|
||||
],
|
||||
"explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.",
|
||||
"first": "2026-10-28T20:54:53.510073517Z",
|
||||
"gaps": [
|
||||
{
|
||||
"cleared": "2026-10-28T20:56:52.499154879Z",
|
||||
"reopened": "2026-10-28T20:59:53.719121257Z"
|
||||
}
|
||||
],
|
||||
"headline": "The controller stopped listening",
|
||||
"key": "core.controller.deaf",
|
||||
"kind": "controller-deaf",
|
||||
"last-observed": "2026-10-28T20:59:53.719121257Z",
|
||||
"needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.",
|
||||
"observations": 1,
|
||||
"raised": "2026-10-28T20:59:53.719121257Z",
|
||||
"resolved": "The controller listens again",
|
||||
"resolver": "self",
|
||||
"severity": "urgent",
|
||||
"show": "mesh-controller.conditions key=core.controller.deaf",
|
||||
"silenced": null,
|
||||
"source": "S4",
|
||||
"subject": {
|
||||
"id": "controller",
|
||||
"machine": "novox",
|
||||
"scope": "core"
|
||||
},
|
||||
"summary": "the controller's event loop has taken nothing for 2m0s while its consumers hold 70 message(s): reports, builds and merges are not being acted on"
|
||||
},
|
||||
{
|
||||
"actions": null,
|
||||
"at": "2026-10-29T18:31:48.308721256Z",
|
||||
"change": "resolver",
|
||||
"count": 1,
|
||||
"epoch": 401364,
|
||||
"event": "condition-changed",
|
||||
"evidence": [
|
||||
{
|
||||
"at": "2026-10-29T18:31:33.979579505Z",
|
||||
"said": "no answer for node-files from g14 to the bus's discovery"
|
||||
}
|
||||
],
|
||||
"explanation": "Nothing for you to do. Whatever serves node-files on g14 does not answer, so its tools do nothing there. A machine asleep is normal.",
|
||||
"headline": "Node-files on g14 is not answering",
|
||||
"key": "seat.node-files.g14.silent",
|
||||
"kind": "holder-silent",
|
||||
"last-observed": "2026-10-29T18:31:33.979579Z",
|
||||
"needs": "",
|
||||
"observations": 1,
|
||||
"raised": "2026-10-29T18:31:33.979579505Z",
|
||||
"resolved": "Node-files on g14 answers again",
|
||||
"resolver": "healer:H3",
|
||||
"severity": "warning",
|
||||
"show": "mesh-controller.conditions key=seat.node-files.g14.silent",
|
||||
"silenced": null,
|
||||
"source": "D3",
|
||||
"subject": {
|
||||
"id": "node-files.g14",
|
||||
"machine": "g14",
|
||||
"scope": "seat"
|
||||
},
|
||||
"summary": "node-files's holder on g14 does not answer the bus: its verbs reach nothing there",
|
||||
"tried": [
|
||||
{
|
||||
"at": "2026-10-29T18:31:48.308721256Z",
|
||||
"by": "healer H3",
|
||||
"outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)",
|
||||
"what": "asserted the bus's streams, consumers and seat workers again"
|
||||
}
|
||||
],
|
||||
"was": "self",
|
||||
"why": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)"
|
||||
},
|
||||
{
|
||||
"actions": null,
|
||||
"at": "2026-10-28T20:56:51.686413523Z",
|
||||
"change": "severity",
|
||||
"count": 1,
|
||||
"epoch": 387175,
|
||||
"event": "condition-changed",
|
||||
"evidence": [
|
||||
{
|
||||
"at": "2026-10-28T20:55:13.087322533Z",
|
||||
"said": "no answer for mesh-placement from ace to the bus's discovery"
|
||||
},
|
||||
{
|
||||
"at": "2026-10-28T20:50:11.256396519Z",
|
||||
"said": "no answer for mesh-placement from ace to the bus's discovery"
|
||||
}
|
||||
],
|
||||
"explanation": "Needs you: the mesh tried to repair this and could not; read the details to decide what to do. Whatever serves mesh-placement on ace does not answer, so its tools do nothing there. A machine asleep is normal.",
|
||||
"headline": "Mesh-placement on ace is not answering",
|
||||
"key": "seat.mesh-placement.ace.silent",
|
||||
"kind": "holder-silent",
|
||||
"last-observed": "2026-10-28T20:55:13.087322Z",
|
||||
"needs": "the mesh tried to repair this and could not; read the details to decide what to do.",
|
||||
"observations": 2,
|
||||
"raised": "2026-10-28T20:50:11.256396519Z",
|
||||
"resolved": "Mesh-placement on ace answers again",
|
||||
"resolver": "operator",
|
||||
"severity": "urgent",
|
||||
"show": "mesh-controller.conditions key=seat.mesh-placement.ace.silent",
|
||||
"silenced": null,
|
||||
"source": "D3",
|
||||
"subject": {
|
||||
"id": "mesh-placement.ace",
|
||||
"machine": "ace",
|
||||
"scope": "seat"
|
||||
},
|
||||
"summary": "mesh-placement's holder on ace does not answer the bus: its verbs reach nothing there",
|
||||
"tried": [
|
||||
{
|
||||
"at": "2026-10-28T20:50:22.435049179Z",
|
||||
"by": "healer H3",
|
||||
"outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)",
|
||||
"what": "asserted the bus's streams, consumers and seat workers again"
|
||||
},
|
||||
{
|
||||
"at": "2026-10-28T20:56:51.686413523Z",
|
||||
"by": "healer H3",
|
||||
"outcome": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)",
|
||||
"what": "handed to the operator: nothing in the mesh will repair it now"
|
||||
}
|
||||
],
|
||||
"was": "warning",
|
||||
"why": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)"
|
||||
},
|
||||
{
|
||||
"actions": null,
|
||||
"at": "2026-10-28T21:18:48.630162912Z",
|
||||
"change": "cleared",
|
||||
"cleared": "2026-10-28T21:18:48.630162912Z",
|
||||
"count": 2,
|
||||
"epoch": 387463,
|
||||
"event": "condition-cleared",
|
||||
"evidence": [
|
||||
{
|
||||
"at": "2026-10-28T21:18:16.589232939Z",
|
||||
"said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z"
|
||||
},
|
||||
{
|
||||
"at": "2026-10-28T21:17:46.778840937Z",
|
||||
"said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z"
|
||||
}
|
||||
],
|
||||
"explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.",
|
||||
"first": "2026-10-28T21:13:16.626696927Z",
|
||||
"gaps": [
|
||||
{
|
||||
"cleared": "2026-10-28T21:13:47.735189608Z",
|
||||
"reopened": "2026-10-28T21:17:46.778840937Z"
|
||||
}
|
||||
],
|
||||
"headline": "The controller stopped listening",
|
||||
"key": "core.controller.deaf",
|
||||
"kind": "controller-deaf",
|
||||
"last-observed": "2026-10-28T21:18:16.589232Z",
|
||||
"needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.",
|
||||
"observations": 2,
|
||||
"raised": "2026-10-28T21:17:46.778840937Z",
|
||||
"resolved": "The controller listens again",
|
||||
"resolver": "self",
|
||||
"severity": "urgent",
|
||||
"show": "mesh-controller.conditions key=core.controller.deaf",
|
||||
"silenced": null,
|
||||
"source": "S4",
|
||||
"subject": {
|
||||
"id": "controller",
|
||||
"machine": "novox",
|
||||
"scope": "core"
|
||||
},
|
||||
"summary": "the controller's event loop has taken nothing for 3m0s while its consumers hold 82 message(s): reports, builds and merges are not being acted on",
|
||||
"why": "S4 no longer observes it"
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user