Give the condition keeper the controller's condition bodies, the findings layout and the take-over key, so readers change only the seat they hear (issue 1952 S2-S4, ADR 0564 clauses 2, 3, 7-9)
This commit is contained in:
@@ -0,0 +1,288 @@
|
|||||||
|
package health
|
||||||
|
|
||||||
|
// **A condition and its events, as the controller says them today** (novox/hq ADR 0564 clause 3, issue 1952): the body
|
||||||
|
// of `condition-raised`, `condition-changed` and `condition-cleared`, ported field for field from mesh-controller
|
||||||
|
// internal/conditions (condition.go, events.go, plain.go at 0fbb5ef3), so the condition keeper says them unchanged under
|
||||||
|
// the seat mesh-conditions and a reader (the router, the grants part, the healers) changes only the seat it hears.
|
||||||
|
//
|
||||||
|
// A condition is a durable fact about something the mesh owns that is wrong: raised when an observer's finding says so,
|
||||||
|
// kept with since-when, evidence and who can resolve it, said on the bus as it changes, and cleared when the finding
|
||||||
|
// goes, never by hand (ADR 0231 §14, ADR 0564 clause 2).
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Severity is how soon the operator is needed: two levels, no more.
|
||||||
|
type Severity string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Urgent needs the operator now.
|
||||||
|
Urgent Severity = "urgent"
|
||||||
|
// Warning needs the operator when they can.
|
||||||
|
Warning Severity = "warning"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The scopes a condition's key starts with: what kind of thing is wrong.
|
||||||
|
const (
|
||||||
|
ScopeMachine = "machine"
|
||||||
|
ScopePlan = "plan"
|
||||||
|
ScopeCall = "call"
|
||||||
|
ScopeBuild = "build"
|
||||||
|
ScopeMerge = "merge"
|
||||||
|
ScopeProvider = "provider"
|
||||||
|
ScopeSeat = "seat"
|
||||||
|
ScopeBus = "bus"
|
||||||
|
ScopeCore = "core"
|
||||||
|
ScopeProbe = "probe"
|
||||||
|
ScopeMesh = "mesh"
|
||||||
|
// ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239).
|
||||||
|
ScopeDelivery = "delivery"
|
||||||
|
// ScopeModule is a module on a machine, by `<module>.<machine>` (novox/hq ADR 0240).
|
||||||
|
ScopeModule = "module"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Scopes is every scope, in the order a person reads them.
|
||||||
|
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
|
||||||
|
ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule}
|
||||||
|
|
||||||
|
// Who resolves a condition.
|
||||||
|
const (
|
||||||
|
// ResolverSelf clears on observation: the signal returns, the probe passes.
|
||||||
|
ResolverSelf = "self"
|
||||||
|
// ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy.
|
||||||
|
ResolverOperator = "operator"
|
||||||
|
// ResolverAgent is work handed to an agent.
|
||||||
|
ResolverAgent = "agent"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResolverHealer is the resolver of a condition a registered healer works on.
|
||||||
|
func ResolverHealer(name string) string { return "healer:" + name }
|
||||||
|
|
||||||
|
// Subject is what the condition is about: its scope, its id within the scope, and the machine it concerns when there is
|
||||||
|
// one.
|
||||||
|
type Subject struct {
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Machine string `json:"machine,omitempty"`
|
||||||
|
// Also are the other machines it concerns: a consumer's, for a provider failing it.
|
||||||
|
Also []string `json:"also,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Evidence is one observation, as it was said.
|
||||||
|
type Evidence struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Said string `json:"said"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tried is one healer's try at a condition, as the condition carries it in `tried`: when, what it did, what came of it,
|
||||||
|
// and which healer (`healer H3`), never a person. The keeper writes it from the healers' attempts (ADR 0564 clause 7).
|
||||||
|
type Tried struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
What string `json:"what"`
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
By string `json:"by"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Silence is a person saying they know: no messages until it ends. The condition stays open.
|
||||||
|
type Silence struct {
|
||||||
|
Until time.Time `json:"until"`
|
||||||
|
By string `json:"by"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gap is a stretch in which a fault was cleared, between its clearing and its reopening.
|
||||||
|
type Gap struct {
|
||||||
|
Cleared time.Time `json:"cleared"`
|
||||||
|
Reopened time.Time `json:"reopened"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Action is one answer the operator may give from a notification: a label, and the seat verb the operator channel calls
|
||||||
|
// with these arguments when it is chosen (novox/hq ADR 0253). Level is how much proof its answer needs (ADR 0234 §8,
|
||||||
|
// ADR 0259).
|
||||||
|
type Action struct {
|
||||||
|
Label string `json:"label"`
|
||||||
|
Verb string `json:"verb"`
|
||||||
|
Machine string `json:"machine,omitempty"`
|
||||||
|
Arguments map[string]string `json:"arguments,omitempty"`
|
||||||
|
Level string `json:"level,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// The assurance levels an action's answer needs (novox/hq ADR 0234 §8).
|
||||||
|
const (
|
||||||
|
LevelAcknowledge = "acknowledge"
|
||||||
|
LevelApprove = "approve"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bounds a condition keeps, as the controller keeps them.
|
||||||
|
const (
|
||||||
|
// KeptEvidence is how many observations a condition keeps, newest first.
|
||||||
|
KeptEvidence = 10
|
||||||
|
// KeptTried is how many tries a condition keeps, newest last.
|
||||||
|
KeptTried = 10
|
||||||
|
// KeptGaps is how many gaps a condition keeps.
|
||||||
|
KeptGaps = 8
|
||||||
|
// MaxSilence is the longest a condition may be silenced at once.
|
||||||
|
MaxSilence = 7 * 24 * time.Hour
|
||||||
|
// ReopenWithin is how soon after it cleared a condition raised again is the same one again, its count increased,
|
||||||
|
// rather than news.
|
||||||
|
ReopenWithin = 10 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// Condition is one open condition, as the keeper keeps it and its events carry it: the controller's body, field for
|
||||||
|
// field (ADR 0564 clause 3). The keeper writes Epoch 0 (Limits 10).
|
||||||
|
type Condition struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Subject Subject `json:"subject"`
|
||||||
|
Severity Severity `json:"severity"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
Headline string `json:"headline"`
|
||||||
|
Explanation string `json:"explanation"`
|
||||||
|
Resolved string `json:"resolved"`
|
||||||
|
Needs string `json:"needs"`
|
||||||
|
Actions []Action `json:"actions"`
|
||||||
|
Evidence []Evidence `json:"evidence"`
|
||||||
|
// Source is the observer's row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
|
||||||
|
Source string `json:"source"`
|
||||||
|
Raised time.Time `json:"raised"`
|
||||||
|
LastObserved time.Time `json:"last-observed"`
|
||||||
|
// First and Gaps are the fault's earlier life: a reopening within ReopenWithin is the same fault.
|
||||||
|
First time.Time `json:"first,omitzero"`
|
||||||
|
Gaps []Gap `json:"gaps,omitempty"`
|
||||||
|
Observations int `json:"observations"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
Tried []Tried `json:"tried,omitempty"`
|
||||||
|
Resolver string `json:"resolver"`
|
||||||
|
// Silenced is null when no silence is in force: said, not left out.
|
||||||
|
Silenced *Silence `json:"silenced"`
|
||||||
|
Epoch uint64 `json:"epoch"`
|
||||||
|
// LooksUnread says why Observations, LastObserved and Evidence are not known; the controller's alone.
|
||||||
|
LooksUnread string `json:"looks-unread,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SilencedAt says whether a person's silence is in force at a moment.
|
||||||
|
func (c Condition) SilencedAt(now time.Time) bool {
|
||||||
|
return c.Silenced != nil && now.Before(c.Silenced.Until)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Escalated says a healer tried and its budget is spent: the operator resolves it now.
|
||||||
|
func (c Condition) Escalated() bool { return c.Resolver == ResolverOperator && len(c.Tried) > 0 }
|
||||||
|
|
||||||
|
// Began is when this fault began as the mesh knows it: its first raising, a reopening within ReopenWithin being the
|
||||||
|
// same fault again.
|
||||||
|
func (c Condition) Began() time.Time {
|
||||||
|
if !c.First.IsZero() && c.First.Before(c.Raised) {
|
||||||
|
return c.First
|
||||||
|
}
|
||||||
|
return c.Raised
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShowKeeper is the verb that shows more about a condition the keeper says (ADR 0564 Limits 9); a condition carried from
|
||||||
|
// the controller's bucket keeps the controller's (ShowController) until the lease write.
|
||||||
|
func ShowKeeper(key string) string { return "mesh-conditions.conditions key=" + key }
|
||||||
|
|
||||||
|
// ShowController is the controller's verb that shows more about a condition it raised.
|
||||||
|
func ShowController(key string) string { return "mesh-controller.conditions key=" + key }
|
||||||
|
|
||||||
|
// SilenceAction is the action that stops a condition's messages for a week, as the keeper offers it (ADR 0564 Limits 9:
|
||||||
|
// `mesh-conditions.silence`). An acknowledgement (ADR 0234 §8); its cause marks it as an answer.
|
||||||
|
func SilenceAction(key string) Action {
|
||||||
|
return Action{Label: "Silence for a week", Verb: "mesh-conditions.silence", Level: LevelAcknowledge,
|
||||||
|
Arguments: map[string]string{"key": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with.
|
||||||
|
const CauseOperatorAnswer = "operator-answer"
|
||||||
|
|
||||||
|
// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's tokens, and a `*` or `>`
|
||||||
|
// would make one a wildcard.
|
||||||
|
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
|
||||||
|
|
||||||
|
// Key composes a condition's key from its parts, `<scope>.<id>.<token>`, the controller's rule (condition.go Key): each
|
||||||
|
// token made safe for the bus, a character the bus would refuse becoming `_`, an empty token `_`. The id may hold several
|
||||||
|
// tokens joined by dots. A finding and the condition it raises have one key.
|
||||||
|
func Key(scope, id, token string) string {
|
||||||
|
var parts []string
|
||||||
|
for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
|
||||||
|
p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
|
||||||
|
if p == "" {
|
||||||
|
p = "_"
|
||||||
|
}
|
||||||
|
parts = append(parts, p)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Order sorts conditions as `conditions` says them: urgent before warning, then oldest first, then by key.
|
||||||
|
func Order(list []Condition) {
|
||||||
|
sort.SliceStable(list, func(i, j int) bool {
|
||||||
|
if list[i].Severity != list[j].Severity {
|
||||||
|
return list[i].Severity == Urgent
|
||||||
|
}
|
||||||
|
if !list[i].Raised.Equal(list[j].Raised) {
|
||||||
|
return list[i].Raised.Before(list[j].Raised)
|
||||||
|
}
|
||||||
|
return list[i].Key < list[j].Key
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The events a condition's life emits, under the seat that keeps it (ADR 0564 clause 3: the controller's names).
|
||||||
|
const (
|
||||||
|
// EventRaised: a condition was raised, new or the same fault again within ReopenWithin (Change says which).
|
||||||
|
EventRaised = "condition-raised"
|
||||||
|
// EventChanged: its severity, its resolver, its silence or its words changed; not every observation.
|
||||||
|
EventChanged = "condition-changed"
|
||||||
|
// EventCleared: the finding went; the condition is removed and the transition kept in history.
|
||||||
|
EventCleared = "condition-cleared"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Events is every event a condition's life emits.
|
||||||
|
var Events = []string{EventRaised, EventChanged, EventCleared}
|
||||||
|
|
||||||
|
// What changed, as an event's Change and a history entry say it.
|
||||||
|
const (
|
||||||
|
ChangeRaised = "raised"
|
||||||
|
ChangeReopened = "reopened"
|
||||||
|
ChangeSeverity = "severity"
|
||||||
|
ChangeResolver = "resolver"
|
||||||
|
ChangeSilenced = "silenced"
|
||||||
|
ChangeUnsilenced = "silence-ended"
|
||||||
|
ChangeCleared = "cleared"
|
||||||
|
// ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR 0258).
|
||||||
|
ChangeWords = "words"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Event is the body of every condition event and the shape a history entry keeps: the condition itself at the top level,
|
||||||
|
// with what happened to it beside.
|
||||||
|
type Event struct {
|
||||||
|
Condition
|
||||||
|
// Event is the event's own name, so a body read without its subject still says what it is.
|
||||||
|
Event string `json:"event"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, words, cleared.
|
||||||
|
Change string `json:"change"`
|
||||||
|
// Was is the value before, for a severity or resolver change.
|
||||||
|
Was string `json:"was,omitempty"`
|
||||||
|
// Why says why it cleared, or why it was silenced.
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
// Cleared is when it cleared, on a clearing.
|
||||||
|
Cleared *time.Time `json:"cleared,omitempty"`
|
||||||
|
// Show is the verb that shows more.
|
||||||
|
Show string `json:"show"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EventFor is the event a change is said under.
|
||||||
|
func EventFor(change string) string {
|
||||||
|
switch change {
|
||||||
|
case ChangeRaised, ChangeReopened:
|
||||||
|
return EventRaised
|
||||||
|
case ChangeCleared:
|
||||||
|
return EventCleared
|
||||||
|
}
|
||||||
|
return EventChanged
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
package health
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the
|
||||||
|
// events the live controller said unchanged. testdata/condition-events.json holds five transitions of the controller's
|
||||||
|
// condition history (mesh-controller_condition-history, whose entries are the events' bodies, events.go Event), captured
|
||||||
|
// on 2026-10-29 with `mesh-controller.conditions history=true days=1`: a raising, a reopening with gaps, a resolver and a
|
||||||
|
// severity change with tried and was, and a clearing with why. Never written by hand.
|
||||||
|
func TestTheControllersConditionEventsReadUnchanged(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("testdata/condition-events.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var captured []json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, &captured); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
changes := map[string]bool{}
|
||||||
|
for _, body := range captured {
|
||||||
|
var e Event
|
||||||
|
if err := json.Unmarshal(body, &e); err != nil {
|
||||||
|
t.Fatalf("%s: %v", body, err)
|
||||||
|
}
|
||||||
|
changes[e.Change] = true
|
||||||
|
again, err := json.Marshal(e)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var want, got map[string]any
|
||||||
|
_ = json.Unmarshal(body, &want)
|
||||||
|
_ = json.Unmarshal(again, &got)
|
||||||
|
if !reflect.DeepEqual(want, got) {
|
||||||
|
t.Errorf("the controller's %s of %s reads back as\n%s\nnot\n%s", e.Change, e.Key, again, body)
|
||||||
|
}
|
||||||
|
if EventFor(e.Change) != e.Event || e.Key != Key(e.Subject.Scope, e.Subject.ID, e.Key[strings.LastIndex(e.Key, ".")+1:]) {
|
||||||
|
t.Errorf("%s: said as %s for a %s, or its key is not its subject's", e.Key, e.Event, e.Change)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range []string{ChangeRaised, ChangeReopened, ChangeResolver, ChangeSeverity, ChangeCleared} {
|
||||||
|
if !changes[c] {
|
||||||
|
t.Errorf("no captured %s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The keeper's own body (Limits 9, 10): epoch 0, said and not left out, silenced null, and its show and silence
|
||||||
|
// naming the new seat.
|
||||||
|
c := Condition{Key: "machine.g14.silent", Actions: []Action{SilenceAction("machine.g14.silent")}}
|
||||||
|
b, _ := json.Marshal(Event{Condition: c, Event: EventRaised, Change: ChangeRaised, Show: ShowKeeper(c.Key)})
|
||||||
|
for _, says := range []string{`"epoch":0`, `"silenced":null`, `"show":"mesh-conditions.conditions key=machine.g14.silent"`,
|
||||||
|
`"verb":"mesh-conditions.silence"`} {
|
||||||
|
if !strings.Contains(string(b), says) {
|
||||||
|
t.Errorf("the keeper's body %s does not say %s", b, says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0564 clauses 2, 7, 8, 9 (issue 1952 S3, S4): the layouts between the observers, the healers and the keeper, and the
|
||||||
|
// take-over key the readers follow.
|
||||||
|
func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
|
||||||
|
good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing",
|
||||||
|
Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"},
|
||||||
|
Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
edit func(*Finding)
|
||||||
|
fails string
|
||||||
|
}{
|
||||||
|
{name: "a finding keyed by the controller's key rule"},
|
||||||
|
// The key rule is the controller's (condition.go Key): a character the bus refuses becomes `_`, so the finding
|
||||||
|
// and the condition it raises share one key.
|
||||||
|
{name: "an unsafe id", edit: func(f *Finding) {
|
||||||
|
f.Subject.ID, f.Key = "keycloak.ace.mesh mail*", "provider.keycloak.ace.mesh_mail_.failing"
|
||||||
|
}},
|
||||||
|
{name: "a key not its parts'", edit: func(f *Finding) { f.Key = "provider.keycloak.failing" }, fails: "not"},
|
||||||
|
{name: "an unknown scope", edit: func(f *Finding) { f.Subject.Scope = "thing" }, fails: "scope"},
|
||||||
|
{name: "no severity", edit: func(f *Finding) { f.Severity = "" }, fails: "urgent or a warning"},
|
||||||
|
{name: "no observer", edit: func(f *Finding) { f.Observer = "" }, fails: "observer"},
|
||||||
|
{name: "eleven observations", edit: func(f *Finding) { f.Evidence = make([]Evidence, KeptEvidence+1) }, fails: "at most"},
|
||||||
|
} {
|
||||||
|
f := good
|
||||||
|
if c.edit != nil {
|
||||||
|
c.edit(&f)
|
||||||
|
}
|
||||||
|
b, err := EncodeFinding(f)
|
||||||
|
if c.fails != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.fails) {
|
||||||
|
t.Errorf("%s: encoded (%v), want refused saying %q", c.name, err, c.fails)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
back, err := DecodeFinding(b)
|
||||||
|
if err != nil || back.Key != f.Key || back.V != FindingVersion || back.Evidence == nil {
|
||||||
|
t.Errorf("%s: read back as %+v, %v", c.name, back, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if FindingsBucket(SelfCheckModule) != "mesh-self-check_findings" || Bucket(HealersModule, AttemptsState) != "mesh-healers_attempts" ||
|
||||||
|
Bucket(KeeperModule, TakeoverState) != "mesh-conditions_takeover" {
|
||||||
|
t.Errorf("the buckets are named %s, %s", FindingsBucket(SelfCheckModule), Bucket(HealersModule, AttemptsState))
|
||||||
|
}
|
||||||
|
|
||||||
|
// An attempt reaches the condition's tried as the controller wrote it: by the healer, its outcome and what it said.
|
||||||
|
if tr := (Attempt{Healer: "H3", Act: "asserted the grants", Outcome: OutcomeEscalated, Said: "budget spent", Begun: at}).Tried(); tr !=
|
||||||
|
(Tried{At: at, What: "asserted the grants", Outcome: "escalated: budget spent", By: "healer H3"}) {
|
||||||
|
t.Errorf("an attempt is tried as %+v", tr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The take-over key (clause 9): absent follows the controller; the keeper's names who chose; anything else refused.
|
||||||
|
for _, c := range []struct {
|
||||||
|
raw string
|
||||||
|
found bool
|
||||||
|
seat, events string
|
||||||
|
fails bool
|
||||||
|
}{
|
||||||
|
{found: false, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"},
|
||||||
|
{raw: `{"v":1,"follow":"controller","by":"the operator"}`, found: true, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"},
|
||||||
|
{raw: `{"v":1,"follow":"keeper","by":"the operator","ask":"ask-1"}`, found: true, seat: "mesh-conditions", events: "mesh-self-check.checked"},
|
||||||
|
{raw: `{"v":1,"follow":"keeper"}`, found: true, fails: true},
|
||||||
|
{raw: `{"v":1,"follow":"both","by":"x"}`, found: true, fails: true},
|
||||||
|
{raw: `{"v":2,"follow":"keeper","by":"x"}`, found: true, fails: true},
|
||||||
|
} {
|
||||||
|
got, err := DecodeTakeover([]byte(c.raw), c.found)
|
||||||
|
if c.fails != (err != nil) {
|
||||||
|
t.Errorf("%q: read as %+v, %v", c.raw, got, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c.fails {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seat, event := got.SelfCheckEvent()
|
||||||
|
if got.Seat() != c.seat || seat+"."+event != c.events {
|
||||||
|
t.Errorf("%q follows %s and %s.%s", c.raw, got.Seat(), seat, event)
|
||||||
|
}
|
||||||
|
if b, err := EncodeTakeover(got); err != nil || !strings.Contains(string(b), `"v":1`) {
|
||||||
|
t.Errorf("%q encodes as %s, %v", c.raw, b, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,341 @@
|
|||||||
|
package health
|
||||||
|
|
||||||
|
// **What the observers write and the keeper reads** (novox/hq ADR 0564 clauses 2, 6, 7, 8 and 9; issue 1952): the
|
||||||
|
// layout of a finding, the self-check's `checked`, the healers' `healer-acted`, an act begun in `attempts` and one the
|
||||||
|
// healers would have taken, and the take-over key the readers follow. The keeper is the one writer of conditions; an
|
||||||
|
// observer writes findings in its own bucket and never another's (clause 2).
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The modules holding the four health seats, and the states between them, as `<module>_<state>` buckets.
|
||||||
|
const (
|
||||||
|
// KeeperModule is the condition keeper, holder of mesh-conditions.
|
||||||
|
KeeperModule = "mesh-conditions"
|
||||||
|
// SelfCheckModule holds mesh-self-check; HealersModule mesh-healers; WatchmanModule mesh-watchdogs.
|
||||||
|
SelfCheckModule = "mesh-self-check"
|
||||||
|
HealersModule = "mesh-healers"
|
||||||
|
WatchmanModule = "mesh-delivery-watchman"
|
||||||
|
|
||||||
|
// FindingsState is every observer's own state of findings, one key per finding (clause 2).
|
||||||
|
FindingsState = "findings"
|
||||||
|
// ConditionsState is the keeper's open conditions, one key each, without their looks (clause 3).
|
||||||
|
ConditionsState = "conditions"
|
||||||
|
// TakeoverState is the keeper's take-over, one key TakeoverKey (clause 9).
|
||||||
|
TakeoverState = "takeover"
|
||||||
|
// TakeoverKey is the take-over state's one key.
|
||||||
|
TakeoverKey = "switch"
|
||||||
|
// AttemptsState is the healers' acts begun, each created before it is made (clause 7); WouldState the acts they
|
||||||
|
// would have taken while the lease names a controller (clause 8), which the keeper never reads.
|
||||||
|
AttemptsState = "attempts"
|
||||||
|
WouldState = "would"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Bucket is a module's state as the bus names it, `<module>_<state>`.
|
||||||
|
func Bucket(module, state string) string { return module + "_" + state }
|
||||||
|
|
||||||
|
// FindingsBucket is an observer's bucket of findings.
|
||||||
|
func FindingsBucket(observer string) string { return Bucket(observer, FindingsState) }
|
||||||
|
|
||||||
|
// FindingVersion is the version of a finding's layout.
|
||||||
|
const FindingVersion = 1
|
||||||
|
|
||||||
|
// Finding is one thing an observer sees wrong now, kept as the key Key in its bucket `findings` until it no longer sees
|
||||||
|
// it (clause 2): written only when what it says changes (ADR 0289 §1, §4), deleted when it goes. A key written raises or
|
||||||
|
// changes the condition of the same key; a key deleted clears it. The keeper makes the condition from it: its words,
|
||||||
|
// severity, subject and source unchanged, its evidence kept newest first.
|
||||||
|
type Finding struct {
|
||||||
|
V int `json:"v"`
|
||||||
|
// Key is Key(Subject.Scope, Subject.ID, Token or Kind): the condition's key.
|
||||||
|
Key string `json:"key"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
// Token is the key's last part when it is not Kind (`silent` for a machine).
|
||||||
|
Token string `json:"token,omitempty"`
|
||||||
|
Subject Subject `json:"subject"`
|
||||||
|
Severity Severity `json:"severity"`
|
||||||
|
// Summary is one line in the mesh's words; Headline, Explanation, Resolved and Needs the plain words of ADR 0253
|
||||||
|
// and Actions the answers a notification offers.
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
Headline string `json:"headline"`
|
||||||
|
Explanation string `json:"explanation"`
|
||||||
|
Resolved string `json:"resolved"`
|
||||||
|
Needs string `json:"needs,omitempty"`
|
||||||
|
Actions []Action `json:"actions,omitempty"`
|
||||||
|
// Evidence is what the observer saw, newest first, at most KeptEvidence. Detail goes here, never in Summary.
|
||||||
|
Evidence []Evidence `json:"evidence"`
|
||||||
|
// Source is the row, probe or event that saw it: `S1`, `D6`, `apply-failing`.
|
||||||
|
Source string `json:"source"`
|
||||||
|
// Resolver is who resolves it, ResolverSelf when empty.
|
||||||
|
Resolver string `json:"resolver,omitempty"`
|
||||||
|
// Observer is the module that wrote it, and Run what of its own saw it last (a self-check run's id, empty for a
|
||||||
|
// watch): the condition's looks are its finding's and its observer's last run (clause 2).
|
||||||
|
Observer string `json:"observer"`
|
||||||
|
Run string `json:"run,omitempty"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindingKey is the key of a finding about scope and id of kind (the token): the condition's key rule.
|
||||||
|
func FindingKey(scope, id, token string) string { return Key(scope, id, token) }
|
||||||
|
|
||||||
|
func (f Finding) token() string {
|
||||||
|
if f.Token != "" {
|
||||||
|
return f.Token
|
||||||
|
}
|
||||||
|
return f.Kind
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check refuses a finding the keeper could not make a condition of, naming why.
|
||||||
|
func (f Finding) Check() error {
|
||||||
|
known := false
|
||||||
|
for _, s := range Scopes {
|
||||||
|
known = known || s == f.Subject.Scope
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case !known:
|
||||||
|
return fmt.Errorf("health: a finding's scope is one of %s, not %q", strings.Join(Scopes, ", "), f.Subject.Scope)
|
||||||
|
case strings.TrimSpace(f.Subject.ID) == "":
|
||||||
|
return fmt.Errorf("health: a %s finding names what it is about", f.Subject.Scope)
|
||||||
|
case strings.TrimSpace(f.Kind) == "":
|
||||||
|
return fmt.Errorf("health: the finding %s has no kind", f.Key)
|
||||||
|
case f.Key != Key(f.Subject.Scope, f.Subject.ID, f.token()):
|
||||||
|
return fmt.Errorf("health: the finding's key %q is not %q, its scope, id and kind", f.Key,
|
||||||
|
Key(f.Subject.Scope, f.Subject.ID, f.token()))
|
||||||
|
case f.Severity != Urgent && f.Severity != Warning:
|
||||||
|
return fmt.Errorf("health: the finding %s is urgent or a warning, not %q", f.Key, f.Severity)
|
||||||
|
case strings.TrimSpace(f.Summary) == "":
|
||||||
|
return fmt.Errorf("health: the finding %s says nothing", f.Key)
|
||||||
|
case strings.TrimSpace(f.Source) == "":
|
||||||
|
return fmt.Errorf("health: the finding %s does not say what saw it", f.Key)
|
||||||
|
case strings.TrimSpace(f.Observer) == "":
|
||||||
|
return fmt.Errorf("health: the finding %s does not name its observer", f.Key)
|
||||||
|
case len(f.Evidence) > KeptEvidence:
|
||||||
|
return fmt.Errorf("health: the finding %s keeps %d observations, at most %d", f.Key, len(f.Evidence), KeptEvidence)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeFinding is f as JSON at FindingVersion; it refuses what Check refuses.
|
||||||
|
func EncodeFinding(f Finding) ([]byte, error) {
|
||||||
|
f.V = FindingVersion
|
||||||
|
if f.Evidence == nil {
|
||||||
|
f.Evidence = []Evidence{}
|
||||||
|
}
|
||||||
|
if err := f.Check(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return json.Marshal(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeFinding reads a finding's value; it refuses another version and what Check refuses.
|
||||||
|
func DecodeFinding(data []byte) (Finding, error) {
|
||||||
|
var f Finding
|
||||||
|
if err := json.Unmarshal(data, &f); err != nil {
|
||||||
|
return Finding{}, fmt.Errorf("health: a finding is not readable: %w", err)
|
||||||
|
}
|
||||||
|
if f.V != FindingVersion {
|
||||||
|
return Finding{}, fmt.Errorf("health: a finding of version %d, and this reads %d", f.V, FindingVersion)
|
||||||
|
}
|
||||||
|
if err := f.Check(); err != nil {
|
||||||
|
return Finding{}, err
|
||||||
|
}
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// EventChecked is the self-check's event, ending every run (clause 6), with the body of the controller's
|
||||||
|
// `doctor-heartbeat`, which mesh-watcher hears in its place.
|
||||||
|
const EventChecked = "checked"
|
||||||
|
|
||||||
|
// The verdicts of a probe in a run.
|
||||||
|
const (
|
||||||
|
VerdictPassed = "passed"
|
||||||
|
VerdictFailed = "failed"
|
||||||
|
VerdictFailedToRun = "failed-to-run"
|
||||||
|
VerdictDeferred = "deferred"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProbeVerdict is one probe's outcome in a run (mesh-controller doctor.go probeVerdict).
|
||||||
|
type ProbeVerdict struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Verdict string `json:"verdict"`
|
||||||
|
Found []string `json:"found,omitempty"`
|
||||||
|
// Unconfirmed are findings one look can be wrong about, seen by this run and not the one before.
|
||||||
|
Unconfirmed []string `json:"unconfirmed,omitempty"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Took string `json:"took,omitempty"`
|
||||||
|
// FoundCount and More are an overview's, never a checked event's.
|
||||||
|
FoundCount int `json:"found-count,omitempty"`
|
||||||
|
More string `json:"more,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckedCounts are a run's verdicts, counted.
|
||||||
|
type CheckedCounts struct {
|
||||||
|
Passed int `json:"passed"`
|
||||||
|
Failed int `json:"failed"`
|
||||||
|
FailedToRun int `json:"failed-to-run"`
|
||||||
|
Deferred int `json:"deferred"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checked is one run of the self-check and the body of `checked`, the controller's doctorRun (doctor.go:230) unchanged.
|
||||||
|
type Checked struct {
|
||||||
|
Run string `json:"run"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Took string `json:"took"`
|
||||||
|
IntervalSeconds int `json:"interval-seconds"`
|
||||||
|
Counts CheckedCounts `json:"counts"`
|
||||||
|
Probes []ProbeVerdict `json:"probes"`
|
||||||
|
// Controller is the machine that ran it (the name kept for mesh-watcher, which reads it), and Why what started it.
|
||||||
|
Controller string `json:"controller"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
Unsaid int `json:"unsaid,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EventHealerActed is the healers' event, one per act (clause 7).
|
||||||
|
const EventHealerActed = "healer-acted"
|
||||||
|
|
||||||
|
// The outcomes of an act.
|
||||||
|
const (
|
||||||
|
// OutcomeActing is an act begun and not yet finished: counted against the budget and the brake from its begin.
|
||||||
|
OutcomeActing = "acting"
|
||||||
|
// OutcomeActed is an act that did what it does; whether it repaired anything is the condition's clearing to say.
|
||||||
|
OutcomeActed = "acted"
|
||||||
|
// OutcomeFailed is an act that could not be done.
|
||||||
|
OutcomeFailed = "failed"
|
||||||
|
// OutcomeEscalated is a budget spent, said: the condition handed to the operator.
|
||||||
|
OutcomeEscalated = "escalated"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HealerActed is the body of `healer-acted`, the controller's (healers.go:345) unchanged; mesh-healers leaves Epoch out
|
||||||
|
// (ADR 0564 Limits 10).
|
||||||
|
type HealerActed struct {
|
||||||
|
Event string `json:"event"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Healer string `json:"healer"`
|
||||||
|
By string `json:"by"`
|
||||||
|
Condition string `json:"condition"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Act string `json:"act"`
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
Said string `json:"said"`
|
||||||
|
Budget string `json:"budget"`
|
||||||
|
Epoch uint64 `json:"epoch,omitempty"`
|
||||||
|
Show string `json:"show"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt is one act of a healer as kept in the healers' state `attempts` (clause 7): created before it is made, so a
|
||||||
|
// holder that dies mid-act has spent it and a second instance cannot begin the same act, and finished after. The key is
|
||||||
|
// the healers' own; the keeper reads every value and writes the condition named its `tried`, resolver and escalation.
|
||||||
|
type Attempt struct {
|
||||||
|
Healer string `json:"healer"`
|
||||||
|
Condition string `json:"condition"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
// BudgetKey is what the budget counts by: the condition's key unless the healer says another.
|
||||||
|
BudgetKey string `json:"budget-key"`
|
||||||
|
Act string `json:"act"`
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
Said string `json:"said,omitempty"`
|
||||||
|
Begun time.Time `json:"begun"`
|
||||||
|
Finished *time.Time `json:"finished,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tried is the attempt as the condition carries it in `tried`.
|
||||||
|
func (a Attempt) Tried() Tried {
|
||||||
|
outcome := a.Outcome
|
||||||
|
if a.Said != "" {
|
||||||
|
outcome += ": " + a.Said
|
||||||
|
}
|
||||||
|
return Tried{At: a.Begun, What: a.Act, Outcome: outcome, By: "healer " + a.Healer}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Would is an act mesh-healers would have taken while the controller's lease names a controller, kept in `would`: it
|
||||||
|
// counts for no budget, settle or brake and reaches no condition (clause 8).
|
||||||
|
type Would struct {
|
||||||
|
Healer string `json:"healer"`
|
||||||
|
Condition string `json:"condition"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Act string `json:"act"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// Why says why it was not taken: the lease names a controller, which heals.
|
||||||
|
Why string `json:"why"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Which source the readers of conditions follow (clause 9).
|
||||||
|
const (
|
||||||
|
FollowController = "controller"
|
||||||
|
FollowKeeper = "keeper"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TakeoverVersion is the version of the take-over key's value.
|
||||||
|
const TakeoverVersion = 1
|
||||||
|
|
||||||
|
// Takeover is the value of the keeper's key `takeover.switch` (clause 9): which source the router, the grants part and
|
||||||
|
// mesh-watcher hear, the controller's condition events and `doctor-heartbeat`, or the keeper's events and the
|
||||||
|
// self-check's `checked`. Written by the keeper alone, on the operator's warrant for a take-over and by `hand-back`. An
|
||||||
|
// absent key is FollowController.
|
||||||
|
type Takeover struct {
|
||||||
|
V int `json:"v"`
|
||||||
|
Follow string `json:"follow"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// By is who chose (the operator, as the warrant proves), and Why why; Ask the approve ask it was chosen on.
|
||||||
|
By string `json:"by"`
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
Ask string `json:"ask,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeTakeover is t as JSON at TakeoverVersion; it refuses a source the readers could not follow.
|
||||||
|
func EncodeTakeover(t Takeover) ([]byte, error) {
|
||||||
|
t.V = TakeoverVersion
|
||||||
|
if err := t.check(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return json.Marshal(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeTakeover reads the key as a getter answered it: an absent key follows the controller.
|
||||||
|
func DecodeTakeover(data []byte, found bool) (Takeover, error) {
|
||||||
|
if !found || len(data) == 0 {
|
||||||
|
return Takeover{V: TakeoverVersion, Follow: FollowController}, nil
|
||||||
|
}
|
||||||
|
var t Takeover
|
||||||
|
if err := json.Unmarshal(data, &t); err != nil {
|
||||||
|
return Takeover{}, fmt.Errorf("health: the take-over key is not readable: %w", err)
|
||||||
|
}
|
||||||
|
if t.V != TakeoverVersion {
|
||||||
|
return Takeover{}, fmt.Errorf("health: the take-over key is of version %d, and this reads %d", t.V, TakeoverVersion)
|
||||||
|
}
|
||||||
|
if err := t.check(); err != nil {
|
||||||
|
return Takeover{}, err
|
||||||
|
}
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t Takeover) check() error {
|
||||||
|
if t.Follow != FollowController && t.Follow != FollowKeeper {
|
||||||
|
return fmt.Errorf("health: the take-over follows %q, not %s or %s", t.Follow, FollowController, FollowKeeper)
|
||||||
|
}
|
||||||
|
if t.Follow == FollowKeeper && strings.TrimSpace(t.By) == "" {
|
||||||
|
return fmt.Errorf("health: a take-over by the keeper names who chose it")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seat is the seat a reader hears condition events on while it follows t: mesh-controller or mesh-conditions.
|
||||||
|
func (t Takeover) Seat() string {
|
||||||
|
if t.Follow == FollowKeeper {
|
||||||
|
return "mesh-conditions"
|
||||||
|
}
|
||||||
|
return "mesh-controller"
|
||||||
|
}
|
||||||
|
|
||||||
|
// SelfCheckEvent is the seat and event a reader of the self-check's run hears while it follows t: the controller's
|
||||||
|
// `doctor-heartbeat`, or mesh-self-check's `checked`.
|
||||||
|
func (t Takeover) SelfCheckEvent() (seat, event string) {
|
||||||
|
if t.Follow == FollowKeeper {
|
||||||
|
return "mesh-self-check", EventChecked
|
||||||
|
}
|
||||||
|
return "mesh-controller", "doctor-heartbeat"
|
||||||
|
}
|
||||||
+26
-29
@@ -52,15 +52,6 @@ func TestAWaitIsReadWithWhatItNames(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnAnswerWithoutStateIsReadFromHealthy(t *testing.T) {
|
|
||||||
for raw, want := range map[string]string{`{"healthy":true,"why":"ok"}`: StateHealthy, `{"healthy":false,"why":"down"}`: StateUnhealthy} {
|
|
||||||
got, err := Read([]byte(raw))
|
|
||||||
if err != nil || got.State != want {
|
|
||||||
t.Fatalf("%s read as %+v, %v", raw, got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAWaitThatCannotBeReadIsUnhealthyAndSaysWhy(t *testing.T) {
|
func TestAWaitThatCannotBeReadIsUnhealthyAndSaysWhy(t *testing.T) {
|
||||||
for name, c := range map[string]struct {
|
for name, c := range map[string]struct {
|
||||||
w []Wait
|
w []Wait
|
||||||
@@ -88,13 +79,6 @@ func TestAWaitThatCannotBeReadIsUnhealthyAndSaysWhy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnUnknownStateIsUnhealthy(t *testing.T) {
|
|
||||||
got, err := Read([]byte(`{"state":"fine","healthy":true}`))
|
|
||||||
if err != nil || got.State != StateUnhealthy || got.Healthy {
|
|
||||||
t.Fatalf("read as %+v, %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWaitsBesideAFaultAreSaidAndExcuseNothing(t *testing.T) {
|
func TestWaitsBesideAFaultAreSaidAndExcuseNothing(t *testing.T) {
|
||||||
got := roundTrip(t, Unhealthy("the share is unreachable", password, Wait{Part: "p", What: "w"}))
|
got := roundTrip(t, Unhealthy("the share is unreachable", password, Wait{Part: "p", What: "w"}))
|
||||||
if got.State != StateUnhealthy || len(got.Waits) != 1 {
|
if got.State != StateUnhealthy || len(got.Waits) != 1 {
|
||||||
@@ -111,12 +95,6 @@ func TestWaitingWithNothingIsUnhealthy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNotAnAnswerIsAnError(t *testing.T) {
|
|
||||||
if _, err := Read([]byte(`"yes"`)); err == nil {
|
|
||||||
t.Fatal("a string read as an answer")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEveryFormOfPathOrAddressIsRefused(t *testing.T) {
|
func TestEveryFormOfPathOrAddressIsRefused(t *testing.T) {
|
||||||
for _, words := range []string{
|
for _, words := range []string{
|
||||||
"the source (/mnt/games)", `the source "/mnt/games"`, "the folder ~/games", "the folder ./games",
|
"the source (/mnt/games)", `the source "/mnt/games"`, "the folder ~/games", "the folder ./games",
|
||||||
@@ -136,15 +114,34 @@ func TestEveryFormOfPathOrAddressIsRefused(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHealthyStateWithHealthyFalseIsUnhealthy(t *testing.T) {
|
|
||||||
got, err := Read([]byte(`{"state":"healthy","healthy":false,"why":"x"}`))
|
|
||||||
if err != nil || got.State != StateUnhealthy || !strings.Contains(got.Why, "contradict") {
|
|
||||||
t.Fatalf("read as %+v, %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWaitingChecksItsWaits(t *testing.T) {
|
func TestWaitingChecksItsWaits(t *testing.T) {
|
||||||
if a := Waiting("it waits", Wait{Part: "p", What: "w"}); a.State != StateUnhealthy || !strings.Contains(a.Why, "neither") {
|
if a := Waiting("it waits", Wait{Part: "p", What: "w"}); a.State != StateUnhealthy || !strings.Contains(a.Why, "neither") {
|
||||||
t.Fatalf("Waiting with a malformed wait is %+v", a)
|
t.Fatalf("Waiting with a malformed wait is %+v", a)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A raw answer is read by what it says: without state from healthy; an unknown state, or healthy with healthy false, as
|
||||||
|
// unhealthy; not an answer at all as an error.
|
||||||
|
func TestARawAnswerIsReadByWhatItSays(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
raw, state, says string
|
||||||
|
fails bool
|
||||||
|
}{
|
||||||
|
{raw: `{"healthy":true,"why":"ok"}`, state: StateHealthy},
|
||||||
|
{raw: `{"healthy":false,"why":"down"}`, state: StateUnhealthy},
|
||||||
|
{raw: `{"state":"fine","healthy":true}`, state: StateUnhealthy},
|
||||||
|
{raw: `{"state":"healthy","healthy":false,"why":"x"}`, state: StateUnhealthy, says: "contradict"},
|
||||||
|
{raw: `"yes"`, fails: true},
|
||||||
|
} {
|
||||||
|
got, err := Read([]byte(c.raw))
|
||||||
|
if c.fails {
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("%s read as an answer", c.raw)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil || got.State != c.state || (c.state == StateUnhealthy && got.Healthy) || !strings.Contains(got.Why, c.says) {
|
||||||
|
t.Errorf("%s read as %+v, %v", c.raw, got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+222
@@ -0,0 +1,222 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"actions": null,
|
||||||
|
"at": "2026-10-29T08:41:18.10610281Z",
|
||||||
|
"change": "raised",
|
||||||
|
"count": 1,
|
||||||
|
"epoch": 395230,
|
||||||
|
"event": "condition-raised",
|
||||||
|
"evidence": [
|
||||||
|
{
|
||||||
|
"at": "2026-10-29T08:41:18.10610281Z",
|
||||||
|
"said": "silent for 2m0s"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"explanation": "Nothing for you to do. The mesh has not heard from g14 for several minutes. A machine asleep or away is normal.",
|
||||||
|
"headline": "g14 is not answering",
|
||||||
|
"key": "machine.g14.silent",
|
||||||
|
"kind": "silent",
|
||||||
|
"last-observed": "2026-10-29T08:41:18.10610281Z",
|
||||||
|
"needs": "",
|
||||||
|
"observations": 1,
|
||||||
|
"raised": "2026-10-29T08:41:18.10610281Z",
|
||||||
|
"resolved": "g14 answers again",
|
||||||
|
"resolver": "self",
|
||||||
|
"severity": "warning",
|
||||||
|
"show": "mesh-controller.conditions key=machine.g14.silent",
|
||||||
|
"silenced": null,
|
||||||
|
"source": "S1",
|
||||||
|
"subject": {
|
||||||
|
"id": "g14",
|
||||||
|
"machine": "g14",
|
||||||
|
"scope": "machine"
|
||||||
|
},
|
||||||
|
"summary": "g14 has not been heard from since 2026-10-29 08:38 UTC (bound 3m0s)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"actions": null,
|
||||||
|
"at": "2026-10-28T20:59:53.719121257Z",
|
||||||
|
"change": "reopened",
|
||||||
|
"count": 2,
|
||||||
|
"epoch": 387175,
|
||||||
|
"event": "condition-raised",
|
||||||
|
"evidence": [
|
||||||
|
{
|
||||||
|
"at": "2026-10-28T20:59:53.719121257Z",
|
||||||
|
"said": "70 pending ([70 on EVENTS]), last taken 2026-10-28T20:57:25Z"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.",
|
||||||
|
"first": "2026-10-28T20:54:53.510073517Z",
|
||||||
|
"gaps": [
|
||||||
|
{
|
||||||
|
"cleared": "2026-10-28T20:56:52.499154879Z",
|
||||||
|
"reopened": "2026-10-28T20:59:53.719121257Z"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"headline": "The controller stopped listening",
|
||||||
|
"key": "core.controller.deaf",
|
||||||
|
"kind": "controller-deaf",
|
||||||
|
"last-observed": "2026-10-28T20:59:53.719121257Z",
|
||||||
|
"needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.",
|
||||||
|
"observations": 1,
|
||||||
|
"raised": "2026-10-28T20:59:53.719121257Z",
|
||||||
|
"resolved": "The controller listens again",
|
||||||
|
"resolver": "self",
|
||||||
|
"severity": "urgent",
|
||||||
|
"show": "mesh-controller.conditions key=core.controller.deaf",
|
||||||
|
"silenced": null,
|
||||||
|
"source": "S4",
|
||||||
|
"subject": {
|
||||||
|
"id": "controller",
|
||||||
|
"machine": "novox",
|
||||||
|
"scope": "core"
|
||||||
|
},
|
||||||
|
"summary": "the controller's event loop has taken nothing for 2m0s while its consumers hold 70 message(s): reports, builds and merges are not being acted on"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"actions": null,
|
||||||
|
"at": "2026-10-29T18:31:48.308721256Z",
|
||||||
|
"change": "resolver",
|
||||||
|
"count": 1,
|
||||||
|
"epoch": 401364,
|
||||||
|
"event": "condition-changed",
|
||||||
|
"evidence": [
|
||||||
|
{
|
||||||
|
"at": "2026-10-29T18:31:33.979579505Z",
|
||||||
|
"said": "no answer for node-files from g14 to the bus's discovery"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"explanation": "Nothing for you to do. Whatever serves node-files on g14 does not answer, so its tools do nothing there. A machine asleep is normal.",
|
||||||
|
"headline": "Node-files on g14 is not answering",
|
||||||
|
"key": "seat.node-files.g14.silent",
|
||||||
|
"kind": "holder-silent",
|
||||||
|
"last-observed": "2026-10-29T18:31:33.979579Z",
|
||||||
|
"needs": "",
|
||||||
|
"observations": 1,
|
||||||
|
"raised": "2026-10-29T18:31:33.979579505Z",
|
||||||
|
"resolved": "Node-files on g14 answers again",
|
||||||
|
"resolver": "healer:H3",
|
||||||
|
"severity": "warning",
|
||||||
|
"show": "mesh-controller.conditions key=seat.node-files.g14.silent",
|
||||||
|
"silenced": null,
|
||||||
|
"source": "D3",
|
||||||
|
"subject": {
|
||||||
|
"id": "node-files.g14",
|
||||||
|
"machine": "g14",
|
||||||
|
"scope": "seat"
|
||||||
|
},
|
||||||
|
"summary": "node-files's holder on g14 does not answer the bus: its verbs reach nothing there",
|
||||||
|
"tried": [
|
||||||
|
{
|
||||||
|
"at": "2026-10-29T18:31:48.308721256Z",
|
||||||
|
"by": "healer H3",
|
||||||
|
"outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)",
|
||||||
|
"what": "asserted the bus's streams, consumers and seat workers again"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"was": "self",
|
||||||
|
"why": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"actions": null,
|
||||||
|
"at": "2026-10-28T20:56:51.686413523Z",
|
||||||
|
"change": "severity",
|
||||||
|
"count": 1,
|
||||||
|
"epoch": 387175,
|
||||||
|
"event": "condition-changed",
|
||||||
|
"evidence": [
|
||||||
|
{
|
||||||
|
"at": "2026-10-28T20:55:13.087322533Z",
|
||||||
|
"said": "no answer for mesh-placement from ace to the bus's discovery"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-10-28T20:50:11.256396519Z",
|
||||||
|
"said": "no answer for mesh-placement from ace to the bus's discovery"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"explanation": "Needs you: the mesh tried to repair this and could not; read the details to decide what to do. Whatever serves mesh-placement on ace does not answer, so its tools do nothing there. A machine asleep is normal.",
|
||||||
|
"headline": "Mesh-placement on ace is not answering",
|
||||||
|
"key": "seat.mesh-placement.ace.silent",
|
||||||
|
"kind": "holder-silent",
|
||||||
|
"last-observed": "2026-10-28T20:55:13.087322Z",
|
||||||
|
"needs": "the mesh tried to repair this and could not; read the details to decide what to do.",
|
||||||
|
"observations": 2,
|
||||||
|
"raised": "2026-10-28T20:50:11.256396519Z",
|
||||||
|
"resolved": "Mesh-placement on ace answers again",
|
||||||
|
"resolver": "operator",
|
||||||
|
"severity": "urgent",
|
||||||
|
"show": "mesh-controller.conditions key=seat.mesh-placement.ace.silent",
|
||||||
|
"silenced": null,
|
||||||
|
"source": "D3",
|
||||||
|
"subject": {
|
||||||
|
"id": "mesh-placement.ace",
|
||||||
|
"machine": "ace",
|
||||||
|
"scope": "seat"
|
||||||
|
},
|
||||||
|
"summary": "mesh-placement's holder on ace does not answer the bus: its verbs reach nothing there",
|
||||||
|
"tried": [
|
||||||
|
{
|
||||||
|
"at": "2026-10-28T20:50:22.435049179Z",
|
||||||
|
"by": "healer H3",
|
||||||
|
"outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)",
|
||||||
|
"what": "asserted the bus's streams, consumers and seat workers again"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-10-28T20:56:51.686413523Z",
|
||||||
|
"by": "healer H3",
|
||||||
|
"outcome": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)",
|
||||||
|
"what": "handed to the operator: nothing in the mesh will repair it now"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"was": "warning",
|
||||||
|
"why": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"actions": null,
|
||||||
|
"at": "2026-10-28T21:18:48.630162912Z",
|
||||||
|
"change": "cleared",
|
||||||
|
"cleared": "2026-10-28T21:18:48.630162912Z",
|
||||||
|
"count": 2,
|
||||||
|
"epoch": 387463,
|
||||||
|
"event": "condition-cleared",
|
||||||
|
"evidence": [
|
||||||
|
{
|
||||||
|
"at": "2026-10-28T21:18:16.589232939Z",
|
||||||
|
"said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"at": "2026-10-28T21:17:46.778840937Z",
|
||||||
|
"said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.",
|
||||||
|
"first": "2026-10-28T21:13:16.626696927Z",
|
||||||
|
"gaps": [
|
||||||
|
{
|
||||||
|
"cleared": "2026-10-28T21:13:47.735189608Z",
|
||||||
|
"reopened": "2026-10-28T21:17:46.778840937Z"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"headline": "The controller stopped listening",
|
||||||
|
"key": "core.controller.deaf",
|
||||||
|
"kind": "controller-deaf",
|
||||||
|
"last-observed": "2026-10-28T21:18:16.589232Z",
|
||||||
|
"needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.",
|
||||||
|
"observations": 2,
|
||||||
|
"raised": "2026-10-28T21:17:46.778840937Z",
|
||||||
|
"resolved": "The controller listens again",
|
||||||
|
"resolver": "self",
|
||||||
|
"severity": "urgent",
|
||||||
|
"show": "mesh-controller.conditions key=core.controller.deaf",
|
||||||
|
"silenced": null,
|
||||||
|
"source": "S4",
|
||||||
|
"subject": {
|
||||||
|
"id": "controller",
|
||||||
|
"machine": "novox",
|
||||||
|
"scope": "core"
|
||||||
|
},
|
||||||
|
"summary": "the controller's event loop has taken nothing for 3m0s while its consumers hold 82 message(s): reports, builds and merges are not being acted on",
|
||||||
|
"why": "S4 no longer observes it"
|
||||||
|
}
|
||||||
|
]
|
||||||
Reference in New Issue
Block a user