package health // **What the observers write and the keeper reads** (novox/hq ADR 0564 clauses 2, 6, 7, 8 and 9; issue 1952): the // layout of a finding, the self-check's `checked`, the healers' `healer-acted`, an act begun in `attempts` and one the // healers would have taken, and the take-over key the readers follow. The keeper is the one writer of conditions; an // observer writes findings in its own bucket and never another's (clause 2). import ( "encoding/json" "fmt" "strings" "time" ) // The modules holding the four health seats, and the states between them, as `_` buckets. const ( // KeeperModule is the condition keeper, holder of mesh-conditions. KeeperModule = "mesh-conditions" // SelfCheckModule holds mesh-self-check; HealersModule mesh-healers; WatchmanModule mesh-watchdogs. SelfCheckModule = "mesh-self-check" HealersModule = "mesh-healers" WatchmanModule = "mesh-delivery-watchman" // FindingsState is every observer's own state of findings, one key per finding (clause 2). FindingsState = "findings" // ConditionsState is the keeper's open conditions, one key each, without their looks (clause 3). ConditionsState = "conditions" // TakeoverState is the keeper's take-over, one key TakeoverKey (clause 9). TakeoverState = "takeover" // TakeoverKey is the take-over state's one key. TakeoverKey = "switch" // AttemptsState is the healers' acts begun, each created before it is made (clause 7); WouldState the acts they // would have taken while the lease names a controller (clause 8), which the keeper never reads. AttemptsState = "attempts" WouldState = "would" ) // Bucket is a module's state as the bus names it, `_`. func Bucket(module, state string) string { return module + "_" + state } // FindingsBucket is an observer's bucket of findings. func FindingsBucket(observer string) string { return Bucket(observer, FindingsState) } // FindingVersion is the version of a finding's layout. const FindingVersion = 1 // Finding is one thing an observer sees wrong now, kept as the key Key in its bucket `findings` until it no longer sees // it (clause 2): written only when what it says changes (ADR 0289 §1, §4), deleted when it goes. A key written raises or // changes the condition of the same key; a key deleted clears it. The keeper makes the condition from it: its words, // severity, subject and source unchanged, its evidence kept newest first. type Finding struct { V int `json:"v"` // Key is Key(Subject.Scope, Subject.ID, Token or Kind): the condition's key. Key string `json:"key"` Kind string `json:"kind"` // Token is the key's last part when it is not Kind (`silent` for a machine). Token string `json:"token,omitempty"` Subject Subject `json:"subject"` Severity Severity `json:"severity"` // Summary is one line in the mesh's words; Headline, Explanation, Resolved and Needs the plain words of ADR 0253 // and Actions the answers a notification offers. Summary string `json:"summary"` Headline string `json:"headline"` Explanation string `json:"explanation"` Resolved string `json:"resolved"` Needs string `json:"needs,omitempty"` Actions []Action `json:"actions,omitempty"` // Evidence is what the observer saw, newest first, at most KeptEvidence. Detail goes here, never in Summary. Evidence []Evidence `json:"evidence"` // Source is the row, probe or event that saw it: `S1`, `D6`, `apply-failing`. Source string `json:"source"` // Resolver is who resolves it, ResolverSelf when empty. Resolver string `json:"resolver,omitempty"` // Observer is the module that wrote it, and Run what of its own saw it last (a self-check run's id, empty for a // watch): the condition's looks are its finding's and its observer's last run (clause 2). Observer string `json:"observer"` Run string `json:"run,omitempty"` At time.Time `json:"at"` } // FindingKey is the key of a finding about scope and id of kind (the token): the condition's key rule. func FindingKey(scope, id, token string) string { return Key(scope, id, token) } func (f Finding) token() string { if f.Token != "" { return f.Token } return f.Kind } // Check refuses a finding the keeper could not make a condition of, naming why. func (f Finding) Check() error { known := false for _, s := range Scopes { known = known || s == f.Subject.Scope } switch { case !known: return fmt.Errorf("health: a finding's scope is one of %s, not %q", strings.Join(Scopes, ", "), f.Subject.Scope) case strings.TrimSpace(f.Subject.ID) == "": return fmt.Errorf("health: a %s finding names what it is about", f.Subject.Scope) case strings.TrimSpace(f.Kind) == "": return fmt.Errorf("health: the finding %s has no kind", f.Key) case f.Key != Key(f.Subject.Scope, f.Subject.ID, f.token()): return fmt.Errorf("health: the finding's key %q is not %q, its scope, id and kind", f.Key, Key(f.Subject.Scope, f.Subject.ID, f.token())) case f.Severity != Urgent && f.Severity != Warning: return fmt.Errorf("health: the finding %s is urgent or a warning, not %q", f.Key, f.Severity) case strings.TrimSpace(f.Summary) == "": return fmt.Errorf("health: the finding %s says nothing", f.Key) case strings.TrimSpace(f.Source) == "": return fmt.Errorf("health: the finding %s does not say what saw it", f.Key) case strings.TrimSpace(f.Observer) == "": return fmt.Errorf("health: the finding %s does not name its observer", f.Key) case len(f.Evidence) > KeptEvidence: return fmt.Errorf("health: the finding %s keeps %d observations, at most %d", f.Key, len(f.Evidence), KeptEvidence) } return nil } // EncodeFinding is f as JSON at FindingVersion; it refuses what Check refuses. func EncodeFinding(f Finding) ([]byte, error) { f.V = FindingVersion if f.Evidence == nil { f.Evidence = []Evidence{} } if err := f.Check(); err != nil { return nil, err } return json.Marshal(f) } // DecodeFinding reads a finding's value; it refuses another version and what Check refuses. func DecodeFinding(data []byte) (Finding, error) { var f Finding if err := json.Unmarshal(data, &f); err != nil { return Finding{}, fmt.Errorf("health: a finding is not readable: %w", err) } if f.V != FindingVersion { return Finding{}, fmt.Errorf("health: a finding of version %d, and this reads %d", f.V, FindingVersion) } if err := f.Check(); err != nil { return Finding{}, err } return f, nil } // EventChecked is the self-check's event, ending every run (clause 6), with the body of the controller's // `doctor-heartbeat`, which mesh-watcher hears in its place. const EventChecked = "checked" // The verdicts of a probe in a run. const ( VerdictPassed = "passed" VerdictFailed = "failed" VerdictFailedToRun = "failed-to-run" VerdictDeferred = "deferred" ) // ProbeVerdict is one probe's outcome in a run (mesh-controller doctor.go probeVerdict). type ProbeVerdict struct { ID string `json:"id"` Verdict string `json:"verdict"` Found []string `json:"found,omitempty"` // Unconfirmed are findings one look can be wrong about, seen by this run and not the one before. Unconfirmed []string `json:"unconfirmed,omitempty"` Error string `json:"error,omitempty"` Took string `json:"took,omitempty"` // FoundCount and More are an overview's, never a checked event's. FoundCount int `json:"found-count,omitempty"` More string `json:"more,omitempty"` } // CheckedCounts are a run's verdicts, counted. type CheckedCounts struct { Passed int `json:"passed"` Failed int `json:"failed"` FailedToRun int `json:"failed-to-run"` Deferred int `json:"deferred"` } // Checked is one run of the self-check and the body of `checked`, the controller's doctorRun (doctor.go:230) unchanged. type Checked struct { Run string `json:"run"` At time.Time `json:"at"` Started time.Time `json:"started"` Took string `json:"took"` IntervalSeconds int `json:"interval-seconds"` Counts CheckedCounts `json:"counts"` Probes []ProbeVerdict `json:"probes"` // Controller is the machine that ran it (the name kept for mesh-watcher, which reads it), and Why what started it. Controller string `json:"controller"` Why string `json:"why"` Unsaid int `json:"unsaid,omitempty"` } // EventHealerActed is the healers' event, one per act (clause 7). const EventHealerActed = "healer-acted" // The outcomes of an act. const ( // OutcomeActing is an act begun and not yet finished: counted against the budget and the brake from its begin. OutcomeActing = "acting" // OutcomeActed is an act that did what it does; whether it repaired anything is the condition's clearing to say. OutcomeActed = "acted" // OutcomeFailed is an act that could not be done. OutcomeFailed = "failed" // OutcomeEscalated is a budget spent, said: the condition handed to the operator. OutcomeEscalated = "escalated" ) // HealerActed is the body of `healer-acted`, the controller's (healers.go:345) unchanged; mesh-healers leaves Epoch out // (ADR 0564 Limits 10). type HealerActed struct { Event string `json:"event"` At time.Time `json:"at"` Healer string `json:"healer"` By string `json:"by"` Condition string `json:"condition"` Kind string `json:"kind"` Act string `json:"act"` Outcome string `json:"outcome"` Said string `json:"said"` Budget string `json:"budget"` Epoch uint64 `json:"epoch,omitempty"` Show string `json:"show"` } // Attempt is one act of a healer as kept in the healers' state `attempts` (clause 7): created before it is made, so a // holder that dies mid-act has spent it and a second instance cannot begin the same act, and finished after. The key is // the healers' own; the keeper reads every value and writes the condition named its `tried`, resolver and escalation. type Attempt struct { Healer string `json:"healer"` Condition string `json:"condition"` Kind string `json:"kind"` // BudgetKey is what the budget counts by: the condition's key unless the healer says another. BudgetKey string `json:"budget-key"` Act string `json:"act"` Outcome string `json:"outcome"` Said string `json:"said,omitempty"` Begun time.Time `json:"begun"` Finished *time.Time `json:"finished,omitempty"` } // Tried is the attempt as the condition carries it in `tried`. func (a Attempt) Tried() Tried { outcome := a.Outcome if a.Said != "" { outcome += ": " + a.Said } return Tried{At: a.Begun, What: a.Act, Outcome: outcome, By: "healer " + a.Healer} } // Would is an act mesh-healers would have taken while the controller's lease names a controller, kept in `would`: it // counts for no budget, settle or brake and reaches no condition (clause 8). type Would struct { Healer string `json:"healer"` Condition string `json:"condition"` Kind string `json:"kind"` Act string `json:"act"` At time.Time `json:"at"` // Why says why it was not taken: the lease names a controller, which heals. Why string `json:"why"` } // Which source the readers of conditions follow (clause 9). const ( FollowController = "controller" FollowKeeper = "keeper" ) // TakeoverVersion is the version of the take-over key's value. const TakeoverVersion = 1 // Takeover is the value of the keeper's key `takeover.switch` (clause 9): which source the router, the grants part and // mesh-watcher hear, the controller's condition events and `doctor-heartbeat`, or the keeper's events and the // self-check's `checked`. Written by the keeper alone, on the operator's warrant for a take-over and by `hand-back`. An // absent key is FollowController. type Takeover struct { V int `json:"v"` Follow string `json:"follow"` At time.Time `json:"at"` // By is who chose (the operator, as the warrant proves), and Why why; Ask the approve ask it was chosen on. By string `json:"by"` Why string `json:"why,omitempty"` Ask string `json:"ask,omitempty"` } // EncodeTakeover is t as JSON at TakeoverVersion; it refuses a source the readers could not follow. func EncodeTakeover(t Takeover) ([]byte, error) { t.V = TakeoverVersion if err := t.check(); err != nil { return nil, err } return json.Marshal(t) } // DecodeTakeover reads the key as a getter answered it: an absent key follows the controller. func DecodeTakeover(data []byte, found bool) (Takeover, error) { if !found || len(data) == 0 { return Takeover{V: TakeoverVersion, Follow: FollowController}, nil } var t Takeover if err := json.Unmarshal(data, &t); err != nil { return Takeover{}, fmt.Errorf("health: the take-over key is not readable: %w", err) } if t.V != TakeoverVersion { return Takeover{}, fmt.Errorf("health: the take-over key is of version %d, and this reads %d", t.V, TakeoverVersion) } if err := t.check(); err != nil { return Takeover{}, err } return t, nil } func (t Takeover) check() error { if t.Follow != FollowController && t.Follow != FollowKeeper { return fmt.Errorf("health: the take-over follows %q, not %s or %s", t.Follow, FollowController, FollowKeeper) } if t.Follow == FollowKeeper && strings.TrimSpace(t.By) == "" { return fmt.Errorf("health: a take-over by the keeper names who chose it") } return nil } // Seat is the seat a reader hears condition events on while it follows t: mesh-controller or mesh-conditions. func (t Takeover) Seat() string { if t.Follow == FollowKeeper { return "mesh-conditions" } return "mesh-controller" } // SelfCheckEvent is the seat and event a reader of the self-check's run hears while it follows t: the controller's // `doctor-heartbeat`, or mesh-self-check's `checked`. func (t Takeover) SelfCheckEvent() (seat, event string) { if t.Follow == FollowKeeper { return "mesh-self-check", EventChecked } return "mesh-controller", "doctor-heartbeat" }