package health // **A condition and its events, as the controller says them today** (novox/hq ADR 0564 clause 3, issue 1952): the body // of `condition-raised`, `condition-changed` and `condition-cleared`, ported field for field from mesh-controller // internal/conditions (condition.go, events.go, plain.go at 0fbb5ef3), so the condition keeper says them unchanged under // the seat mesh-conditions and a reader (the router, the grants part, the healers) changes only the seat it hears. // // A condition is a durable fact about something the mesh owns that is wrong: raised when an observer's finding says so, // kept with since-when, evidence and who can resolve it, said on the bus as it changes, and cleared when the finding // goes, never by hand (ADR 0231 §14, ADR 0564 clause 2). import ( "regexp" "sort" "strings" "time" ) // Severity is how soon the operator is needed: two levels, no more. type Severity string const ( // Urgent needs the operator now. Urgent Severity = "urgent" // Warning needs the operator when they can. Warning Severity = "warning" ) // The scopes a condition's key starts with: what kind of thing is wrong. const ( ScopeMachine = "machine" ScopePlan = "plan" ScopeCall = "call" ScopeBuild = "build" ScopeMerge = "merge" ScopeProvider = "provider" ScopeSeat = "seat" ScopeBus = "bus" ScopeCore = "core" ScopeProbe = "probe" ScopeMesh = "mesh" // ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239). ScopeDelivery = "delivery" // ScopeModule is a module on a machine, by `.` (novox/hq ADR 0240). ScopeModule = "module" ) // Scopes is every scope, in the order a person reads them. var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider, ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule} // Who resolves a condition. const ( // ResolverSelf clears on observation: the signal returns, the probe passes. ResolverSelf = "self" // ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy. ResolverOperator = "operator" // ResolverAgent is work handed to an agent. ResolverAgent = "agent" ) // ResolverHealer is the resolver of a condition a registered healer works on. func ResolverHealer(name string) string { return "healer:" + name } // Subject is what the condition is about: its scope, its id within the scope, and the machine it concerns when there is // one. type Subject struct { Scope string `json:"scope"` ID string `json:"id"` Machine string `json:"machine,omitempty"` // Also are the other machines it concerns: a consumer's, for a provider failing it. Also []string `json:"also,omitempty"` } // Evidence is one observation, as it was said. type Evidence struct { At time.Time `json:"at"` Said string `json:"said"` } // Tried is one healer's try at a condition, as the condition carries it in `tried`: when, what it did, what came of it, // and which healer (`healer H3`), never a person. The keeper writes it from the healers' attempts (ADR 0564 clause 7). type Tried struct { At time.Time `json:"at"` What string `json:"what"` Outcome string `json:"outcome"` By string `json:"by"` } // Silence is a person saying they know: no messages until it ends. The condition stays open. type Silence struct { Until time.Time `json:"until"` By string `json:"by"` Why string `json:"why"` Since time.Time `json:"since"` } // Gap is a stretch in which a fault was cleared, between its clearing and its reopening. type Gap struct { Cleared time.Time `json:"cleared"` Reopened time.Time `json:"reopened"` } // Action is one answer the operator may give from a notification: a label, and the seat verb the operator channel calls // with these arguments when it is chosen (novox/hq ADR 0253). Level is how much proof its answer needs (ADR 0234 §8, // ADR 0259). type Action struct { Label string `json:"label"` Verb string `json:"verb"` Machine string `json:"machine,omitempty"` Arguments map[string]string `json:"arguments,omitempty"` Level string `json:"level,omitempty"` } // The assurance levels an action's answer needs (novox/hq ADR 0234 §8). const ( LevelAcknowledge = "acknowledge" LevelApprove = "approve" ) // The bounds a condition keeps, as the controller keeps them. const ( // KeptEvidence is how many observations a condition keeps, newest first. KeptEvidence = 10 // KeptTried is how many tries a condition keeps, newest last. KeptTried = 10 // KeptGaps is how many gaps a condition keeps. KeptGaps = 8 // MaxSilence is the longest a condition may be silenced at once. MaxSilence = 7 * 24 * time.Hour // ReopenWithin is how soon after it cleared a condition raised again is the same one again, its count increased, // rather than news. ReopenWithin = 10 * time.Minute ) // Condition is one open condition, as the keeper keeps it and its events carry it: the controller's body, field for // field (ADR 0564 clause 3). The keeper writes Epoch 0 (Limits 10). type Condition struct { Key string `json:"key"` Kind string `json:"kind"` Subject Subject `json:"subject"` Severity Severity `json:"severity"` Summary string `json:"summary"` Headline string `json:"headline"` Explanation string `json:"explanation"` Resolved string `json:"resolved"` Needs string `json:"needs"` Actions []Action `json:"actions"` Evidence []Evidence `json:"evidence"` // Source is the observer's row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`. Source string `json:"source"` Raised time.Time `json:"raised"` LastObserved time.Time `json:"last-observed"` // First and Gaps are the fault's earlier life: a reopening within ReopenWithin is the same fault. First time.Time `json:"first,omitzero"` Gaps []Gap `json:"gaps,omitempty"` Observations int `json:"observations"` Count int `json:"count"` Tried []Tried `json:"tried,omitempty"` Resolver string `json:"resolver"` // Silenced is null when no silence is in force: said, not left out. Silenced *Silence `json:"silenced"` Epoch uint64 `json:"epoch"` // LooksUnread says why Observations, LastObserved and Evidence are not known; the controller's alone. LooksUnread string `json:"looks-unread,omitempty"` } // SilencedAt says whether a person's silence is in force at a moment. func (c Condition) SilencedAt(now time.Time) bool { return c.Silenced != nil && now.Before(c.Silenced.Until) } // Escalated says a healer tried and its budget is spent: the operator resolves it now. func (c Condition) Escalated() bool { return c.Resolver == ResolverOperator && len(c.Tried) > 0 } // Began is when this fault began as the mesh knows it: its first raising, a reopening within ReopenWithin being the // same fault again. func (c Condition) Began() time.Time { if !c.First.IsZero() && c.First.Before(c.Raised) { return c.First } return c.Raised } // ShowKeeper is the verb that shows more about a condition the keeper says (ADR 0564 Limits 9); a condition carried from // the controller's bucket keeps the controller's (ShowController) until the lease write. func ShowKeeper(key string) string { return "mesh-conditions.conditions key=" + key } // ShowController is the controller's verb that shows more about a condition it raised. func ShowController(key string) string { return "mesh-controller.conditions key=" + key } // SilenceAction is the action that stops a condition's messages for a week, as the keeper offers it (ADR 0564 Limits 9: // `mesh-conditions.silence`). An acknowledgement (ADR 0234 §8); its cause marks it as an answer. func SilenceAction(key string) Action { return Action{Label: "Silence for a week", Verb: "mesh-conditions.silence", Level: LevelAcknowledge, Arguments: map[string]string{"key": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}} } // CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with. const CauseOperatorAnswer = "operator-answer" // unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's tokens, and a `*` or `>` // would make one a wildcard. var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) // Key composes a condition's key from its parts, `..`, the controller's rule (condition.go Key): each // token made safe for the bus, a character the bus would refuse becoming `_`, an empty token `_`. The id may hold several // tokens joined by dots. A finding and the condition it raises have one key. func Key(scope, id, token string) string { var parts []string for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) { p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_") if p == "" { p = "_" } parts = append(parts, p) } return strings.Join(parts, ".") } // Order sorts conditions as `conditions` says them: urgent before warning, then oldest first, then by key. func Order(list []Condition) { sort.SliceStable(list, func(i, j int) bool { if list[i].Severity != list[j].Severity { return list[i].Severity == Urgent } if !list[i].Raised.Equal(list[j].Raised) { return list[i].Raised.Before(list[j].Raised) } return list[i].Key < list[j].Key }) } // The events a condition's life emits, under the seat that keeps it (ADR 0564 clause 3: the controller's names). const ( // EventRaised: a condition was raised, new or the same fault again within ReopenWithin (Change says which). EventRaised = "condition-raised" // EventChanged: its severity, its resolver, its silence or its words changed; not every observation. EventChanged = "condition-changed" // EventCleared: the finding went; the condition is removed and the transition kept in history. EventCleared = "condition-cleared" ) // Events is every event a condition's life emits. var Events = []string{EventRaised, EventChanged, EventCleared} // What changed, as an event's Change and a history entry say it. const ( ChangeRaised = "raised" ChangeReopened = "reopened" ChangeSeverity = "severity" ChangeResolver = "resolver" ChangeSilenced = "silenced" ChangeUnsilenced = "silence-ended" ChangeCleared = "cleared" // ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR 0258). ChangeWords = "words" ) // Event is the body of every condition event and the shape a history entry keeps: the condition itself at the top level, // with what happened to it beside. type Event struct { Condition // Event is the event's own name, so a body read without its subject still says what it is. Event string `json:"event"` At time.Time `json:"at"` // Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, words, cleared. Change string `json:"change"` // Was is the value before, for a severity or resolver change. Was string `json:"was,omitempty"` // Why says why it cleared, or why it was silenced. Why string `json:"why,omitempty"` // Cleared is when it cleared, on a clearing. Cleared *time.Time `json:"cleared,omitempty"` // Show is the verb that shows more. Show string `json:"show"` } // EventFor is the event a change is said under. func EventFor(change string) string { switch change { case ChangeRaised, ChangeReopened: return EventRaised case ChangeCleared: return EventCleared } return EventChanged }