mesh-sdk/go/health/condition.go

package health

// **A condition and its events, as the controller says them today** (novox/hq ADR 0564 clause 3, issue 1952): the body
// of `condition-raised`, `condition-changed` and `condition-cleared`, ported field for field from mesh-controller
// internal/conditions (condition.go, events.go, plain.go at 0fbb5ef3), so the condition keeper says them unchanged under
// the seat mesh-conditions and a reader (the router, the grants part, the healers) changes only the seat it hears.
//
// A condition is a durable fact about something the mesh owns that is wrong: raised when an observer's finding says so,
// kept with since-when, evidence and who can resolve it, said on the bus as it changes, and cleared when the finding
// goes, never by hand (ADR 0231 §14, ADR 0564 clause 2).

import (
	"regexp"
	"sort"
	"strings"
	"time"
)

// Severity is how soon the operator is needed: two levels, no more.
type Severity string

const (
	// Urgent needs the operator now.
	Urgent Severity = "urgent"
	// Warning needs the operator when they can.
	Warning Severity = "warning"
)

// The scopes a condition's key starts with: what kind of thing is wrong.
const (
	ScopeMachine  = "machine"
	ScopePlan     = "plan"
	ScopeCall     = "call"
	ScopeBuild    = "build"
	ScopeMerge    = "merge"
	ScopeProvider = "provider"
	ScopeSeat     = "seat"
	ScopeBus      = "bus"
	ScopeCore     = "core"
	ScopeProbe    = "probe"
	ScopeMesh     = "mesh"
	// ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239).
	ScopeDelivery = "delivery"
	// ScopeModule is a module on a machine, by `<module>.<machine>` (novox/hq ADR 0240).
	ScopeModule = "module"
)

// Scopes is every scope, in the order a person reads them.
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
	ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule}

// Who resolves a condition.
const (
	// ResolverSelf clears on observation: the signal returns, the probe passes.
	ResolverSelf = "self"
	// ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy.
	ResolverOperator = "operator"
	// ResolverAgent is work handed to an agent.
	ResolverAgent = "agent"
)

// ResolverHealer is the resolver of a condition a registered healer works on.
func ResolverHealer(name string) string { return "healer:" + name }

// Subject is what the condition is about: its scope, its id within the scope, and the machine it concerns when there is
// one.
type Subject struct {
	Scope   string `json:"scope"`
	ID      string `json:"id"`
	Machine string `json:"machine,omitempty"`
	// Also are the other machines it concerns: a consumer's, for a provider failing it.
	Also []string `json:"also,omitempty"`
}

// Evidence is one observation, as it was said.
type Evidence struct {
	At   time.Time `json:"at"`
	Said string    `json:"said"`
}

// Tried is one healer's try at a condition, as the condition carries it in `tried`: when, what it did, what came of it,
// and which healer (`healer H3`), never a person. The keeper writes it from the healers' attempts (ADR 0564 clause 7).
type Tried struct {
	At      time.Time `json:"at"`
	What    string    `json:"what"`
	Outcome string    `json:"outcome"`
	By      string    `json:"by"`
}

// Silence is a person saying they know: no messages until it ends. The condition stays open.
type Silence struct {
	Until time.Time `json:"until"`
	By    string    `json:"by"`
	Why   string    `json:"why"`
	Since time.Time `json:"since"`
}

// Gap is a stretch in which a fault was cleared, between its clearing and its reopening.
type Gap struct {
	Cleared  time.Time `json:"cleared"`
	Reopened time.Time `json:"reopened"`
}

// Action is one answer the operator may give from a notification: a label, and the seat verb the operator channel calls
// with these arguments when it is chosen (novox/hq ADR 0253). Level is how much proof its answer needs (ADR 0234 §8,
// ADR 0259).
type Action struct {
	Label     string            `json:"label"`
	Verb      string            `json:"verb"`
	Machine   string            `json:"machine,omitempty"`
	Arguments map[string]string `json:"arguments,omitempty"`
	Level     string            `json:"level,omitempty"`
}

// The assurance levels an action's answer needs (novox/hq ADR 0234 §8).
const (
	LevelAcknowledge = "acknowledge"
	LevelApprove     = "approve"
)

// The bounds a condition keeps, as the controller keeps them.
const (
	// KeptEvidence is how many observations a condition keeps, newest first.
	KeptEvidence = 10
	// KeptTried is how many tries a condition keeps, newest last.
	KeptTried = 10
	// KeptGaps is how many gaps a condition keeps.
	KeptGaps = 8
	// MaxSilence is the longest a condition may be silenced at once.
	MaxSilence = 7 * 24 * time.Hour
	// ReopenWithin is how soon after it cleared a condition raised again is the same one again, its count increased,
	// rather than news.
	ReopenWithin = 10 * time.Minute
)

// Condition is one open condition, as the keeper keeps it and its events carry it: the controller's body, field for
// field (ADR 0564 clause 3). The keeper writes Epoch 0 (Limits 10).
type Condition struct {
	Key         string     `json:"key"`
	Kind        string     `json:"kind"`
	Subject     Subject    `json:"subject"`
	Severity    Severity   `json:"severity"`
	Summary     string     `json:"summary"`
	Headline    string     `json:"headline"`
	Explanation string     `json:"explanation"`
	Resolved    string     `json:"resolved"`
	Needs       string     `json:"needs"`
	Actions     []Action   `json:"actions"`
	Evidence    []Evidence `json:"evidence"`
	// Source is the observer's row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
	Source       string    `json:"source"`
	Raised       time.Time `json:"raised"`
	LastObserved time.Time `json:"last-observed"`
	// First and Gaps are the fault's earlier life: a reopening within ReopenWithin is the same fault.
	First        time.Time `json:"first,omitzero"`
	Gaps         []Gap     `json:"gaps,omitempty"`
	Observations int       `json:"observations"`
	Count        int       `json:"count"`
	Tried        []Tried   `json:"tried,omitempty"`
	Resolver     string    `json:"resolver"`
	// Silenced is null when no silence is in force: said, not left out.
	Silenced *Silence `json:"silenced"`
	Epoch    uint64   `json:"epoch"`
	// LooksUnread says why Observations, LastObserved and Evidence are not known; the controller's alone.
	LooksUnread string `json:"looks-unread,omitempty"`
}

// SilencedAt says whether a person's silence is in force at a moment.
func (c Condition) SilencedAt(now time.Time) bool {
	return c.Silenced != nil && now.Before(c.Silenced.Until)
}

// Escalated says a healer tried and its budget is spent: the operator resolves it now.
func (c Condition) Escalated() bool { return c.Resolver == ResolverOperator && len(c.Tried) > 0 }

// Began is when this fault began as the mesh knows it: its first raising, a reopening within ReopenWithin being the
// same fault again.
func (c Condition) Began() time.Time {
	if !c.First.IsZero() && c.First.Before(c.Raised) {
		return c.First
	}
	return c.Raised
}

// ShowKeeper is the verb that shows more about a condition the keeper says (ADR 0564 Limits 9); a condition carried from
// the controller's bucket keeps the controller's (ShowController) until the lease write.
func ShowKeeper(key string) string { return "mesh-conditions.conditions key=" + key }

// ShowController is the controller's verb that shows more about a condition it raised.
func ShowController(key string) string { return "mesh-controller.conditions key=" + key }

// SilenceAction is the action that stops a condition's messages for a week, as the keeper offers it (ADR 0564 Limits 9:
// `mesh-conditions.silence`). An acknowledgement (ADR 0234 §8); its cause marks it as an answer.
func SilenceAction(key string) Action {
	return Action{Label: "Silence for a week", Verb: "mesh-conditions.silence", Level: LevelAcknowledge,
		Arguments: map[string]string{"key": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}}
}

// CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with.
const CauseOperatorAnswer = "operator-answer"

// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's tokens, and a `*` or `>`
// would make one a wildcard.
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)

// Key composes a condition's key from its parts, `<scope>.<id>.<token>`, the controller's rule (condition.go Key): each
// token made safe for the bus, a character the bus would refuse becoming `_`, an empty token `_`. The id may hold several
// tokens joined by dots. A finding and the condition it raises have one key.
func Key(scope, id, token string) string {
	var parts []string
	for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
		p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
		if p == "" {
			p = "_"
		}
		parts = append(parts, p)
	}
	return strings.Join(parts, ".")
}

// Order sorts conditions as `conditions` says them: urgent before warning, then oldest first, then by key.
func Order(list []Condition) {
	sort.SliceStable(list, func(i, j int) bool {
		if list[i].Severity != list[j].Severity {
			return list[i].Severity == Urgent
		}
		if !list[i].Raised.Equal(list[j].Raised) {
			return list[i].Raised.Before(list[j].Raised)
		}
		return list[i].Key < list[j].Key
	})
}

// The events a condition's life emits, under the seat that keeps it (ADR 0564 clause 3: the controller's names).
const (
	// EventRaised: a condition was raised, new or the same fault again within ReopenWithin (Change says which).
	EventRaised = "condition-raised"
	// EventChanged: its severity, its resolver, its silence or its words changed; not every observation.
	EventChanged = "condition-changed"
	// EventCleared: the finding went; the condition is removed and the transition kept in history.
	EventCleared = "condition-cleared"
)

// Events is every event a condition's life emits.
var Events = []string{EventRaised, EventChanged, EventCleared}

// What changed, as an event's Change and a history entry say it.
const (
	ChangeRaised     = "raised"
	ChangeReopened   = "reopened"
	ChangeSeverity   = "severity"
	ChangeResolver   = "resolver"
	ChangeSilenced   = "silenced"
	ChangeUnsilenced = "silence-ended"
	ChangeCleared    = "cleared"
	// ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR 0258).
	ChangeWords = "words"
)

// Event is the body of every condition event and the shape a history entry keeps: the condition itself at the top level,
// with what happened to it beside.
type Event struct {
	Condition
	// Event is the event's own name, so a body read without its subject still says what it is.
	Event string    `json:"event"`
	At    time.Time `json:"at"`
	// Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, words, cleared.
	Change string `json:"change"`
	// Was is the value before, for a severity or resolver change.
	Was string `json:"was,omitempty"`
	// Why says why it cleared, or why it was silenced.
	Why string `json:"why,omitempty"`
	// Cleared is when it cleared, on a clearing.
	Cleared *time.Time `json:"cleared,omitempty"`
	// Show is the verb that shows more.
	Show string `json:"show"`
}

// EventFor is the event a change is said under.
func EventFor(change string) string {
	switch change {
	case ChangeRaised, ChangeReopened:
		return EventRaised
	case ChangeCleared:
		return EventCleared
	}
	return EventChanged
}