mesh-controller/internal/conditions/condition.go

// Package conditions is the condition store (novox/hq to-be 45 §2, ADR 0227 rules 5 and 6).
//
// **A condition is a durable fact about something the mesh owns that is wrong.** Until this, every
// one of the forty-eight core failures of research 031 was noticed because a person or an agent
// looked: the mesh's own answers carried the fact for whoever asked, and told nobody. A condition is
// raised when an observation says something is wrong past its bound, kept with since-when, evidence
// and who can resolve it, said on the bus as it changes, and cleared when an observation says it is
// resolved — never by hand.
//
// The controller is the store's only writer (to-be 45 §1). Two of its processes may write at once —
// the serving controller's watchdogs, and a command a person runs to silence one — so every write
// is a compare-and-set on the key's revision, and a write that lost the race reads again and redoes
// itself rather than overwriting what the other said.
package conditions

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"time"
)

// Severity is how soon the operator is needed: two levels, no more (to-be 45 §2).
type Severity string

const (
	// Urgent needs the operator now.
	Urgent Severity = "urgent"
	// Warning needs the operator when they can.
	Warning Severity = "warning"
)

// The scopes a condition's key starts with: what kind of thing is wrong.
const (
	ScopeMachine  = "machine"
	ScopePlan     = "plan"
	ScopeCall     = "call"
	ScopeBuild    = "build"
	ScopeMerge    = "merge"
	ScopeProvider = "provider"
	ScopeSeat     = "seat"
	ScopeBus      = "bus"
	ScopeCore     = "core"
	ScopeProbe    = "probe"
	ScopeMesh     = "mesh"
	// ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239).
	ScopeDelivery = "delivery"
	// ScopeModule is a module on a machine, by `<module>.<machine>`: what it runs is not healthy there
	// (novox/hq ADR 0240).
	ScopeModule = "module"
)

// Scopes is every scope, in the order a person reads them.
var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
	ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule}

// Who resolves a condition.
const (
	// ResolverSelf clears on observation: the signal returns, the probe passes.
	ResolverSelf = "self"
	// ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy.
	ResolverOperator = "operator"
	// ResolverAgent is work handed to an agent (research 017; not raised by anything yet).
	ResolverAgent = "agent"
)

// ResolverHealer is the resolver of a condition a registered healer works on (Phase 3).
func ResolverHealer(name string) string { return "healer:" + name }

// Subject is what the condition is about: its scope, its id within the scope, and the machine it
// concerns when there is one.
type Subject struct {
	Scope   string `json:"scope"`
	ID      string `json:"id"`
	Machine string `json:"machine,omitempty"`
	// Also are the other machines it concerns: a consumer's, for a provider failing it.
	Also []string `json:"also,omitempty"`
}

// Evidence is one observation, as it was said.
type Evidence struct {
	At   time.Time `json:"at"`
	Said string    `json:"said"`
}

// Attempt is one healer's try at a condition (to-be 45 §7): when, what it did, what came of it, and
// which healer — so a person reading `conditions show` sees the mesh repairing itself, by whom.
type Attempt struct {
	At      time.Time `json:"at"`
	What    string    `json:"what"`
	Outcome string    `json:"outcome"`
	// By is the healer, as a person reads it: `healer H1`. Never a person: an act by hand is the
	// hand-act log's, not a condition's attempt.
	By string `json:"by"`
}

// KeptAttempts is how many attempts a condition keeps, newest last: a healer's budget is a few, and
// a condition reopened again and again carries its tries forward only so far.
const KeptAttempts = 10

// Silence is a person saying they know: no messages until it ends (to-be 45 §2). Recorded as a hand
// act; the condition stays open, and `status` still says it.
type Silence struct {
	Until time.Time `json:"until"`
	By    string    `json:"by"`
	Why   string    `json:"why"`
	Since time.Time `json:"since"`
}

// KeptEvidence is how many observations a condition keeps, newest first.
const KeptEvidence = 10

// MaxSilence is the longest a condition may be silenced at once: past it, a person says so again.
const MaxSilence = 7 * 24 * time.Hour

// ReopenWithin is how soon after it cleared a condition raised again is the same one again, with its
// count increased, rather than news (to-be 45 §2).
const ReopenWithin = 10 * time.Minute

// Condition is one open condition, as the store keeps it and its events carry it.
type Condition struct {
	Key string `json:"key"`
	// Kind is the condition kind: from the signals table, the probe registry or an event kind.
	Kind     string   `json:"kind"`
	Subject  Subject  `json:"subject"`
	Severity Severity `json:"severity"`
	// Summary is one line in the mesh's words, for whoever looks closer: it may name plans, commits and
	// the verbs that act.
	Summary string `json:"summary"`
	// Headline, Explanation and Resolved are what the operator reads, in plain words (novox/hq ADR 0253,
	// plain.go): a few words naming the thing and what is wrong; one or two sentences on what happened,
	// what it means and whether to act; the one line said when it clears.
	Headline    string `json:"headline"`
	Explanation string `json:"explanation"`
	Resolved    string `json:"resolved"`
	// Needs is what the operator does about it, one sentence; empty when nothing (the explanation then
	// opens "Nothing for you to do."). Actions are the answers a notification offers (plain.go).
	Needs   string   `json:"needs"`
	Actions []Action `json:"actions"`
	// Evidence is the newest observations, at most KeptEvidence, newest first.
	Evidence []Evidence `json:"evidence"`
	// Source is the signals-table row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`.
	Source string `json:"source"`
	// Raised is when it was first observed this time; LastObserved the newest observation.
	Raised       time.Time `json:"raised"`
	LastObserved time.Time `json:"last-observed"`
	// First is when this fault was first raised, a reopening within ReopenWithin counted as the same
	// fault; Gaps are the stretches between, each from a clearing to the reopening after it. Absent on a
	// first raising, and on one written before they were kept. What asks whether the fault was there at a
	// moment — the gate, at a send — reads OpenAt, never Raised (novox/hq issue 348): a fault cleared and
	// reopened after a send was there at the send unless the send fell in one of its gaps.
	First time.Time `json:"first,omitzero"`
	Gaps  []Gap     `json:"gaps,omitempty"`
	// Observations is how many times it was observed since raised: a look that sees it, for a source
	// that looks at the present; for one that reads a record of what happened (Observation.Happened),
	// how many times it happened, and LastObserved when it last did (novox/hq issue 402).
	Observations int `json:"observations"`
	// Count is how many times it has been raised, a reopening within ReopenWithin counted.
	Count    int       `json:"count"`
	Tried    []Attempt `json:"tried,omitempty"`
	Resolver string    `json:"resolver"`
	// Silenced is null when no silence is in force: said, not left out, so a reader need not guess.
	Silenced *Silence `json:"silenced"`
	// Epoch is the controller lease epoch that last wrote it (to-be 45 §6). Zero where it was written
	// by a controller serving without the lease, or before the lease existed.
	Epoch uint64 `json:"epoch"`
}

// SilencedAt says whether a person's silence is in force at a moment.
func (c Condition) SilencedAt(now time.Time) bool {
	return c.Silenced != nil && now.Before(c.Silenced.Until)
}

// Escalated says a healer tried and its budget is spent: the operator resolves it now (to-be 45 §2,
// "budget spent ──► OPEN, resolver: operator, severity: urgent"). An observation does not lower its
// severity again: the watchdog that sees it every half minute would otherwise undo the escalation.
func (c Condition) Escalated() bool { return c.Resolver == ResolverOperator && len(c.Tried) > 0 }

// Show is the verb that shows more about a condition, as a message carries it.
func (c Condition) Show() string { return "mesh-controller.conditions key=" + c.Key }

// Observation is one watchdog, probe or event saying something is wrong now.
type Observation struct {
	Scope string
	// ID is the thing within the scope; several tokens joined by dots where the thing is named by
	// several (a provider's module, its machine and the consumer).
	ID string
	// Token is the last part of the key, short for the kind: `silent` for a machine, `failing` for a
	// provider. Kind's own word when empty.
	Token   string
	Kind    string
	Machine string
	// Also are the other machines it concerns.
	Also     []string
	Severity Severity
	Summary  string
	// Headline, Explanation and Resolved are the plain words the operator reads (plain.go). Left empty,
	// the wording registered for Kind says them.
	Headline    string
	Explanation string
	Resolved    string
	Needs       string
	Actions     []Action
	// Said is this observation's evidence, in the mesh's words; Summary when empty. **Detail goes
	// here, never in Summary**: an address, a socket's error, a path or a name with its domain is
	// kept in the condition's evidence, which stays inside the mesh. The summary leaves it — to the
	// operator's channel, whose content rule withholds a message that carries any of them (ADR 0234
	// §6), and names machines in words.
	Said     string
	Source   string
	Resolver string
	// Happened is set by a source that reads a record of what happened rather than looking at the
	// present — a bus advisory, remembered for an hour and read again on every look — to when it last
	// happened; Times is how many times the record counts it. The condition then counts what happened,
	// not the looks (novox/hq issue 402): one refusal looked at every half minute read as "observed 15
	// time(s)", with a line of evidence every 30 seconds, and was taken for a refusal going on. A look
	// that brings nothing newer than the condition's last observation adds no observation and no
	// evidence. Zero for a source that looks at the present, whose every look is an observation.
	Happened time.Time
	Times    int
	// Confirm says a single look can be wrong about this finding — a question over the network that
	// went unanswered, a time measured once on a loaded machine. The keeper does not read it: the
	// source that looks again does, and raises it only when the next look sees it too, or while it is
	// already open (novox/hq issue 277).
	Confirm bool
}

// Key is where the observation's condition is kept: `<scope>.<id>.<kind>`, so the same fault said
// again is the same condition.
func (o Observation) Key() string {
	token := o.Token
	if token == "" {
		token = o.Kind
	}
	return Key(o.Scope, o.ID, token)
}

// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's
// tokens, and a `*` or `>` would make one a wildcard.
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)

// Key composes a condition's key from its parts, each token made safe for the bus: a character the
// bus would refuse becomes `_`, so a key is never refused for the name of the thing it is about.
func Key(scope, id, token string) string {
	var parts []string
	for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) {
		p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
		if p == "" {
			p = "_"
		}
		parts = append(parts, p)
	}
	return strings.Join(parts, ".")
}

// check refuses an observation that could not be said: a condition with no kind, no scope the mesh
// knows, or no severity is one nobody could route.
func (o Observation) check() error {
	known := false
	for _, s := range Scopes {
		if s == o.Scope {
			known = true
		}
	}
	switch {
	case !known:
		return fmt.Errorf("a condition's scope is one of %s, not %q", strings.Join(Scopes, ", "), o.Scope)
	case strings.TrimSpace(o.ID) == "":
		return fmt.Errorf("a %s condition names what it is about", o.Scope)
	case strings.TrimSpace(o.Kind) == "":
		return fmt.Errorf("the condition %s has no kind", o.Key())
	case o.Severity != Urgent && o.Severity != Warning:
		return fmt.Errorf("the condition %s is urgent or a warning, not %q", o.Key(), o.Severity)
	case strings.TrimSpace(o.Summary) == "":
		return fmt.Errorf("the condition %s says nothing", o.Key())
	case strings.TrimSpace(o.Source) == "":
		return fmt.Errorf("the condition %s does not say what raised it", o.Key())
	case o.Times != 0 && o.Happened.IsZero():
		// Times is what a record counts beside when it last happened: alone, the raise would ignore it
		// and an update count it, so the count would mean two things (novox/hq issue 440).
		return fmt.Errorf("the condition %s says how often it happened (%d) but not when", o.Key(), o.Times)
	}
	return nil
}

// Order sorts conditions as `status` says them: urgent before warning, then oldest first.
func Order(list []Condition) {
	sort.SliceStable(list, func(i, j int) bool {
		if list[i].Severity != list[j].Severity {
			return list[i].Severity == Urgent
		}
		if !list[i].Raised.Equal(list[j].Raised) {
			return list[i].Raised.Before(list[j].Raised)
		}
		return list[i].Key < list[j].Key
	})
}

// Gap is a stretch in which a fault was cleared, between its clearing and its reopening.
type Gap struct {
	Cleared  time.Time `json:"cleared"`
	Reopened time.Time `json:"reopened"`
}

// KeptGaps is how many gaps a condition keeps. Past it the oldest go, and the fault is said to have begun
// at the reopening after the newest of them: earlier history forgotten, never a fault said older than known.
const KeptGaps = 8

// Began is when this fault began as the mesh knows it: its first raising, a reopening within
// ReopenWithin being the same fault again (novox/hq issue 348).
func (c Condition) Began() time.Time {
	if !c.First.IsZero() && c.First.Before(c.Raised) {
		return c.First
	}
	return c.Raised
}

// OpenAt says the fault was there at t: it began at or before t, and t fell in none of its gaps. A fault
// that cleared before a send and came back after it was not there at the send — that is the send's to
// answer for — while one that flapped after the send was (novox/hq issue 348).
func (c Condition) OpenAt(t time.Time) bool {
	if t.Before(c.Began()) {
		return false
	}
	for _, g := range c.Gaps {
		if !t.Before(g.Cleared) && t.Before(g.Reopened) {
			return false
		}
	}
	return true
}

// reopen is c raised again at now after a clearing at cleared, of a fault that began at first with gaps:
// the same fault, with one more gap.
func (c *Condition) reopen(first time.Time, gaps []Gap, cleared, now time.Time) {
	if first.IsZero() || !first.Before(now) {
		return
	}
	c.First = first
	c.Gaps = append(append([]Gap(nil), gaps...), Gap{Cleared: cleared, Reopened: now})
	if over := len(c.Gaps) - KeptGaps; over > 0 {
		c.First = c.Gaps[over-1].Reopened
		c.Gaps = c.Gaps[over:]
	}
}