mesh-sdk/go/health/findings.go

package health

// **What the observers write and the keeper reads** (novox/hq ADR 0564 clauses 2, 6, 7, 8 and 9; issue 1952): the
// layout of a finding, the self-check's `checked`, the healers' `healer-acted`, an act begun in `attempts` and one the
// healers would have taken, and the take-over key the readers follow. The keeper is the one writer of conditions; an
// observer writes findings in its own bucket and never another's (clause 2).

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"
)

// The modules holding the four health seats, and the states between them, as `<module>_<state>` buckets.
const (
	// KeeperModule is the condition keeper, holder of mesh-conditions.
	KeeperModule = "mesh-conditions"
	// SelfCheckModule holds mesh-self-check; HealersModule mesh-healers; WatchmanModule mesh-watchdogs.
	SelfCheckModule = "mesh-self-check"
	HealersModule   = "mesh-healers"
	WatchmanModule  = "mesh-delivery-watchman"

	// FindingsState is every observer's own state of findings, one key per finding (clause 2).
	FindingsState = "findings"
	// ConditionsState is the keeper's open conditions, one key each, without their looks (clause 3).
	ConditionsState = "conditions"
	// TakeoverState is the keeper's take-over, one key TakeoverKey (clause 9).
	TakeoverState = "takeover"
	// TakeoverKey is the take-over state's one key.
	TakeoverKey = "switch"
	// AttemptsState is the healers' acts begun, each created before it is made (clause 7); WouldState the acts they
	// would have taken while the lease names a controller (clause 8), which the keeper never reads.
	AttemptsState = "attempts"
	WouldState    = "would"
)

// Bucket is a module's state as the bus names it, `<module>_<state>`.
func Bucket(module, state string) string { return module + "_" + state }

// FindingsBucket is an observer's bucket of findings.
func FindingsBucket(observer string) string { return Bucket(observer, FindingsState) }

// FindingVersion is the version of a finding's layout.
const FindingVersion = 1

// Finding is one thing an observer sees wrong now, kept as the key Key in its bucket `findings` until it no longer sees
// it (clause 2): written only when what it says changes (ADR 0289 §1, §4), deleted when it goes. A key written raises or
// changes the condition of the same key; a key deleted clears it. The keeper makes the condition from it: its words,
// severity, subject and source unchanged, its evidence kept newest first.
type Finding struct {
	V int `json:"v"`
	// Key is Key(Subject.Scope, Subject.ID, Token or Kind): the condition's key.
	Key  string `json:"key"`
	Kind string `json:"kind"`
	// Token is the key's last part when it is not Kind (`silent` for a machine).
	Token    string   `json:"token,omitempty"`
	Subject  Subject  `json:"subject"`
	Severity Severity `json:"severity"`
	// Summary is one line in the mesh's words; Headline, Explanation, Resolved and Needs the plain words of ADR 0253
	// and Actions the answers a notification offers.
	Summary     string   `json:"summary"`
	Headline    string   `json:"headline"`
	Explanation string   `json:"explanation"`
	Resolved    string   `json:"resolved"`
	Needs       string   `json:"needs,omitempty"`
	Actions     []Action `json:"actions,omitempty"`
	// Evidence is what the observer saw, newest first, at most KeptEvidence. Detail goes here, never in Summary.
	Evidence []Evidence `json:"evidence"`
	// Source is the row, probe or event that saw it: `S1`, `D6`, `apply-failing`.
	Source string `json:"source"`
	// Resolver is who resolves it, ResolverSelf when empty.
	Resolver string `json:"resolver,omitempty"`
	// Observer is the module that wrote it, and Run what of its own saw it last (a self-check run's id, empty for a
	// watch): the condition's looks are its finding's and its observer's last run (clause 2).
	Observer string    `json:"observer"`
	Run      string    `json:"run,omitempty"`
	At       time.Time `json:"at"`
}

// FindingKey is the key of a finding about scope and id of kind (the token): the condition's key rule.
func FindingKey(scope, id, token string) string { return Key(scope, id, token) }

func (f Finding) token() string {
	if f.Token != "" {
		return f.Token
	}
	return f.Kind
}

// Check refuses a finding the keeper could not make a condition of, naming why.
func (f Finding) Check() error {
	known := false
	for _, s := range Scopes {
		known = known || s == f.Subject.Scope
	}
	switch {
	case !known:
		return fmt.Errorf("health: a finding's scope is one of %s, not %q", strings.Join(Scopes, ", "), f.Subject.Scope)
	case strings.TrimSpace(f.Subject.ID) == "":
		return fmt.Errorf("health: a %s finding names what it is about", f.Subject.Scope)
	case strings.TrimSpace(f.Kind) == "":
		return fmt.Errorf("health: the finding %s has no kind", f.Key)
	case f.Key != Key(f.Subject.Scope, f.Subject.ID, f.token()):
		return fmt.Errorf("health: the finding's key %q is not %q, its scope, id and kind", f.Key,
			Key(f.Subject.Scope, f.Subject.ID, f.token()))
	case f.Severity != Urgent && f.Severity != Warning:
		return fmt.Errorf("health: the finding %s is urgent or a warning, not %q", f.Key, f.Severity)
	case strings.TrimSpace(f.Summary) == "":
		return fmt.Errorf("health: the finding %s says nothing", f.Key)
	case strings.TrimSpace(f.Source) == "":
		return fmt.Errorf("health: the finding %s does not say what saw it", f.Key)
	case strings.TrimSpace(f.Observer) == "":
		return fmt.Errorf("health: the finding %s does not name its observer", f.Key)
	case len(f.Evidence) > KeptEvidence:
		return fmt.Errorf("health: the finding %s keeps %d observations, at most %d", f.Key, len(f.Evidence), KeptEvidence)
	}
	return nil
}

// EncodeFinding is f as JSON at FindingVersion; it refuses what Check refuses.
func EncodeFinding(f Finding) ([]byte, error) {
	f.V = FindingVersion
	if f.Evidence == nil {
		f.Evidence = []Evidence{}
	}
	if err := f.Check(); err != nil {
		return nil, err
	}
	return json.Marshal(f)
}

// DecodeFinding reads a finding's value; it refuses another version and what Check refuses.
func DecodeFinding(data []byte) (Finding, error) {
	var f Finding
	if err := json.Unmarshal(data, &f); err != nil {
		return Finding{}, fmt.Errorf("health: a finding is not readable: %w", err)
	}
	if f.V != FindingVersion {
		return Finding{}, fmt.Errorf("health: a finding of version %d, and this reads %d", f.V, FindingVersion)
	}
	if err := f.Check(); err != nil {
		return Finding{}, err
	}
	return f, nil
}

// EventChecked is the self-check's event, ending every run (clause 6), with the body of the controller's
// `doctor-heartbeat`, which mesh-watcher hears in its place.
const EventChecked = "checked"

// The verdicts of a probe in a run.
const (
	VerdictPassed      = "passed"
	VerdictFailed      = "failed"
	VerdictFailedToRun = "failed-to-run"
	VerdictDeferred    = "deferred"
)

// ProbeVerdict is one probe's outcome in a run (mesh-controller doctor.go probeVerdict).
type ProbeVerdict struct {
	ID      string   `json:"id"`
	Verdict string   `json:"verdict"`
	Found   []string `json:"found,omitempty"`
	// Unconfirmed are findings one look can be wrong about, seen by this run and not the one before.
	Unconfirmed []string `json:"unconfirmed,omitempty"`
	Error       string   `json:"error,omitempty"`
	Took        string   `json:"took,omitempty"`
	// FoundCount and More are an overview's, never a checked event's.
	FoundCount int    `json:"found-count,omitempty"`
	More       string `json:"more,omitempty"`
}

// CheckedCounts are a run's verdicts, counted.
type CheckedCounts struct {
	Passed      int `json:"passed"`
	Failed      int `json:"failed"`
	FailedToRun int `json:"failed-to-run"`
	Deferred    int `json:"deferred"`
}

// Checked is one run of the self-check and the body of `checked`, the controller's doctorRun (doctor.go:230) unchanged.
type Checked struct {
	Run             string         `json:"run"`
	At              time.Time      `json:"at"`
	Started         time.Time      `json:"started"`
	Took            string         `json:"took"`
	IntervalSeconds int            `json:"interval-seconds"`
	Counts          CheckedCounts  `json:"counts"`
	Probes          []ProbeVerdict `json:"probes"`
	// Controller is the machine that ran it (the name kept for mesh-watcher, which reads it), and Why what started it.
	Controller string `json:"controller"`
	Why        string `json:"why"`
	Unsaid     int    `json:"unsaid,omitempty"`
}

// EventHealerActed is the healers' event, one per act (clause 7).
const EventHealerActed = "healer-acted"

// The outcomes of an act.
const (
	// OutcomeActing is an act begun and not yet finished: counted against the budget and the brake from its begin.
	OutcomeActing = "acting"
	// OutcomeActed is an act that did what it does; whether it repaired anything is the condition's clearing to say.
	OutcomeActed = "acted"
	// OutcomeFailed is an act that could not be done.
	OutcomeFailed = "failed"
	// OutcomeEscalated is a budget spent, said: the condition handed to the operator.
	OutcomeEscalated = "escalated"
)

// HealerActed is the body of `healer-acted`, the controller's (healers.go:345) unchanged; mesh-healers leaves Epoch out
// (ADR 0564 Limits 10).
type HealerActed struct {
	Event     string    `json:"event"`
	At        time.Time `json:"at"`
	Healer    string    `json:"healer"`
	By        string    `json:"by"`
	Condition string    `json:"condition"`
	Kind      string    `json:"kind"`
	Act       string    `json:"act"`
	Outcome   string    `json:"outcome"`
	Said      string    `json:"said"`
	Budget    string    `json:"budget"`
	Epoch     uint64    `json:"epoch,omitempty"`
	Show      string    `json:"show"`
}

// Attempt is one act of a healer as kept in the healers' state `attempts` (clause 7): created before it is made, so a
// holder that dies mid-act has spent it and a second instance cannot begin the same act, and finished after. The key is
// the healers' own; the keeper reads every value and writes the condition named its `tried`, resolver and escalation.
type Attempt struct {
	Healer    string `json:"healer"`
	Condition string `json:"condition"`
	Kind      string `json:"kind"`
	// BudgetKey is what the budget counts by: the condition's key unless the healer says another.
	BudgetKey string     `json:"budget-key"`
	Act       string     `json:"act"`
	Outcome   string     `json:"outcome"`
	Said      string     `json:"said,omitempty"`
	Begun     time.Time  `json:"begun"`
	Finished  *time.Time `json:"finished,omitempty"`
}

// Tried is the attempt as the condition carries it in `tried`.
func (a Attempt) Tried() Tried {
	outcome := a.Outcome
	if a.Said != "" {
		outcome += ": " + a.Said
	}
	return Tried{At: a.Begun, What: a.Act, Outcome: outcome, By: "healer " + a.Healer}
}

// Would is an act mesh-healers would have taken while the controller's lease names a controller, kept in `would`: it
// counts for no budget, settle or brake and reaches no condition (clause 8).
type Would struct {
	Healer    string    `json:"healer"`
	Condition string    `json:"condition"`
	Kind      string    `json:"kind"`
	Act       string    `json:"act"`
	At        time.Time `json:"at"`
	// Why says why it was not taken: the lease names a controller, which heals.
	Why string `json:"why"`
}

// Which source the readers of conditions follow (clause 9).
const (
	FollowController = "controller"
	FollowKeeper     = "keeper"
)

// TakeoverVersion is the version of the take-over key's value.
const TakeoverVersion = 1

// Takeover is the value of the keeper's key `takeover.switch` (clause 9): which source the router, the grants part and
// mesh-watcher hear, the controller's condition events and `doctor-heartbeat`, or the keeper's events and the
// self-check's `checked`. Written by the keeper alone, on the operator's warrant for a take-over and by `hand-back`. An
// absent key is FollowController.
type Takeover struct {
	V      int       `json:"v"`
	Follow string    `json:"follow"`
	At     time.Time `json:"at"`
	// By is who chose (the operator, as the warrant proves), and Why why; Ask the approve ask it was chosen on.
	By  string `json:"by"`
	Why string `json:"why,omitempty"`
	Ask string `json:"ask,omitempty"`
}

// EncodeTakeover is t as JSON at TakeoverVersion; it refuses a source the readers could not follow.
func EncodeTakeover(t Takeover) ([]byte, error) {
	t.V = TakeoverVersion
	if err := t.check(); err != nil {
		return nil, err
	}
	return json.Marshal(t)
}

// DecodeTakeover reads the key as a getter answered it: an absent key follows the controller.
func DecodeTakeover(data []byte, found bool) (Takeover, error) {
	if !found || len(data) == 0 {
		return Takeover{V: TakeoverVersion, Follow: FollowController}, nil
	}
	var t Takeover
	if err := json.Unmarshal(data, &t); err != nil {
		return Takeover{}, fmt.Errorf("health: the take-over key is not readable: %w", err)
	}
	if t.V != TakeoverVersion {
		return Takeover{}, fmt.Errorf("health: the take-over key is of version %d, and this reads %d", t.V, TakeoverVersion)
	}
	if err := t.check(); err != nil {
		return Takeover{}, err
	}
	return t, nil
}

func (t Takeover) check() error {
	if t.Follow != FollowController && t.Follow != FollowKeeper {
		return fmt.Errorf("health: the take-over follows %q, not %s or %s", t.Follow, FollowController, FollowKeeper)
	}
	if t.Follow == FollowKeeper && strings.TrimSpace(t.By) == "" {
		return fmt.Errorf("health: a take-over by the keeper names who chose it")
	}
	return nil
}

// Seat is the seat a reader hears condition events on while it follows t: mesh-controller or mesh-conditions.
func (t Takeover) Seat() string {
	if t.Follow == FollowKeeper {
		return "mesh-conditions"
	}
	return "mesh-controller"
}

// SelfCheckEvent is the seat and event a reader of the self-check's run hears while it follows t: the controller's
// `doctor-heartbeat`, or mesh-self-check's `checked`.
func (t Takeover) SelfCheckEvent() (seat, event string) {
	if t.Follow == FollowKeeper {
		return "mesh-self-check", EventChecked
	}
	return "mesh-controller", "doctor-heartbeat"
}