418 lines
13 KiB
Go
418 lines
13 KiB
Go
package main
|
|
|
|
// The self-check (novox/hq ADR 0564 §6): the registry of probes, each a live invariant of a design, run every five
|
|
// minutes, each probe within thirty seconds. What a probe finds is written as a finding in this module's bucket
|
|
// `findings`, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the
|
|
// bucket `verdicts` for a week and the last as the key `last` of `verdict`, and is said as `checked`, which the
|
|
// watchman and mesh-watcher hear (S10, ADR 0227 §6). A probe that errors or does not finish is never a pass: it is
|
|
// said in the verdict at once and written as a finding of its own when the next run cannot run it either.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
)
|
|
|
|
// Module and Seat are this module and the seat it holds once for the mesh.
|
|
const (
|
|
Module = "mesh-self-check"
|
|
Seat = "mesh-self-check"
|
|
)
|
|
|
|
// The states it keeps, and the event that ends every run.
|
|
const (
|
|
FindingsState = health.FindingsState
|
|
VerdictState = "verdict"
|
|
VerdictsState = "verdicts"
|
|
LastKey = "last"
|
|
CheckedEvent = health.EventChecked
|
|
// CheckedSubject is where the seat says it: a seat's events are on mesh.seat.<seat>.event.<name>, which its
|
|
// consumers hear (catalogue.SeatSays).
|
|
CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent
|
|
)
|
|
|
|
// The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1).
|
|
var (
|
|
Every = 5 * time.Minute
|
|
FirstAfter = time.Minute
|
|
Within = 30 * time.Second
|
|
)
|
|
|
|
// The verdicts a probe can have, as mesh-sdk's health names them.
|
|
const (
|
|
VerdictPass = health.VerdictPassed
|
|
VerdictFail = health.VerdictFailed
|
|
VerdictFailedToRun = health.VerdictFailedToRun
|
|
VerdictDeferred = health.VerdictDeferred
|
|
)
|
|
|
|
// sourceSelf is what writes a probe's failure to run: the self-check itself.
|
|
const sourceSelf = "doctor"
|
|
|
|
// Store is the bus's buckets as this module reaches them, through its runtime.
|
|
type Store interface {
|
|
Get(state, key string) ([]byte, bool, error)
|
|
Put(state, key string, value any) error
|
|
Delete(state, key string) error
|
|
Keys(state string) ([]string, error)
|
|
}
|
|
|
|
// On says whether the setting lists the probe.
|
|
func (c *Checker) On(id string) bool {
|
|
if c.Enabled == nil {
|
|
return true
|
|
}
|
|
return slices.Contains(c.Enabled(), id)
|
|
}
|
|
|
|
// DefaultProbes are the probes that run unless the setting says otherwise.
|
|
var DefaultProbes = []string{"D13", "D-fill"}
|
|
|
|
// Probe is one live invariant: its id, what it asserts, the kind it raises, and the run; or why it waits.
|
|
type Probe struct {
|
|
ID string
|
|
Asserts string
|
|
From string
|
|
Kind string
|
|
Raises []string
|
|
// Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6):
|
|
// answered deferred, never a pass.
|
|
Deferred string
|
|
Run func(ctx context.Context, c *Checker) ([]Observation, error)
|
|
}
|
|
|
|
// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's
|
|
// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it.
|
|
type (
|
|
ProbeVerdict = health.ProbeVerdict
|
|
Counts = health.CheckedCounts
|
|
Run = health.Checked
|
|
)
|
|
|
|
// Checker is the self-check: the registry and what its probes reach.
|
|
type Checker struct {
|
|
Store Store
|
|
// Enabled is the setting `probes`: the ids of the probes that run. Nil runs every probe in the registry; one not
|
|
// listed runs nothing and raises nothing (the operator, 2026-10-30).
|
|
Enabled func() []string
|
|
Ask func(key string, body any) (json.RawMessage, error)
|
|
// Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish).
|
|
Publish func(subject string, body any, id string) error
|
|
Now func() time.Time
|
|
Logf func(format string, args ...any)
|
|
Host string
|
|
Registry []Probe
|
|
|
|
running sync.Mutex
|
|
mu sync.Mutex
|
|
n uint64
|
|
last *Run
|
|
// seen are the keys the run before saw, confirmed or not: a finding a single look can be wrong about is
|
|
// written only when two runs in a row see it, or while it is already written (novox/hq issue 277).
|
|
seen map[string]bool
|
|
// unsaid counts the runs whose `checked` could not be said since the start.
|
|
unsaid int
|
|
// pending is when each machine's agent account was first seen waiting for its setuid search (DA, D-root).
|
|
pending pendingSince
|
|
}
|
|
|
|
// LastRun is the last run, nil before the first.
|
|
func (c *Checker) LastRun() *Run {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.last
|
|
}
|
|
|
|
type result struct {
|
|
found []Observation
|
|
err error
|
|
took time.Duration
|
|
}
|
|
|
|
// RunOnce runs every probe of the registry, writes what changed in `findings`, keeps the verdict and says `checked`.
|
|
// One run at a time: a person's run during a scheduled one waits for it.
|
|
func (c *Checker) RunOnce(ctx context.Context, why string) Run {
|
|
c.running.Lock()
|
|
defer c.running.Unlock()
|
|
c.mu.Lock()
|
|
c.n++
|
|
n := c.n
|
|
c.mu.Unlock()
|
|
started := c.Now()
|
|
run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(),
|
|
IntervalSeconds: int(Every / time.Second), Controller: c.Host, Why: why}
|
|
|
|
results := make([]result, len(c.Registry))
|
|
var wg sync.WaitGroup
|
|
for i, p := range c.Registry {
|
|
if p.Run == nil || !c.On(p.ID) {
|
|
continue
|
|
}
|
|
wg.Add(1)
|
|
go func(i int, p Probe) {
|
|
defer wg.Done()
|
|
results[i] = c.probe(ctx, p)
|
|
}(i, p)
|
|
}
|
|
wg.Wait()
|
|
|
|
written, readErr := c.written()
|
|
seen := map[string]bool{}
|
|
want := map[string]Observation{} // every finding this run writes, by key
|
|
ran := map[string]bool{} // the sources whose findings this run judged whole
|
|
var blind []Observation
|
|
for i, p := range c.Registry {
|
|
if !c.On(p.ID) {
|
|
// Off by the setting: it ran nothing and sees nothing, so what it raised before is cleared, and it has no
|
|
// verdict in the run.
|
|
ran[p.ID] = true
|
|
continue
|
|
}
|
|
v := ProbeVerdict{ID: p.ID}
|
|
r := results[i]
|
|
switch {
|
|
case p.Run == nil:
|
|
v.Verdict = VerdictDeferred
|
|
v.Error = "deferred: " + p.Deferred
|
|
run.Counts.Deferred++
|
|
case r.err != nil:
|
|
v.Verdict, v.Error = VerdictFailedToRun, r.err.Error()
|
|
run.Counts.FailedToRun++
|
|
blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true,
|
|
Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf,
|
|
Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts),
|
|
Said: firstLine(r.err.Error())})
|
|
default:
|
|
ran[p.ID] = true
|
|
for _, f := range r.found {
|
|
if f.Kind == "" {
|
|
f.Kind = p.Kind
|
|
}
|
|
f.Source = p.ID
|
|
f = worded(f)
|
|
key := f.Key()
|
|
seen[key] = true
|
|
if f.Confirm && !c.seenBefore(key) && !written.has(key) {
|
|
v.Unconfirmed = append(v.Unconfirmed, f.Summary)
|
|
continue
|
|
}
|
|
want[key] = f
|
|
v.Found = append(v.Found, f.Summary)
|
|
}
|
|
if len(v.Found) == 0 {
|
|
v.Verdict = VerdictPass
|
|
run.Counts.Passed++
|
|
} else {
|
|
v.Verdict = VerdictFail
|
|
run.Counts.Failed++
|
|
}
|
|
}
|
|
if p.Run != nil {
|
|
v.Took = r.took.Round(time.Millisecond).String()
|
|
}
|
|
run.Probes = append(run.Probes, v)
|
|
}
|
|
ran[sourceSelf] = true
|
|
for _, f := range blind {
|
|
f = worded(f)
|
|
key := f.Key()
|
|
seen[key] = true
|
|
if c.seenBefore(key) || written.has(key) {
|
|
want[key] = f
|
|
}
|
|
}
|
|
if readErr != nil {
|
|
// Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so.
|
|
c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr)
|
|
} else if err := c.write(written, want, ran, run.Run); err != nil {
|
|
c.Logf("[%s] what this run found is not all written: %v", Module, err)
|
|
}
|
|
c.mu.Lock()
|
|
c.seen = seen
|
|
c.mu.Unlock()
|
|
|
|
run.At = c.Now().UTC()
|
|
run.Took = run.At.Sub(started).Round(time.Millisecond).String()
|
|
c.mu.Lock()
|
|
run.Unsaid = c.unsaid
|
|
c.mu.Unlock()
|
|
c.keep(run)
|
|
return run
|
|
}
|
|
|
|
// probe runs one probe within its bound; a panic or an overrun is an error, never a pass.
|
|
func (c *Checker) probe(ctx context.Context, p Probe) result {
|
|
probing, cancel := context.WithTimeout(ctx, Within)
|
|
defer cancel()
|
|
began := c.Now()
|
|
done := make(chan result, 1)
|
|
go func() {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
done <- result{err: fmt.Errorf("the probe panicked: %v", r)}
|
|
}
|
|
}()
|
|
found, err := p.Run(probing, c)
|
|
done <- result{found: found, err: err}
|
|
}()
|
|
select {
|
|
case r := <-done:
|
|
r.took = c.Now().Sub(began)
|
|
return r
|
|
case <-probing.Done():
|
|
return result{err: fmt.Errorf("it did not finish within %s", Within), took: c.Now().Sub(began)}
|
|
}
|
|
}
|
|
|
|
func (c *Checker) seenBefore(key string) bool {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.seen[key]
|
|
}
|
|
|
|
// writtenFindings are the findings in the bucket, by key.
|
|
type writtenFindings map[string]health.Finding
|
|
|
|
func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok }
|
|
|
|
// written reads every finding this module has written.
|
|
func (c *Checker) written() (writtenFindings, error) {
|
|
keys, err := c.Store.Keys(FindingsState)
|
|
if err != nil {
|
|
return writtenFindings{}, err
|
|
}
|
|
out := writtenFindings{}
|
|
for _, k := range keys {
|
|
raw, found, err := c.Store.Get(FindingsState, k)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
if !found {
|
|
continue
|
|
}
|
|
f, err := health.DecodeFinding(raw)
|
|
if err != nil {
|
|
return out, fmt.Errorf("the finding %s is not readable: %w", k, err)
|
|
}
|
|
out[k] = f
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding
|
|
// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4).
|
|
func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error {
|
|
var errs []string
|
|
keys := make([]string, 0, len(want))
|
|
for k := range want {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
now := c.Now()
|
|
for _, k := range keys {
|
|
f := want[k].Finding(run, now)
|
|
if was, ok := written[k]; ok && says(was) == says(f) {
|
|
continue
|
|
}
|
|
raw, err := health.EncodeFinding(f)
|
|
if err == nil {
|
|
err = c.Store.Put(FindingsState, k, json.RawMessage(raw))
|
|
}
|
|
if err != nil {
|
|
errs = append(errs, k+": "+err.Error())
|
|
}
|
|
}
|
|
for k, f := range written {
|
|
if _, still := want[k]; still || !ran[f.Source] {
|
|
continue
|
|
}
|
|
if err := c.Store.Delete(FindingsState, k); err != nil {
|
|
errs = append(errs, k+": "+err.Error())
|
|
}
|
|
}
|
|
if len(errs) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(errs, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// keep keeps the run as the last verdict and in the week's verdicts, and says it as `checked`. Not said is said in
|
|
// the log: the watchman's S10 says the self-check silent when `checked` stops.
|
|
func (c *Checker) keep(run Run) {
|
|
c.mu.Lock()
|
|
c.last = &run
|
|
c.mu.Unlock()
|
|
if err := c.Store.Put(VerdictState, LastKey, run); err != nil {
|
|
c.Logf("[%s] the run %s is not kept as the last verdict: %v", Module, run.Run, err)
|
|
}
|
|
if err := c.Store.Put(VerdictsState, run.Run, Overview(run)); err != nil {
|
|
c.Logf("[%s] the run %s is not kept among the week's verdicts: %v", Module, run.Run, err)
|
|
}
|
|
if c.Publish == nil {
|
|
return
|
|
}
|
|
if err := c.Publish(CheckedSubject, run, run.Run); err != nil {
|
|
c.mu.Lock()
|
|
c.unsaid++
|
|
c.mu.Unlock()
|
|
c.Logf("[%s] the run %s could NOT be said as checked, so the watchers will say the self-check is silent: %v",
|
|
Module, run.Run, err)
|
|
}
|
|
}
|
|
|
|
// overviewFindings is how many findings of one probe an overview carries (novox/hq issue 314).
|
|
const overviewFindings = 20
|
|
|
|
// Overview is a run with each probe's findings at most overviewFindings, with their count and how to read them all.
|
|
func Overview(run Run) Run {
|
|
probes := make([]ProbeVerdict, len(run.Probes))
|
|
for i, p := range run.Probes {
|
|
total := len(p.Found) + len(p.Unconfirmed)
|
|
if total > overviewFindings {
|
|
p.FoundCount = len(p.Found)
|
|
if len(p.Found) > overviewFindings {
|
|
p.Found = p.Found[:overviewFindings:overviewFindings]
|
|
}
|
|
if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room {
|
|
p.Unconfirmed = p.Unconfirmed[:room:room]
|
|
}
|
|
p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all",
|
|
p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID)
|
|
}
|
|
probes[i] = p
|
|
}
|
|
run.Probes = probes
|
|
return run
|
|
}
|
|
|
|
// Keep runs the registry on its schedule until ctx ends: the first run FirstAfter the start, then every Every.
|
|
func (c *Checker) Keep(ctx context.Context) {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(FirstAfter):
|
|
}
|
|
tick := time.NewTicker(Every)
|
|
defer tick.Stop()
|
|
for {
|
|
c.RunOnce(ctx, "the schedule")
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-tick.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func firstLine(s string) string {
|
|
line, _, _ := strings.Cut(s, "\n")
|
|
return line
|
|
}
|