Its states presence, pulse and stuck and its first views were named from the constant WatchmanModule; they are now `<cred.Module>_<state>` (watch.BucketsOf), and the watchman's stuck keys of presence take its own presence bucket's name. Builds: design 47 G A-296 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
805 lines
30 KiB
Go
805 lines
30 KiB
Go
package watch
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"regexp"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
|
||
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||
)
|
||
|
||
// The rows of the signals table the watchman holds (novox/hq ADR 0564 §4, Limits 3 and 11). Each is a key of the pulse
|
||
// bucket whose time to live is the row's bound, renewed by the signal's own event: its expiry is the finding, so no loop
|
||
// looks every half minute. A signal's pulse is kept under the pulse bucket's pseudo-bucket `signal`, `signal.<row>.<id>`,
|
||
// and its value says what its expiry does (signalDue), so a pulse that stood across a restart still acts when it expires.
|
||
//
|
||
// What each row hears is on the bus, never in the controller's store:
|
||
// - S1, a machine silent: the watchman's own presence of `node.<node>` (its stuck key), not while the machine said it
|
||
// sleeps or shuts down (the power seat's events, ADR 0211);
|
||
// - S8, a provider's failing word repeated: the providers' `provisioner.failing` and `provisioner.recovered` (ADR 0224);
|
||
// the failing word is itself the finding `provider-failing`, and its pulse of 30 minutes, renewed by each repeat, is S8;
|
||
// - S9, the bus's advisories: each occurrence a finding, cleared by the expiry of an hour's pulse renewed by the next;
|
||
// - S10, the self-check's `checked`: a pulse of three runs' length, from the first run heard;
|
||
// - S11, a machine's tool runner heartbeat on `mesh.control.<node>.tools-alive`: a pulse of three intervals;
|
||
// - S21, a module's have failed: a pulse of one apply interval from the first failure, whose expiry while it still fails
|
||
// is the finding `apply-failing`;
|
||
// - a module's have still failed a minute after it first said so, `unhealthy`, urgent after four hours; and a have that waits for
|
||
// the operator, `needs-operator` (ADR 0240 §12, §17).
|
||
|
||
// The bounds of the rows.
|
||
const (
|
||
// ProviderSaysAgainWithin is how long a provider's failing word stays current without being said again (S8): twice
|
||
// the quarter of an hour it repeats it at.
|
||
ProviderSaysAgainWithin = 30 * time.Minute
|
||
// AdvisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9).
|
||
AdvisoryQuiet = time.Hour
|
||
// SelfCheckEvery is the self-check's run interval; S10's bound is three of them.
|
||
SelfCheckEvery = 5 * time.Minute
|
||
// HeartbeatEvery is a tool runner's interval when it says none; S11's bound is three of them.
|
||
HeartbeatEvery = 60 * time.Second
|
||
// ApplyEvery is one apply interval (S21): a failure still there after it is not one the next apply clears.
|
||
ApplyEvery = 5 * time.Minute
|
||
// SilentBound is the bound S1 says, three of the intervals a machine has always used (the controller's silentBound).
|
||
SilentBound = 3 * HeartbeatEvery
|
||
)
|
||
|
||
// signalBucket is the pseudo-bucket of a signal's pulse key.
|
||
const signalBucket = "signal"
|
||
|
||
// signalDue is what a signal's pulse does when it expires: raise the finding (find), or end it (end, an advisory's quiet
|
||
// hour). Kept as the pulse's Owes, so it is read back at a restart.
|
||
type signalDue struct {
|
||
Do string `json:"do"`
|
||
Finding Finding `json:"finding"`
|
||
// While names what must still stand at the expiry for a find to be written: a have's key still failing or unhealthy;
|
||
// empty for a row whose expiry alone is the finding.
|
||
While string `json:"while,omitempty"`
|
||
}
|
||
|
||
const (
|
||
dueFind = "find"
|
||
dueEnd = "end"
|
||
// dueApply judges a machine's S21 again (applyCheck); dueRecount counts DEAD_LETTERS again (a restricted letter aged out).
|
||
dueApply = "apply"
|
||
dueRecount = "recount"
|
||
// dueCreated is no due: the mark, with no time to live, that the bus module said an entry created (busstate.go).
|
||
dueCreated = "created"
|
||
)
|
||
|
||
// signalKey is the pulse key of a row's signal about id.
|
||
func signalKey(row, id string) string {
|
||
parts := []string{signalBucket, row}
|
||
for _, p := range strings.Split(id, ".") {
|
||
if p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_"); p == "" {
|
||
p = "_"
|
||
}
|
||
parts = append(parts, p)
|
||
}
|
||
return strings.Join(parts, ".")
|
||
}
|
||
|
||
// unsafeKey is what a key's token may not hold: the bus takes letters, digits and `-_/=`.
|
||
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
|
||
|
||
// signal writes (or renews) the pulse of a row's signal, whose expiry does due.
|
||
func (w *Watchman) signal(row, id string, ttl time.Duration, due signalDue) {
|
||
pk := signalKey(row, id)
|
||
owes, err := json.Marshal(due)
|
||
if err != nil {
|
||
w.say("no signal %s: %v", pk, err)
|
||
return
|
||
}
|
||
value, err := meshwatchdogsclient.EncodePulse(meshwatchdogsclient.Pulse{Bucket: signalBucket, Key: strings.TrimPrefix(pk, signalBucket+"."), Owes: string(owes)})
|
||
if err != nil {
|
||
w.say("no signal %s: %v", pk, err)
|
||
return
|
||
}
|
||
if err := w.out.PutPulse(pk, value, ttl); err != nil {
|
||
w.say("cannot write the signal %s: %v", pk, err)
|
||
return
|
||
}
|
||
w.signals[pk] = due
|
||
}
|
||
|
||
// unsignal ends a row's signal about id: its pulse deleted, so it never expires.
|
||
func (w *Watchman) unsignal(row, id string) {
|
||
pk := signalKey(row, id)
|
||
if _, ok := w.signals[pk]; !ok {
|
||
return
|
||
}
|
||
delete(w.signals, pk)
|
||
if err := w.out.DeletePulse(pk); err != nil {
|
||
w.say("cannot delete the signal %s: %v", pk, err)
|
||
}
|
||
}
|
||
|
||
// SeedSignal tells the watchman a signal's pulse that stands at its start, with its value: it is kept, since its event
|
||
// renews it and its expiry still acts.
|
||
func (w *Watchman) SeedSignal(key string, value []byte) {
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
if p, err := meshwatchdogsclient.DecodePulse(value); err == nil && p.Bucket == signalBucket {
|
||
var due signalDue
|
||
if json.Unmarshal([]byte(p.Owes), &due) == nil && due.Do == dueCreated {
|
||
w.busCreated[key] = true
|
||
return
|
||
}
|
||
}
|
||
if due, ok := dueOf(value); ok {
|
||
w.signals[key] = due
|
||
}
|
||
}
|
||
|
||
func dueOf(value []byte) (signalDue, bool) {
|
||
p, err := meshwatchdogsclient.DecodePulse(value)
|
||
if err != nil || p.Bucket != signalBucket {
|
||
return signalDue{}, false
|
||
}
|
||
var due signalDue
|
||
if json.Unmarshal([]byte(p.Owes), &due) != nil || (due.Do != dueFind && due.Do != dueEnd && due.Do != dueApply && due.Do != dueRecount) {
|
||
return signalDue{}, false
|
||
}
|
||
return due, true
|
||
}
|
||
|
||
// signalExpired is a signal's pulse expired: its due is done.
|
||
func (w *Watchman) signalExpired(pk string) {
|
||
due, ok := w.signals[pk]
|
||
if !ok {
|
||
return
|
||
}
|
||
delete(w.signals, pk)
|
||
switch due.Do {
|
||
case dueApply:
|
||
w.applyCheck(due.While)
|
||
case dueRecount:
|
||
if w.recount != nil {
|
||
go w.recount()
|
||
}
|
||
case dueEnd:
|
||
w.unfind(due.Finding.Key())
|
||
case dueFind:
|
||
if due.While == uncovered {
|
||
w.opened[due.Finding.ID] = due.Finding
|
||
w.rejudgeRecords()
|
||
return
|
||
}
|
||
if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) {
|
||
return
|
||
}
|
||
w.find(due.Finding)
|
||
}
|
||
}
|
||
|
||
// stillWrong says a silent machine's finding still stands, which a row's finding waited on.
|
||
func (w *Watchman) stillWrong(key, _ string) bool {
|
||
if node, ok := strings.CutPrefix(key, "silent:"); ok {
|
||
_, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)]
|
||
return open
|
||
}
|
||
return false
|
||
}
|
||
|
||
// The kinds the watchman's rows write, as the controller's conditions name them.
|
||
const (
|
||
KindSilent = "silent"
|
||
KindStuck = "stuck"
|
||
KindProviderFailing = "provider-failing"
|
||
KindProviderSilent = "provider-silent"
|
||
KindSelfCheckSilent = "self-check-silent"
|
||
KindToolsSilent = "tools-silent"
|
||
KindApplyFailing = "apply-failing"
|
||
KindNeedsOperator = "needs-operator"
|
||
KindDeadLetters = "max-deliveries"
|
||
)
|
||
|
||
func stamp(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") }
|
||
|
||
// ---- S1 and the power seat ----
|
||
|
||
// PowerEvents are the power seat's events that say whether a machine is about to be away or is back (ADR 0211).
|
||
var PowerEvents = map[string]bool{"sleeping": true, "shutting-down": true, "booted": true, "woke": true}
|
||
|
||
// Power is a machine's power event: sleeping and shutting-down say it is away, booted and woke that it is back. A machine
|
||
// that said it is away is not silent; one back that is still offline is watched again from now.
|
||
func (w *Watchman) Power(node, state string) {
|
||
if !PowerEvents[state] || node == "" {
|
||
return
|
||
}
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
away := state == "sleeping" || state == "shutting-down"
|
||
w.away[node] = away
|
||
key, err := meshwatchdogsclient.PresenceNodeKey(node)
|
||
if err != nil {
|
||
return
|
||
}
|
||
defer w.applyCheck(node)
|
||
if away {
|
||
w.unfind(FindingKey(ScopeMachine, node, KindSilent))
|
||
w.unsignal("S1", node)
|
||
return
|
||
}
|
||
if p, ok := w.presence[key]; ok && !p.online {
|
||
p.since = w.now()
|
||
w.presence[key] = p
|
||
w.ensureOffline(key)
|
||
}
|
||
}
|
||
|
||
// stuckFinding is the finding of a stuck key as it stands (ADR 0433 §2, §3, now the watchman's own): a node's presence not
|
||
// moved is S1, the machine silent, unless it said it is away; any other key not moved is `delivery.<key>.stuck`; a key
|
||
// moved ends its finding.
|
||
func (w *Watchman) stuckFinding(sk string, s meshwatchdogsclient.Stuck) {
|
||
if s.Bucket == w.own.Presence {
|
||
if node, part, _, err := meshwatchdogsclient.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" {
|
||
key := FindingKey(ScopeMachine, node, KindSilent)
|
||
if s.Moved {
|
||
w.machineBack(node)
|
||
} else {
|
||
w.machineAway(node, s.Since)
|
||
}
|
||
if s.Moved || w.away[node] {
|
||
w.unfind(key)
|
||
w.unsignal("S1", node)
|
||
w.applyCheck(node)
|
||
return
|
||
}
|
||
f := Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1",
|
||
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), SilentBound),
|
||
Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since}
|
||
w.find(f)
|
||
w.applyCheck(node)
|
||
return
|
||
}
|
||
}
|
||
key := FindingKey(ScopeDelivery, sk, KindStuck)
|
||
if s.Moved {
|
||
w.unfind(key)
|
||
return
|
||
}
|
||
waited := s.Waited
|
||
if waited == "" {
|
||
waited = "a move"
|
||
}
|
||
w.find(Finding{Scope: ScopeDelivery, ID: sk, Kind: KindStuck, Severity: Warning, Source: "stuck",
|
||
Summary: fmt.Sprintf("%s has not moved since %s: it waited for %s", sk, stamp(s.Since), waited),
|
||
Said: fmt.Sprintf("%s: waiting for %s since %s", sk, waited, s.Since.UTC().Format(time.RFC3339)), Since: s.Since})
|
||
}
|
||
|
||
// ---- S8 ----
|
||
|
||
// Standing is a provider's word about one consumer (ADR 0224): Module is the emitter, read from the subject.
|
||
type Standing struct {
|
||
Module string `json:"-"`
|
||
Failing bool `json:"-"`
|
||
Provider string `json:"provider"`
|
||
ProviderNode string `json:"provider-node"`
|
||
Consumer string `json:"consumer"`
|
||
Node string `json:"node"`
|
||
Class string `json:"class,omitempty"`
|
||
Error string `json:"error,omitempty"`
|
||
Since time.Time `json:"since"`
|
||
Attempts int `json:"attempts"`
|
||
Why string `json:"why,omitempty"`
|
||
}
|
||
|
||
// ReadStanding reads a provider's word from its subject, `mesh.mod.<module>.event.provisioner.failing` or `.recovered`.
|
||
func ReadStanding(subject string, body []byte) (Standing, error) {
|
||
rest, ok := strings.CutPrefix(subject, "mesh.mod.")
|
||
if !ok {
|
||
return Standing{}, fmt.Errorf("%s is not a provider's word", subject)
|
||
}
|
||
module, event, ok := strings.Cut(rest, ".event.")
|
||
if !ok || module == "" || (event != "provisioner.failing" && event != "provisioner.recovered") {
|
||
return Standing{}, fmt.Errorf("%s is not a provider's word", subject)
|
||
}
|
||
var st Standing
|
||
if err := json.Unmarshal(body, &st); err != nil {
|
||
return Standing{}, fmt.Errorf("%s's word could not be read: %w", module, err)
|
||
}
|
||
if st.Consumer == "" {
|
||
return Standing{}, fmt.Errorf("%s's word names no consumer", module)
|
||
}
|
||
st.Module, st.Failing = module, event == "provisioner.failing"
|
||
return st, nil
|
||
}
|
||
|
||
// Stood is a provider's word: failing writes `provider.<module>.<node>.<consumer>.failing` and renews S8's pulse, whose
|
||
// expiry writes `provider-silent`; recovered deletes both (Limits 11).
|
||
func (w *Watchman) Stood(st Standing) {
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
id := st.Module + "." + st.ProviderNode + "." + st.Consumer
|
||
failing := FindingKey(ScopeProvider, id, "failing")
|
||
silentKey := FindingKey(ScopeProvider, id, "silent")
|
||
if !st.Failing {
|
||
w.unfind(failing)
|
||
w.unfind(silentKey)
|
||
w.unsignal("S8", id)
|
||
return
|
||
}
|
||
class := st.Class
|
||
if class == "" {
|
||
class = "failing"
|
||
}
|
||
whom := st.Consumer
|
||
if st.Node != "" {
|
||
whom += " on " + st.Node
|
||
}
|
||
said := class
|
||
if st.Error != "" {
|
||
said += ": " + firstLine(st.Error)
|
||
}
|
||
var also []string
|
||
if st.Node != "" && st.Node != st.ProviderNode {
|
||
also = []string{st.Node}
|
||
}
|
||
w.find(Finding{Scope: ScopeProvider, ID: id, Token: "failing", Kind: KindProviderFailing, Machine: st.ProviderNode, Also: also,
|
||
Severity: Warning, Source: "provisioner.failing", Since: st.Since, Said: said,
|
||
Summary: fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode, whom, class,
|
||
st.Attempts, stamp(st.Since))})
|
||
w.unfind(silentKey)
|
||
now := w.now()
|
||
w.signal("S8", id, ProviderSaysAgainWithin, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeProvider, ID: id, Token: "silent",
|
||
Kind: KindProviderSilent, Machine: st.ProviderNode, Severity: Warning, Source: "S8", Since: now,
|
||
Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped saying anything, so "+
|
||
"its last word is all the mesh has", st.Module, st.ProviderNode, st.Consumer, stamp(now)),
|
||
Said: fmt.Sprintf("not said again within %s", ProviderSaysAgainWithin)}})
|
||
}
|
||
|
||
func firstLine(s string) string {
|
||
s, _, _ = strings.Cut(strings.TrimSpace(s), "\n")
|
||
return s
|
||
}
|
||
|
||
// ---- S9 ----
|
||
|
||
// Advisory is one thing the bus said (S9): its kind (slow-consumer, connection-closed, max-deliveries, refused,
|
||
// consumer-lost), what it is about, its words, and the machine when it names one.
|
||
type Advisory struct {
|
||
Kind, ID, Said, Machine string
|
||
}
|
||
|
||
// Advised is an advisory heard: its finding `bus.<id>.<kind>` is written at once, counting the times since its first, and
|
||
// an hour's pulse renewed by each, whose expiry ends it.
|
||
func (w *Watchman) Advised(a Advisory) {
|
||
if a.Kind == "" || a.ID == "" {
|
||
return
|
||
}
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
f := Finding{Scope: ScopeBus, ID: a.ID, Kind: a.Kind, Machine: a.Machine, Severity: Warning, Source: "S9", Said: a.Said}
|
||
key := f.Key()
|
||
n := w.advised[key] + 1
|
||
if _, open := w.findings[key]; !open {
|
||
n = 1
|
||
w.advisedSince[key] = w.now()
|
||
}
|
||
w.advised[key] = n
|
||
since := w.advisedSince[key]
|
||
f.Since, f.Summary = since, a.Said
|
||
if n > 1 {
|
||
f.Summary = fmt.Sprintf("%s (%d times since %s)", a.Said, n, since.UTC().Format("15:04 MST"))
|
||
}
|
||
w.find(f)
|
||
w.signal("S9", key, AdvisoryQuiet, signalDue{Do: dueEnd, Finding: f})
|
||
}
|
||
|
||
// ---- S10 ----
|
||
|
||
// Checked is the self-check's `checked` heard: its pulse of three runs' length is renewed, and its silence ended. Before
|
||
// the first run heard there is no pulse, so a self-check not yet placed is not said silent.
|
||
func (w *Watchman) Checked(machine string) {
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
w.unfind(FindingKey(ScopeCore, "doctor", KindSelfCheckSilent))
|
||
now := w.now()
|
||
w.signal("S10", "doctor", 3*SelfCheckEvery, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeCore, ID: "doctor",
|
||
Kind: KindSelfCheckSilent, Machine: machine, Severity: Urgent, Source: "S10", Since: now,
|
||
Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's invariants are not "+
|
||
"being checked", stamp(now), SelfCheckEvery),
|
||
Said: fmt.Sprintf("no run within %s", 3*SelfCheckEvery)}})
|
||
}
|
||
|
||
// ---- S11 ----
|
||
|
||
// ToolsAlive is a machine's tool runner heartbeat, saying its interval (zero: HeartbeatEvery): its pulse of three
|
||
// intervals is renewed and its silence ended.
|
||
func (w *Watchman) ToolsAlive(node string, every time.Duration) {
|
||
if node == "" {
|
||
return
|
||
}
|
||
if every <= 0 {
|
||
every = HeartbeatEvery
|
||
}
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
w.unfind(FindingKey(ScopeMachine, node, KindToolsSilent))
|
||
if w.away[node] {
|
||
w.unsignal("S11", node)
|
||
return
|
||
}
|
||
now := w.now()
|
||
w.signal("S11", node, 3*every, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeMachine, ID: node, Kind: KindToolsSilent,
|
||
Machine: node, Severity: Warning, Source: "S11", Since: now,
|
||
Summary: fmt.Sprintf("the node tools on %s have not said they are there since %s (bound %s): nothing can ask that "+
|
||
"machine anything", node, stamp(now), 3*every),
|
||
Said: fmt.Sprintf("silent for more than %s", 3*every)}})
|
||
}
|
||
|
||
// ToolsNode is the machine a tool runner heartbeat's subject names, `mesh.control.<node>.tools-alive`.
|
||
func ToolsNode(subject string) string {
|
||
rest, ok := strings.CutPrefix(subject, "mesh.control.")
|
||
if !ok {
|
||
return ""
|
||
}
|
||
node, ok := strings.CutSuffix(rest, ".tools-alive")
|
||
if !ok || strings.Contains(node, ".") {
|
||
return ""
|
||
}
|
||
return node
|
||
}
|
||
|
||
// ---- S21, from the provisioner clients' reports ----
|
||
|
||
// Provision is a change of a key of mesh-provisioner's `provisions` (ADR 0570 §7): a provision in `waits` is a part of its
|
||
// consumer waiting for the operator, said as `module.<consumer>.<node>.needs-operator` naming each waiting provision and
|
||
// why; the finding ends when no provision of that consumer on that node waits. A key deleted, or one that cannot be read,
|
||
// waits no more.
|
||
func (w *Watchman) Provision(key string, value []byte, op Op) {
|
||
node, consumer, _, _, err := meshprovisionerclient.ParseKey(key)
|
||
if err != nil {
|
||
return
|
||
}
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
id := consumer + "." + node
|
||
if w.waiting[id] == nil {
|
||
w.waiting[id] = map[string]string{}
|
||
}
|
||
delete(w.waiting[id], key)
|
||
if op == Put {
|
||
if p, err := meshprovisionerclient.Decode(value); err != nil {
|
||
w.say("the provision %s cannot be read, and is taken as not waiting: %v", key, err)
|
||
} else if p.State == meshprovisionerclient.Waits {
|
||
w.waiting[id][key] = fmt.Sprintf("%s from %s on %s: %s", p.Provision, p.Provider.Module, p.Provider.Node, p.Why)
|
||
}
|
||
}
|
||
w.askedWait(key, value, op)
|
||
fkey := FindingKey(ScopeModule, id, KindNeedsOperator)
|
||
if len(w.waiting[id]) == 0 {
|
||
delete(w.waiting, id)
|
||
w.unfind(fkey)
|
||
return
|
||
}
|
||
var said []string
|
||
for _, k := range sortedKeys(w.waiting[id]) {
|
||
said = append(said, w.waiting[id][k])
|
||
}
|
||
w.find(Finding{Scope: ScopeModule, ID: id, Kind: KindNeedsOperator, Machine: node, Severity: Warning, Source: "provisions",
|
||
Summary: fmt.Sprintf("%s on %s waits for the operator: %s", consumer, node, strings.Join(said, "; ")),
|
||
Said: strings.Join(said, "; ")})
|
||
}
|
||
|
||
// askedWait is the wait of a provision its provider has not provided (ADR 0603 §8): a key asked is mirrored from the time the
|
||
// holder wrote it, bounded as a step is, and provided, failed, waiting for the operator (its own finding), retiring or deleted
|
||
// ends it. Called with the lock held.
|
||
func (w *Watchman) askedWait(key string, value []byte, op Op) {
|
||
bucket := ProvisionsBucket
|
||
if op == Expired {
|
||
return
|
||
}
|
||
w.stands(bucket, key)
|
||
if op == Delete {
|
||
w.moved(bucket, key)
|
||
return
|
||
}
|
||
p, err := meshprovisionerclient.Decode(value)
|
||
if err != nil || p.State != meshprovisionerclient.Asked {
|
||
w.moved(bucket, key)
|
||
return
|
||
}
|
||
since := p.At
|
||
if since.IsZero() {
|
||
since = w.now()
|
||
}
|
||
words := fmt.Sprintf("%s from %s on %s to be provided for %s on %s, bound %s", p.Provision, p.Provider.Module, p.Provider.Node,
|
||
p.Consumer.Module, p.Consumer.Node, w.stepBound)
|
||
if p.Why != "" {
|
||
words += ": " + p.Why
|
||
}
|
||
w.mirrorKept(bucket, key, words, "asked", since)
|
||
}
|
||
|
||
func sortedKeys(m map[string]string) []string {
|
||
out := make([]string, 0, len(m))
|
||
for k := range m {
|
||
out = append(out, k)
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
// ProvReport is a machine's provisioner client's word on one module's files (`prov.applied.<node>.<module>`, ADR 0362 §3).
|
||
type ProvReport struct {
|
||
Node string `json:"node"`
|
||
Module string `json:"module"`
|
||
Status string `json:"status"`
|
||
Reason string `json:"reason,omitempty"`
|
||
At time.Time `json:"at"`
|
||
}
|
||
|
||
// ProvFailed is the status of a report whose files were not placed.
|
||
const ProvFailed = "failed"
|
||
|
||
// Provisioned is a provisioner client's newest report: a failure is kept for S21, any other word ends it.
|
||
func (w *Watchman) Provisioned(r ProvReport) {
|
||
if r.Node == "" || r.Module == "" {
|
||
return
|
||
}
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
if w.provFailed[r.Node] == nil {
|
||
w.provFailed[r.Node] = map[string]ProvReport{}
|
||
}
|
||
if r.Status == ProvFailed {
|
||
w.provFailed[r.Node][r.Module] = r
|
||
} else {
|
||
delete(w.provFailed[r.Node], r.Module)
|
||
}
|
||
w.applyCheck(r.Node)
|
||
}
|
||
|
||
// heard says the watchman hears the machine now: online, and not said away. S21 is judged only while it is (the
|
||
// controller's watchApplyFailing): S1 says silent, and asleep is not lost.
|
||
func (w *Watchman) heard(node string) bool {
|
||
key, err := meshwatchdogsclient.PresenceNodeKey(node)
|
||
if err != nil || w.away[node] {
|
||
return false
|
||
}
|
||
p, ok := w.presence[key]
|
||
return ok && p.online
|
||
}
|
||
|
||
// applyCheck is S21 for one machine, `machine.<node>.apply-failing` (the controller's watchApplyFailing): raised once the
|
||
// first failure it still has is older than one apply interval while the machine is heard, naming the modules its node
|
||
// failed to apply (their haves failed), or else those whose files its provisioner client failed to place; ended by the
|
||
// clean apply. Before the bound a pulse judges it again at the bound.
|
||
func (w *Watchman) applyCheck(node string) {
|
||
key := FindingKey(ScopeMachine, node, KindApplyFailing)
|
||
type failure struct{ module, words string }
|
||
var placed []failure
|
||
var since time.Time
|
||
for module, r := range w.provFailed[node] {
|
||
placed = append(placed, failure{module, firstLine(orWords(r.Reason, "no reason given"))})
|
||
if since.IsZero() || r.At.Before(since) {
|
||
since = r.At
|
||
}
|
||
}
|
||
if since.IsZero() || !w.heard(node) {
|
||
w.unsignal("S21", node)
|
||
w.unfind(key)
|
||
return
|
||
}
|
||
if age := w.now().Sub(since); age <= ApplyEvery {
|
||
if _, pending := w.signals[signalKey("S21", node)]; !pending {
|
||
w.signal("S21", node, ApplyEvery-age+time.Second, signalDue{Do: dueApply, While: node,
|
||
Finding: Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Severity: Warning, Summary: "judged again at its bound", Source: "S21"}})
|
||
}
|
||
return
|
||
}
|
||
failed := placed
|
||
sort.Slice(failed, func(i, j int) bool { return failed[i].module < failed[j].module })
|
||
var names, words []string
|
||
tool := ""
|
||
for _, f := range failed {
|
||
names = append(names, f.module)
|
||
words = append(words, f.module+": "+f.words)
|
||
if rest, ok := strings.CutPrefix(f.words, "refused: "); ok {
|
||
f.words = rest
|
||
}
|
||
if i := strings.Index(f.words, PackageManagerKey); i >= 0 && tool == "" {
|
||
tool = strings.TrimSuffix(strings.SplitN(f.words[i+len(PackageManagerKey):], ")", 2)[0], ")")
|
||
}
|
||
}
|
||
f := Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Machine: node, Severity: Warning, Source: "S21", Since: since,
|
||
Said: strings.Join(words, "; "), Headline: node + " is not applying its update"}
|
||
f.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", node,
|
||
strings.Join(names, ", "), stamp(since), ApplyEvery)
|
||
f.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s. The details say why.",
|
||
node, strings.Join(names, ", "))
|
||
if tool != "" {
|
||
f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is placed, "+
|
||
"since %s (bound %s)", node, tool, stamp(since), ApplyEvery)
|
||
f.Headline = node + " cannot install packages"
|
||
}
|
||
w.find(f)
|
||
}
|
||
|
||
// PackageManagerKey is how a node-engine names a package manager that does not answer among what failed.
|
||
const PackageManagerKey = "the package manager ("
|
||
|
||
// ---- S9: the bus module's closures ----
|
||
|
||
// BusClosure is one connection the bus closed, as the bus module says it (`bus.slow-consumer`, `bus.connection-closed`).
|
||
type BusClosure struct {
|
||
User string `json:"user"`
|
||
Name string `json:"name,omitempty"`
|
||
CID uint64 `json:"cid"`
|
||
Reason string `json:"reason"`
|
||
Opened time.Time `json:"opened"`
|
||
Closed time.Time `json:"closed"`
|
||
Pending int64 `json:"pending_bytes,omitempty"`
|
||
}
|
||
|
||
// The kinds of the bus module's closures, as conditions name them.
|
||
const (
|
||
KindSlowConsumer = "slow-consumer"
|
||
KindConnectionClosed = "connection-closed"
|
||
)
|
||
|
||
// Closed is one closure the bus module said (S9, the controller's BusClosures and busUserClosed): one finding per user and
|
||
// kind, `bus.<user>.<kind>`, each closure counted once however often it is said, ended an hour after the newest.
|
||
func (w *Watchman) Closed(kind string, c BusClosure) {
|
||
if c.User == "" || c.Closed.IsZero() || (kind != KindSlowConsumer && kind != KindConnectionClosed) {
|
||
return
|
||
}
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
f := Finding{Scope: ScopeBus, ID: c.User, Kind: kind, Severity: Warning, Source: "S9"}
|
||
key := f.Key()
|
||
if _, open := w.findings[key]; !open {
|
||
w.closures[key] = map[string]time.Time{}
|
||
}
|
||
identity := fmt.Sprintf("%s/%d/%s", c.User, c.CID, c.Opened.UTC().Format(time.RFC3339Nano))
|
||
if _, seen := w.closures[key][identity]; seen {
|
||
return
|
||
}
|
||
w.closures[key][identity] = c.Closed
|
||
var first, last time.Time
|
||
for _, at := range w.closures[key] {
|
||
if first.IsZero() || at.Before(first) {
|
||
first = at
|
||
}
|
||
if at.After(last) {
|
||
last = at
|
||
}
|
||
}
|
||
n := len(w.closures[key])
|
||
who, machine := BusUserWords(c.User)
|
||
f.Machine, f.Since = machine, first
|
||
times, count := "once", "once"
|
||
if n > 1 {
|
||
times = fmt.Sprintf("%d times since %s", n, first.UTC().Format("15:04 MST"))
|
||
count = fmt.Sprintf("%d times", n)
|
||
}
|
||
if kind == KindSlowConsumer {
|
||
f.Said = fmt.Sprintf("the bus closed %s's connection %d as a slow consumer: %s", c.User, c.CID, c.Reason)
|
||
f.Summary = fmt.Sprintf("the bus closed the connection of %s as a slow consumer %s in the last hour, newest at %s: it "+
|
||
"could not take what it was sent fast enough, and what was on its way to it was lost", who, times, last.UTC().Format("15:04 MST"))
|
||
f.Headline = clip(capital(who)+" was too slow for the bus", 60)
|
||
f.Explanation = capital(fmt.Sprintf("%s could not keep up with what the bus sent it, so the bus closed its connection (%s "+
|
||
"in the last hour) and what was on its way was lost. It connects again by itself; this clears an hour after the last time.", who, count))
|
||
} else {
|
||
reason := strings.ToLower(c.Reason)
|
||
f.Said = fmt.Sprintf("the bus closed %s's connection %d: %s", c.User, c.CID, c.Reason)
|
||
f.Summary = fmt.Sprintf("the bus closed the connection of %s for breaking its rules %s in the last hour, newest at %s: %s",
|
||
who, times, last.UTC().Format("15:04 MST"), reason)
|
||
f.Headline = clip(capital(who)+" was cut off by the bus", 60)
|
||
f.Explanation = capital(fmt.Sprintf("The bus closed the connection of %s because it broke the bus's rules (%s, %s in the "+
|
||
"last hour), so what it was sending was lost. This clears an hour after the last time.", who, reason, count))
|
||
}
|
||
w.find(f)
|
||
if quiet := AdvisoryQuiet - w.now().Sub(last); quiet > 0 {
|
||
w.signal("S9", key, quiet, signalDue{Do: dueEnd, Finding: f})
|
||
} else {
|
||
w.unfind(key)
|
||
}
|
||
}
|
||
|
||
// BusUserWords is a bus user as the operator says it, and its machine (the controller's busUserWords).
|
||
func BusUserWords(user string) (string, string) {
|
||
switch {
|
||
case user == "controller":
|
||
return "the controller", ""
|
||
case strings.HasPrefix(user, "node."):
|
||
m := strings.TrimPrefix(user, "node.")
|
||
return "the node-engine on " + m, m
|
||
}
|
||
if m, module, ok := strings.Cut(user, "."); ok {
|
||
return module + " on " + m, m
|
||
}
|
||
return "a client of the bus", ""
|
||
}
|
||
|
||
func capital(s string) string {
|
||
if s == "" {
|
||
return s
|
||
}
|
||
return strings.ToUpper(s[:1]) + s[1:]
|
||
}
|
||
|
||
func clip(s string, n int) string {
|
||
if len(s) <= n {
|
||
return s
|
||
}
|
||
cut := s[:n]
|
||
if i := strings.LastIndex(cut, " "); i > 0 {
|
||
cut = cut[:i]
|
||
}
|
||
return cut
|
||
}
|
||
|
||
// ---- dead letters aging out ----
|
||
|
||
// SetRecount sets how DEAD_LETTERS is counted again when a restricted letter ages out.
|
||
func (w *Watchman) SetRecount(f func()) {
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
w.recount = f
|
||
}
|
||
|
||
// DeadAging is a restricted letter of a consumer (`<stream>.<consumer>`) that ages out at at: a pulse expiring a second
|
||
// after it counts DEAD_LETTERS again, so the consumer's finding ends when its letters are gone (ADR 0376 §6), with no
|
||
// timer. The earliest of a consumer's letters is the one pulsed.
|
||
func (w *Watchman) DeadAging(key string, at time.Time) {
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
left := at.Sub(w.now())
|
||
if left <= 0 {
|
||
left = 0
|
||
}
|
||
pk := signalKey("dead", key)
|
||
if cur, ok := w.signals[pk]; ok && !cur.Finding.Since.IsZero() && cur.Finding.Since.Before(at) && cur.Finding.Since.After(w.now()) {
|
||
return
|
||
}
|
||
w.signal("dead", key, left+time.Second, signalDue{Do: dueRecount,
|
||
Finding: Finding{Scope: ScopeBus, ID: key, Kind: KindDeadLetters, Severity: Warning, Summary: "counted again when it ages out", Source: "S9", Since: at}})
|
||
}
|
||
|
||
func orWords(s, otherwise string) string {
|
||
if strings.TrimSpace(s) == "" {
|
||
return otherwise
|
||
}
|
||
return s
|
||
}
|
||
|
||
// SignalRows are the rows the watchman holds, in the table's order, for `signals`.
|
||
var SignalRows = []struct{ Row, Signal, Bound, Kind string }{
|
||
{"S1", "a machine's presence (not while it said it sleeps or shuts down)", SilentBound.String(), KindSilent},
|
||
{"S8", "a provider's failing word repeated", ProviderSaysAgainWithin.String(), KindProviderSilent},
|
||
{"S9", "the bus's advisories", "any occurrence; clears after " + AdvisoryQuiet.String() + " without another", "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost"},
|
||
{"S10", "the self-check's checked", (3 * SelfCheckEvery).String(), KindSelfCheckSilent},
|
||
{"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent},
|
||
{"S21", "a machine's provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing},
|
||
{"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits},
|
||
{actingRow, "an acting part's wait held, or a put's, a put without hold that stands (the seats' wait-held, ended by wait-ended)", "the wait's own bound", KindActingPartLate + ", " + KindPutStands},
|
||
{storeRow, "a store of the delivery's parts filling or full (the seats' filling, full and room)", "filling from " + fmt.Sprintf("%.0f", FillingFrom*100) + " %, full urgent", KindStoreFilling},
|
||
{recordRow, "a delivery record open (the seat mesh-delivery's record-opened, ended by record-ended)", RecordOpenBound.String(), KindRecordOpen},
|
||
{"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator},
|
||
}
|
||
|
||
// Signal is one signal's pulse as `signals` shows it.
|
||
type Signal struct {
|
||
Key string `json:"key"`
|
||
Do string `json:"do"`
|
||
Finding Finding `json:"finding"`
|
||
}
|
||
|
||
// Signals is every signal's pulse the watchman holds, sorted by key.
|
||
func (w *Watchman) Signals() []Signal {
|
||
w.mu.Lock()
|
||
defer w.mu.Unlock()
|
||
out := make([]Signal, 0, len(w.signals))
|
||
for k, d := range w.signals {
|
||
out = append(out, Signal{Key: k, Do: d.Do, Finding: d.Finding})
|
||
}
|
||
sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
|
||
return out
|
||
}
|