Files
mesh-delivery/modules/mesh-delivery-watchman/internal/watch/signals.go
T
jschoubben e8d1bf96be Names N3 (watchman): the watchman names its own buckets from its credential's module
Its states presence, pulse and stuck and its first views were named from the constant
WatchmanModule; they are now `<cred.Module>_<state>` (watch.BucketsOf), and the watchman's
stuck keys of presence take its own presence bucket's name.

Builds: design 47 G A-296

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 19:09:46 +01:00

805 lines
30 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package watch
import (
"encoding/json"
"fmt"
"regexp"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
)
// The rows of the signals table the watchman holds (novox/hq ADR 0564 §4, Limits 3 and 11). Each is a key of the pulse
// bucket whose time to live is the row's bound, renewed by the signal's own event: its expiry is the finding, so no loop
// looks every half minute. A signal's pulse is kept under the pulse bucket's pseudo-bucket `signal`, `signal.<row>.<id>`,
// and its value says what its expiry does (signalDue), so a pulse that stood across a restart still acts when it expires.
//
// What each row hears is on the bus, never in the controller's store:
// - S1, a machine silent: the watchman's own presence of `node.<node>` (its stuck key), not while the machine said it
// sleeps or shuts down (the power seat's events, ADR 0211);
// - S8, a provider's failing word repeated: the providers' `provisioner.failing` and `provisioner.recovered` (ADR 0224);
// the failing word is itself the finding `provider-failing`, and its pulse of 30 minutes, renewed by each repeat, is S8;
// - S9, the bus's advisories: each occurrence a finding, cleared by the expiry of an hour's pulse renewed by the next;
// - S10, the self-check's `checked`: a pulse of three runs' length, from the first run heard;
// - S11, a machine's tool runner heartbeat on `mesh.control.<node>.tools-alive`: a pulse of three intervals;
// - S21, a module's have failed: a pulse of one apply interval from the first failure, whose expiry while it still fails
// is the finding `apply-failing`;
// - a module's have still failed a minute after it first said so, `unhealthy`, urgent after four hours; and a have that waits for
// the operator, `needs-operator` (ADR 0240 §12, §17).
// The bounds of the rows.
const (
// ProviderSaysAgainWithin is how long a provider's failing word stays current without being said again (S8): twice
// the quarter of an hour it repeats it at.
ProviderSaysAgainWithin = 30 * time.Minute
// AdvisoryQuiet is how long the bus must be quiet about a thing before its advisory clears (S9).
AdvisoryQuiet = time.Hour
// SelfCheckEvery is the self-check's run interval; S10's bound is three of them.
SelfCheckEvery = 5 * time.Minute
// HeartbeatEvery is a tool runner's interval when it says none; S11's bound is three of them.
HeartbeatEvery = 60 * time.Second
// ApplyEvery is one apply interval (S21): a failure still there after it is not one the next apply clears.
ApplyEvery = 5 * time.Minute
// SilentBound is the bound S1 says, three of the intervals a machine has always used (the controller's silentBound).
SilentBound = 3 * HeartbeatEvery
)
// signalBucket is the pseudo-bucket of a signal's pulse key.
const signalBucket = "signal"
// signalDue is what a signal's pulse does when it expires: raise the finding (find), or end it (end, an advisory's quiet
// hour). Kept as the pulse's Owes, so it is read back at a restart.
type signalDue struct {
Do string `json:"do"`
Finding Finding `json:"finding"`
// While names what must still stand at the expiry for a find to be written: a have's key still failing or unhealthy;
// empty for a row whose expiry alone is the finding.
While string `json:"while,omitempty"`
}
const (
dueFind = "find"
dueEnd = "end"
// dueApply judges a machine's S21 again (applyCheck); dueRecount counts DEAD_LETTERS again (a restricted letter aged out).
dueApply = "apply"
dueRecount = "recount"
// dueCreated is no due: the mark, with no time to live, that the bus module said an entry created (busstate.go).
dueCreated = "created"
)
// signalKey is the pulse key of a row's signal about id.
func signalKey(row, id string) string {
parts := []string{signalBucket, row}
for _, p := range strings.Split(id, ".") {
if p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_"); p == "" {
p = "_"
}
parts = append(parts, p)
}
return strings.Join(parts, ".")
}
// unsafeKey is what a key's token may not hold: the bus takes letters, digits and `-_/=`.
var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`)
// signal writes (or renews) the pulse of a row's signal, whose expiry does due.
func (w *Watchman) signal(row, id string, ttl time.Duration, due signalDue) {
pk := signalKey(row, id)
owes, err := json.Marshal(due)
if err != nil {
w.say("no signal %s: %v", pk, err)
return
}
value, err := meshwatchdogsclient.EncodePulse(meshwatchdogsclient.Pulse{Bucket: signalBucket, Key: strings.TrimPrefix(pk, signalBucket+"."), Owes: string(owes)})
if err != nil {
w.say("no signal %s: %v", pk, err)
return
}
if err := w.out.PutPulse(pk, value, ttl); err != nil {
w.say("cannot write the signal %s: %v", pk, err)
return
}
w.signals[pk] = due
}
// unsignal ends a row's signal about id: its pulse deleted, so it never expires.
func (w *Watchman) unsignal(row, id string) {
pk := signalKey(row, id)
if _, ok := w.signals[pk]; !ok {
return
}
delete(w.signals, pk)
if err := w.out.DeletePulse(pk); err != nil {
w.say("cannot delete the signal %s: %v", pk, err)
}
}
// SeedSignal tells the watchman a signal's pulse that stands at its start, with its value: it is kept, since its event
// renews it and its expiry still acts.
func (w *Watchman) SeedSignal(key string, value []byte) {
w.mu.Lock()
defer w.mu.Unlock()
if p, err := meshwatchdogsclient.DecodePulse(value); err == nil && p.Bucket == signalBucket {
var due signalDue
if json.Unmarshal([]byte(p.Owes), &due) == nil && due.Do == dueCreated {
w.busCreated[key] = true
return
}
}
if due, ok := dueOf(value); ok {
w.signals[key] = due
}
}
func dueOf(value []byte) (signalDue, bool) {
p, err := meshwatchdogsclient.DecodePulse(value)
if err != nil || p.Bucket != signalBucket {
return signalDue{}, false
}
var due signalDue
if json.Unmarshal([]byte(p.Owes), &due) != nil || (due.Do != dueFind && due.Do != dueEnd && due.Do != dueApply && due.Do != dueRecount) {
return signalDue{}, false
}
return due, true
}
// signalExpired is a signal's pulse expired: its due is done.
func (w *Watchman) signalExpired(pk string) {
due, ok := w.signals[pk]
if !ok {
return
}
delete(w.signals, pk)
switch due.Do {
case dueApply:
w.applyCheck(due.While)
case dueRecount:
if w.recount != nil {
go w.recount()
}
case dueEnd:
w.unfind(due.Finding.Key())
case dueFind:
if due.While == uncovered {
w.opened[due.Finding.ID] = due.Finding
w.rejudgeRecords()
return
}
if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) {
return
}
w.find(due.Finding)
}
}
// stillWrong says a silent machine's finding still stands, which a row's finding waited on.
func (w *Watchman) stillWrong(key, _ string) bool {
if node, ok := strings.CutPrefix(key, "silent:"); ok {
_, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)]
return open
}
return false
}
// The kinds the watchman's rows write, as the controller's conditions name them.
const (
KindSilent = "silent"
KindStuck = "stuck"
KindProviderFailing = "provider-failing"
KindProviderSilent = "provider-silent"
KindSelfCheckSilent = "self-check-silent"
KindToolsSilent = "tools-silent"
KindApplyFailing = "apply-failing"
KindNeedsOperator = "needs-operator"
KindDeadLetters = "max-deliveries"
)
func stamp(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") }
// ---- S1 and the power seat ----
// PowerEvents are the power seat's events that say whether a machine is about to be away or is back (ADR 0211).
var PowerEvents = map[string]bool{"sleeping": true, "shutting-down": true, "booted": true, "woke": true}
// Power is a machine's power event: sleeping and shutting-down say it is away, booted and woke that it is back. A machine
// that said it is away is not silent; one back that is still offline is watched again from now.
func (w *Watchman) Power(node, state string) {
if !PowerEvents[state] || node == "" {
return
}
w.mu.Lock()
defer w.mu.Unlock()
away := state == "sleeping" || state == "shutting-down"
w.away[node] = away
key, err := meshwatchdogsclient.PresenceNodeKey(node)
if err != nil {
return
}
defer w.applyCheck(node)
if away {
w.unfind(FindingKey(ScopeMachine, node, KindSilent))
w.unsignal("S1", node)
return
}
if p, ok := w.presence[key]; ok && !p.online {
p.since = w.now()
w.presence[key] = p
w.ensureOffline(key)
}
}
// stuckFinding is the finding of a stuck key as it stands (ADR 0433 §2, §3, now the watchman's own): a node's presence not
// moved is S1, the machine silent, unless it said it is away; any other key not moved is `delivery.<key>.stuck`; a key
// moved ends its finding.
func (w *Watchman) stuckFinding(sk string, s meshwatchdogsclient.Stuck) {
if s.Bucket == w.own.Presence {
if node, part, _, err := meshwatchdogsclient.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" {
key := FindingKey(ScopeMachine, node, KindSilent)
if s.Moved {
w.machineBack(node)
} else {
w.machineAway(node, s.Since)
}
if s.Moved || w.away[node] {
w.unfind(key)
w.unsignal("S1", node)
w.applyCheck(node)
return
}
f := Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1",
Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), SilentBound),
Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since}
w.find(f)
w.applyCheck(node)
return
}
}
key := FindingKey(ScopeDelivery, sk, KindStuck)
if s.Moved {
w.unfind(key)
return
}
waited := s.Waited
if waited == "" {
waited = "a move"
}
w.find(Finding{Scope: ScopeDelivery, ID: sk, Kind: KindStuck, Severity: Warning, Source: "stuck",
Summary: fmt.Sprintf("%s has not moved since %s: it waited for %s", sk, stamp(s.Since), waited),
Said: fmt.Sprintf("%s: waiting for %s since %s", sk, waited, s.Since.UTC().Format(time.RFC3339)), Since: s.Since})
}
// ---- S8 ----
// Standing is a provider's word about one consumer (ADR 0224): Module is the emitter, read from the subject.
type Standing struct {
Module string `json:"-"`
Failing bool `json:"-"`
Provider string `json:"provider"`
ProviderNode string `json:"provider-node"`
Consumer string `json:"consumer"`
Node string `json:"node"`
Class string `json:"class,omitempty"`
Error string `json:"error,omitempty"`
Since time.Time `json:"since"`
Attempts int `json:"attempts"`
Why string `json:"why,omitempty"`
}
// ReadStanding reads a provider's word from its subject, `mesh.mod.<module>.event.provisioner.failing` or `.recovered`.
func ReadStanding(subject string, body []byte) (Standing, error) {
rest, ok := strings.CutPrefix(subject, "mesh.mod.")
if !ok {
return Standing{}, fmt.Errorf("%s is not a provider's word", subject)
}
module, event, ok := strings.Cut(rest, ".event.")
if !ok || module == "" || (event != "provisioner.failing" && event != "provisioner.recovered") {
return Standing{}, fmt.Errorf("%s is not a provider's word", subject)
}
var st Standing
if err := json.Unmarshal(body, &st); err != nil {
return Standing{}, fmt.Errorf("%s's word could not be read: %w", module, err)
}
if st.Consumer == "" {
return Standing{}, fmt.Errorf("%s's word names no consumer", module)
}
st.Module, st.Failing = module, event == "provisioner.failing"
return st, nil
}
// Stood is a provider's word: failing writes `provider.<module>.<node>.<consumer>.failing` and renews S8's pulse, whose
// expiry writes `provider-silent`; recovered deletes both (Limits 11).
func (w *Watchman) Stood(st Standing) {
w.mu.Lock()
defer w.mu.Unlock()
id := st.Module + "." + st.ProviderNode + "." + st.Consumer
failing := FindingKey(ScopeProvider, id, "failing")
silentKey := FindingKey(ScopeProvider, id, "silent")
if !st.Failing {
w.unfind(failing)
w.unfind(silentKey)
w.unsignal("S8", id)
return
}
class := st.Class
if class == "" {
class = "failing"
}
whom := st.Consumer
if st.Node != "" {
whom += " on " + st.Node
}
said := class
if st.Error != "" {
said += ": " + firstLine(st.Error)
}
var also []string
if st.Node != "" && st.Node != st.ProviderNode {
also = []string{st.Node}
}
w.find(Finding{Scope: ScopeProvider, ID: id, Token: "failing", Kind: KindProviderFailing, Machine: st.ProviderNode, Also: also,
Severity: Warning, Source: "provisioner.failing", Since: st.Since, Said: said,
Summary: fmt.Sprintf("%s on %s keeps failing %s: %s, %d attempt(s) since %s", st.Module, st.ProviderNode, whom, class,
st.Attempts, stamp(st.Since))})
w.unfind(silentKey)
now := w.now()
w.signal("S8", id, ProviderSaysAgainWithin, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeProvider, ID: id, Token: "silent",
Kind: KindProviderSilent, Machine: st.ProviderNode, Severity: Warning, Source: "S8", Since: now,
Summary: fmt.Sprintf("%s on %s said it keeps failing %s and has said nothing since %s: it stopped saying anything, so "+
"its last word is all the mesh has", st.Module, st.ProviderNode, st.Consumer, stamp(now)),
Said: fmt.Sprintf("not said again within %s", ProviderSaysAgainWithin)}})
}
func firstLine(s string) string {
s, _, _ = strings.Cut(strings.TrimSpace(s), "\n")
return s
}
// ---- S9 ----
// Advisory is one thing the bus said (S9): its kind (slow-consumer, connection-closed, max-deliveries, refused,
// consumer-lost), what it is about, its words, and the machine when it names one.
type Advisory struct {
Kind, ID, Said, Machine string
}
// Advised is an advisory heard: its finding `bus.<id>.<kind>` is written at once, counting the times since its first, and
// an hour's pulse renewed by each, whose expiry ends it.
func (w *Watchman) Advised(a Advisory) {
if a.Kind == "" || a.ID == "" {
return
}
w.mu.Lock()
defer w.mu.Unlock()
f := Finding{Scope: ScopeBus, ID: a.ID, Kind: a.Kind, Machine: a.Machine, Severity: Warning, Source: "S9", Said: a.Said}
key := f.Key()
n := w.advised[key] + 1
if _, open := w.findings[key]; !open {
n = 1
w.advisedSince[key] = w.now()
}
w.advised[key] = n
since := w.advisedSince[key]
f.Since, f.Summary = since, a.Said
if n > 1 {
f.Summary = fmt.Sprintf("%s (%d times since %s)", a.Said, n, since.UTC().Format("15:04 MST"))
}
w.find(f)
w.signal("S9", key, AdvisoryQuiet, signalDue{Do: dueEnd, Finding: f})
}
// ---- S10 ----
// Checked is the self-check's `checked` heard: its pulse of three runs' length is renewed, and its silence ended. Before
// the first run heard there is no pulse, so a self-check not yet placed is not said silent.
func (w *Watchman) Checked(machine string) {
w.mu.Lock()
defer w.mu.Unlock()
w.unfind(FindingKey(ScopeCore, "doctor", KindSelfCheckSilent))
now := w.now()
w.signal("S10", "doctor", 3*SelfCheckEvery, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeCore, ID: "doctor",
Kind: KindSelfCheckSilent, Machine: machine, Severity: Urgent, Source: "S10", Since: now,
Summary: fmt.Sprintf("the self-check has not finished a run since %s (it runs every %s): the mesh's invariants are not "+
"being checked", stamp(now), SelfCheckEvery),
Said: fmt.Sprintf("no run within %s", 3*SelfCheckEvery)}})
}
// ---- S11 ----
// ToolsAlive is a machine's tool runner heartbeat, saying its interval (zero: HeartbeatEvery): its pulse of three
// intervals is renewed and its silence ended.
func (w *Watchman) ToolsAlive(node string, every time.Duration) {
if node == "" {
return
}
if every <= 0 {
every = HeartbeatEvery
}
w.mu.Lock()
defer w.mu.Unlock()
w.unfind(FindingKey(ScopeMachine, node, KindToolsSilent))
if w.away[node] {
w.unsignal("S11", node)
return
}
now := w.now()
w.signal("S11", node, 3*every, signalDue{Do: dueFind, Finding: Finding{Scope: ScopeMachine, ID: node, Kind: KindToolsSilent,
Machine: node, Severity: Warning, Source: "S11", Since: now,
Summary: fmt.Sprintf("the node tools on %s have not said they are there since %s (bound %s): nothing can ask that "+
"machine anything", node, stamp(now), 3*every),
Said: fmt.Sprintf("silent for more than %s", 3*every)}})
}
// ToolsNode is the machine a tool runner heartbeat's subject names, `mesh.control.<node>.tools-alive`.
func ToolsNode(subject string) string {
rest, ok := strings.CutPrefix(subject, "mesh.control.")
if !ok {
return ""
}
node, ok := strings.CutSuffix(rest, ".tools-alive")
if !ok || strings.Contains(node, ".") {
return ""
}
return node
}
// ---- S21, from the provisioner clients' reports ----
// Provision is a change of a key of mesh-provisioner's `provisions` (ADR 0570 §7): a provision in `waits` is a part of its
// consumer waiting for the operator, said as `module.<consumer>.<node>.needs-operator` naming each waiting provision and
// why; the finding ends when no provision of that consumer on that node waits. A key deleted, or one that cannot be read,
// waits no more.
func (w *Watchman) Provision(key string, value []byte, op Op) {
node, consumer, _, _, err := meshprovisionerclient.ParseKey(key)
if err != nil {
return
}
w.mu.Lock()
defer w.mu.Unlock()
id := consumer + "." + node
if w.waiting[id] == nil {
w.waiting[id] = map[string]string{}
}
delete(w.waiting[id], key)
if op == Put {
if p, err := meshprovisionerclient.Decode(value); err != nil {
w.say("the provision %s cannot be read, and is taken as not waiting: %v", key, err)
} else if p.State == meshprovisionerclient.Waits {
w.waiting[id][key] = fmt.Sprintf("%s from %s on %s: %s", p.Provision, p.Provider.Module, p.Provider.Node, p.Why)
}
}
w.askedWait(key, value, op)
fkey := FindingKey(ScopeModule, id, KindNeedsOperator)
if len(w.waiting[id]) == 0 {
delete(w.waiting, id)
w.unfind(fkey)
return
}
var said []string
for _, k := range sortedKeys(w.waiting[id]) {
said = append(said, w.waiting[id][k])
}
w.find(Finding{Scope: ScopeModule, ID: id, Kind: KindNeedsOperator, Machine: node, Severity: Warning, Source: "provisions",
Summary: fmt.Sprintf("%s on %s waits for the operator: %s", consumer, node, strings.Join(said, "; ")),
Said: strings.Join(said, "; ")})
}
// askedWait is the wait of a provision its provider has not provided (ADR 0603 §8): a key asked is mirrored from the time the
// holder wrote it, bounded as a step is, and provided, failed, waiting for the operator (its own finding), retiring or deleted
// ends it. Called with the lock held.
func (w *Watchman) askedWait(key string, value []byte, op Op) {
bucket := ProvisionsBucket
if op == Expired {
return
}
w.stands(bucket, key)
if op == Delete {
w.moved(bucket, key)
return
}
p, err := meshprovisionerclient.Decode(value)
if err != nil || p.State != meshprovisionerclient.Asked {
w.moved(bucket, key)
return
}
since := p.At
if since.IsZero() {
since = w.now()
}
words := fmt.Sprintf("%s from %s on %s to be provided for %s on %s, bound %s", p.Provision, p.Provider.Module, p.Provider.Node,
p.Consumer.Module, p.Consumer.Node, w.stepBound)
if p.Why != "" {
words += ": " + p.Why
}
w.mirrorKept(bucket, key, words, "asked", since)
}
func sortedKeys(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}
// ProvReport is a machine's provisioner client's word on one module's files (`prov.applied.<node>.<module>`, ADR 0362 §3).
type ProvReport struct {
Node string `json:"node"`
Module string `json:"module"`
Status string `json:"status"`
Reason string `json:"reason,omitempty"`
At time.Time `json:"at"`
}
// ProvFailed is the status of a report whose files were not placed.
const ProvFailed = "failed"
// Provisioned is a provisioner client's newest report: a failure is kept for S21, any other word ends it.
func (w *Watchman) Provisioned(r ProvReport) {
if r.Node == "" || r.Module == "" {
return
}
w.mu.Lock()
defer w.mu.Unlock()
if w.provFailed[r.Node] == nil {
w.provFailed[r.Node] = map[string]ProvReport{}
}
if r.Status == ProvFailed {
w.provFailed[r.Node][r.Module] = r
} else {
delete(w.provFailed[r.Node], r.Module)
}
w.applyCheck(r.Node)
}
// heard says the watchman hears the machine now: online, and not said away. S21 is judged only while it is (the
// controller's watchApplyFailing): S1 says silent, and asleep is not lost.
func (w *Watchman) heard(node string) bool {
key, err := meshwatchdogsclient.PresenceNodeKey(node)
if err != nil || w.away[node] {
return false
}
p, ok := w.presence[key]
return ok && p.online
}
// applyCheck is S21 for one machine, `machine.<node>.apply-failing` (the controller's watchApplyFailing): raised once the
// first failure it still has is older than one apply interval while the machine is heard, naming the modules its node
// failed to apply (their haves failed), or else those whose files its provisioner client failed to place; ended by the
// clean apply. Before the bound a pulse judges it again at the bound.
func (w *Watchman) applyCheck(node string) {
key := FindingKey(ScopeMachine, node, KindApplyFailing)
type failure struct{ module, words string }
var placed []failure
var since time.Time
for module, r := range w.provFailed[node] {
placed = append(placed, failure{module, firstLine(orWords(r.Reason, "no reason given"))})
if since.IsZero() || r.At.Before(since) {
since = r.At
}
}
if since.IsZero() || !w.heard(node) {
w.unsignal("S21", node)
w.unfind(key)
return
}
if age := w.now().Sub(since); age <= ApplyEvery {
if _, pending := w.signals[signalKey("S21", node)]; !pending {
w.signal("S21", node, ApplyEvery-age+time.Second, signalDue{Do: dueApply, While: node,
Finding: Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Severity: Warning, Summary: "judged again at its bound", Source: "S21"}})
}
return
}
failed := placed
sort.Slice(failed, func(i, j int) bool { return failed[i].module < failed[j].module })
var names, words []string
tool := ""
for _, f := range failed {
names = append(names, f.module)
words = append(words, f.module+": "+f.words)
if rest, ok := strings.CutPrefix(f.words, "refused: "); ok {
f.words = rest
}
if i := strings.Index(f.words, PackageManagerKey); i >= 0 && tool == "" {
tool = strings.TrimSuffix(strings.SplitN(f.words[i+len(PackageManagerKey):], ")", 2)[0], ")")
}
}
f := Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Machine: node, Severity: Warning, Source: "S21", Since: since,
Said: strings.Join(words, "; "), Headline: node + " is not applying its update"}
f.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", node,
strings.Join(names, ", "), stamp(since), ApplyEvery)
f.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s. The details say why.",
node, strings.Join(names, ", "))
if tool != "" {
f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is placed, "+
"since %s (bound %s)", node, tool, stamp(since), ApplyEvery)
f.Headline = node + " cannot install packages"
}
w.find(f)
}
// PackageManagerKey is how a node-engine names a package manager that does not answer among what failed.
const PackageManagerKey = "the package manager ("
// ---- S9: the bus module's closures ----
// BusClosure is one connection the bus closed, as the bus module says it (`bus.slow-consumer`, `bus.connection-closed`).
type BusClosure struct {
User string `json:"user"`
Name string `json:"name,omitempty"`
CID uint64 `json:"cid"`
Reason string `json:"reason"`
Opened time.Time `json:"opened"`
Closed time.Time `json:"closed"`
Pending int64 `json:"pending_bytes,omitempty"`
}
// The kinds of the bus module's closures, as conditions name them.
const (
KindSlowConsumer = "slow-consumer"
KindConnectionClosed = "connection-closed"
)
// Closed is one closure the bus module said (S9, the controller's BusClosures and busUserClosed): one finding per user and
// kind, `bus.<user>.<kind>`, each closure counted once however often it is said, ended an hour after the newest.
func (w *Watchman) Closed(kind string, c BusClosure) {
if c.User == "" || c.Closed.IsZero() || (kind != KindSlowConsumer && kind != KindConnectionClosed) {
return
}
w.mu.Lock()
defer w.mu.Unlock()
f := Finding{Scope: ScopeBus, ID: c.User, Kind: kind, Severity: Warning, Source: "S9"}
key := f.Key()
if _, open := w.findings[key]; !open {
w.closures[key] = map[string]time.Time{}
}
identity := fmt.Sprintf("%s/%d/%s", c.User, c.CID, c.Opened.UTC().Format(time.RFC3339Nano))
if _, seen := w.closures[key][identity]; seen {
return
}
w.closures[key][identity] = c.Closed
var first, last time.Time
for _, at := range w.closures[key] {
if first.IsZero() || at.Before(first) {
first = at
}
if at.After(last) {
last = at
}
}
n := len(w.closures[key])
who, machine := BusUserWords(c.User)
f.Machine, f.Since = machine, first
times, count := "once", "once"
if n > 1 {
times = fmt.Sprintf("%d times since %s", n, first.UTC().Format("15:04 MST"))
count = fmt.Sprintf("%d times", n)
}
if kind == KindSlowConsumer {
f.Said = fmt.Sprintf("the bus closed %s's connection %d as a slow consumer: %s", c.User, c.CID, c.Reason)
f.Summary = fmt.Sprintf("the bus closed the connection of %s as a slow consumer %s in the last hour, newest at %s: it "+
"could not take what it was sent fast enough, and what was on its way to it was lost", who, times, last.UTC().Format("15:04 MST"))
f.Headline = clip(capital(who)+" was too slow for the bus", 60)
f.Explanation = capital(fmt.Sprintf("%s could not keep up with what the bus sent it, so the bus closed its connection (%s "+
"in the last hour) and what was on its way was lost. It connects again by itself; this clears an hour after the last time.", who, count))
} else {
reason := strings.ToLower(c.Reason)
f.Said = fmt.Sprintf("the bus closed %s's connection %d: %s", c.User, c.CID, c.Reason)
f.Summary = fmt.Sprintf("the bus closed the connection of %s for breaking its rules %s in the last hour, newest at %s: %s",
who, times, last.UTC().Format("15:04 MST"), reason)
f.Headline = clip(capital(who)+" was cut off by the bus", 60)
f.Explanation = capital(fmt.Sprintf("The bus closed the connection of %s because it broke the bus's rules (%s, %s in the "+
"last hour), so what it was sending was lost. This clears an hour after the last time.", who, reason, count))
}
w.find(f)
if quiet := AdvisoryQuiet - w.now().Sub(last); quiet > 0 {
w.signal("S9", key, quiet, signalDue{Do: dueEnd, Finding: f})
} else {
w.unfind(key)
}
}
// BusUserWords is a bus user as the operator says it, and its machine (the controller's busUserWords).
func BusUserWords(user string) (string, string) {
switch {
case user == "controller":
return "the controller", ""
case strings.HasPrefix(user, "node."):
m := strings.TrimPrefix(user, "node.")
return "the node-engine on " + m, m
}
if m, module, ok := strings.Cut(user, "."); ok {
return module + " on " + m, m
}
return "a client of the bus", ""
}
func capital(s string) string {
if s == "" {
return s
}
return strings.ToUpper(s[:1]) + s[1:]
}
func clip(s string, n int) string {
if len(s) <= n {
return s
}
cut := s[:n]
if i := strings.LastIndex(cut, " "); i > 0 {
cut = cut[:i]
}
return cut
}
// ---- dead letters aging out ----
// SetRecount sets how DEAD_LETTERS is counted again when a restricted letter ages out.
func (w *Watchman) SetRecount(f func()) {
w.mu.Lock()
defer w.mu.Unlock()
w.recount = f
}
// DeadAging is a restricted letter of a consumer (`<stream>.<consumer>`) that ages out at at: a pulse expiring a second
// after it counts DEAD_LETTERS again, so the consumer's finding ends when its letters are gone (ADR 0376 §6), with no
// timer. The earliest of a consumer's letters is the one pulsed.
func (w *Watchman) DeadAging(key string, at time.Time) {
w.mu.Lock()
defer w.mu.Unlock()
left := at.Sub(w.now())
if left <= 0 {
left = 0
}
pk := signalKey("dead", key)
if cur, ok := w.signals[pk]; ok && !cur.Finding.Since.IsZero() && cur.Finding.Since.Before(at) && cur.Finding.Since.After(w.now()) {
return
}
w.signal("dead", key, left+time.Second, signalDue{Do: dueRecount,
Finding: Finding{Scope: ScopeBus, ID: key, Kind: KindDeadLetters, Severity: Warning, Summary: "counted again when it ages out", Source: "S9", Since: at}})
}
func orWords(s, otherwise string) string {
if strings.TrimSpace(s) == "" {
return otherwise
}
return s
}
// SignalRows are the rows the watchman holds, in the table's order, for `signals`.
var SignalRows = []struct{ Row, Signal, Bound, Kind string }{
{"S1", "a machine's presence (not while it said it sleeps or shuts down)", SilentBound.String(), KindSilent},
{"S8", "a provider's failing word repeated", ProviderSaysAgainWithin.String(), KindProviderSilent},
{"S9", "the bus's advisories", "any occurrence; clears after " + AdvisoryQuiet.String() + " without another", "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost"},
{"S10", "the self-check's checked", (3 * SelfCheckEvery).String(), KindSelfCheckSilent},
{"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent},
{"S21", "a machine's provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing},
{"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits},
{actingRow, "an acting part's wait held, or a put's, a put without hold that stands (the seats' wait-held, ended by wait-ended)", "the wait's own bound", KindActingPartLate + ", " + KindPutStands},
{storeRow, "a store of the delivery's parts filling or full (the seats' filling, full and room)", "filling from " + fmt.Sprintf("%.0f", FillingFrom*100) + " %, full urgent", KindStoreFilling},
{recordRow, "a delivery record open (the seat mesh-delivery's record-opened, ended by record-ended)", RecordOpenBound.String(), KindRecordOpen},
{"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator},
}
// Signal is one signal's pulse as `signals` shows it.
type Signal struct {
Key string `json:"key"`
Do string `json:"do"`
Finding Finding `json:"finding"`
}
// Signals is every signal's pulse the watchman holds, sorted by key.
func (w *Watchman) Signals() []Signal {
w.mu.Lock()
defer w.mu.Unlock()
out := make([]Signal, 0, len(w.signals))
for k, d := range w.signals {
out = append(out, Signal{Key: k, Do: d.Do, Finding: d.Finding})
}
sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
return out
}