jschoubben
43d4f511c5
ADR 0590 supersedes the lease write: mesh-grants says no lease event any more, so the watchman's lease-altered and removal-failed findings and their row had nothing to hear. The ended controller's consumers are removed as hand acts, so their deletion is no consumer lost; only the one on DEAD_LETTER_NOTICES is still read, since its removal hands the notices to the watchman. The builder hears artifacts-gone from the artifact sweep seat alone, and mesh-enrolment's tokens no longer report an issuing gate that nothing can turn off. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
382 lines
14 KiB
Go
382 lines
14 KiB
Go
// Package watch is the watchman's logic (novox/hq ADR 0433, ADR 0431, issue 1135): it turns the bus's connect and
|
|
// disconnect events, the connections request and the buckets it mirrors (the builds bucket; the controller's
|
|
// checks-running bucket went with the build queue in R4g, ADR 0525 §9) into presence, pulse and stuck keys. It holds no timer: every reaction is an event, a watch entry or a request the caller makes.
|
|
package watch
|
|
|
|
import (
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// Op is what happened to a watched key.
|
|
type Op int
|
|
|
|
const (
|
|
// Put is a key written.
|
|
Put Op = iota
|
|
// Delete is a key deleted by a writer (the watchman's own delete, or a builder ending a check).
|
|
Delete
|
|
// Expired is the server's marker for a key whose time to live ran out.
|
|
Expired
|
|
)
|
|
|
|
// Out is where the watchman writes. Every call is a write the caller makes on the bus.
|
|
type Out interface {
|
|
PutPresence(key string, online bool) error
|
|
// PutPulse writes the pulse key with a time to live, replacing a live one.
|
|
PutPulse(key string, value []byte, ttl time.Duration) error
|
|
// DeletePulse deletes the pulse key (a delete, not an expiry).
|
|
DeletePulse(key string) error
|
|
PutStuck(key string, value []byte) error
|
|
// PutFinding writes one of the watchman's findings; DeleteFinding deletes one it no longer observes.
|
|
PutFinding(key string, value []byte) error
|
|
DeleteFinding(key string) error
|
|
}
|
|
|
|
type presence struct {
|
|
online bool
|
|
since time.Time
|
|
}
|
|
|
|
type pulse struct {
|
|
bucket, key, waited string
|
|
since time.Time
|
|
head string
|
|
token string
|
|
seeded bool
|
|
}
|
|
|
|
// Watchman is the state the watchman keeps in memory; it is rebuilt from the buckets at every start. Its methods are
|
|
// safe to call from several watches at once.
|
|
type Watchman struct {
|
|
mu sync.Mutex
|
|
out Out
|
|
now func() time.Time
|
|
logf func(format string, args ...any)
|
|
counts map[string]int // open connections per presence key
|
|
presence map[string]presence
|
|
pulses map[string]*pulse // live pulses, by pulse key
|
|
stuck map[string]delivery.Stuck
|
|
wants map[string]string // the fingerprint of each want that stands, by want key
|
|
haves map[string]string // the fingerprint each node runs of a module, by have key
|
|
bound time.Duration
|
|
// findings is every finding written, by key, as written; seededFindings those read at the start and not yet observed
|
|
// again.
|
|
findings map[string][]byte
|
|
seededFindings map[string]bool
|
|
// signals is every signal's pulse standing, by pulse key, with what its expiry does.
|
|
signals map[string]signalDue
|
|
// away is each machine that said it sleeps or shuts down, until it says it is back.
|
|
away map[string]bool
|
|
// haveState is each module's have as last read, unhealthy its writes in a row that said it failed, advised each
|
|
// advisory's count while its finding stands and advisedSince since when.
|
|
haveState map[string]delivery.Have
|
|
unhealthy map[string]int
|
|
advised map[string]int
|
|
advisedSince map[string]time.Time
|
|
// control is each node running the controller; failedSince each have's first failure still standing; provFailed each
|
|
// machine's provisioner client's failed reports; closures each bus closure counted, by finding key and identity.
|
|
control map[string]bool
|
|
failedSince map[string]time.Time
|
|
provFailed map[string]map[string]ProvReport
|
|
closures map[string]map[string]time.Time
|
|
recount func()
|
|
// waiting is, by `<consumer>.<node>`, each of its provisions in waits, with its words.
|
|
waiting map[string]map[string]string
|
|
// health is each node's own health report, and tunnels each node's tunnel key from its record.
|
|
health map[string]delivery.NodeHealth
|
|
tunnels map[string]string
|
|
// stepBound is how long a build step may run once a node took the ask; a test shortens it.
|
|
stepBound time.Duration
|
|
// standing is, until the start ends (Reconcile), each mirrored key read standing, by the bucket a stuck key names it
|
|
// under; readWhole the buckets whose keys were all read at this start (ReadWhole). reconciled is the start's end.
|
|
standing map[string]map[string]bool
|
|
readWhole map[string]bool
|
|
reconciled bool
|
|
}
|
|
|
|
// New is a watchman writing to out, reading the time from now.
|
|
func New(out Out, now func() time.Time) *Watchman {
|
|
return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{},
|
|
pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound,
|
|
findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{},
|
|
haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{},
|
|
control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{},
|
|
health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{},
|
|
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}}
|
|
}
|
|
|
|
// Logf sets where a write that failed is said; it is silent until set.
|
|
func (w *Watchman) Logf(f func(format string, args ...any)) { w.logf = f }
|
|
|
|
func (w *Watchman) say(format string, args ...any) {
|
|
if w.logf != nil {
|
|
w.logf(format, args...)
|
|
}
|
|
}
|
|
|
|
// mirror makes the pulse of the move (bucket, key) that began at since, unless it stands already for the same move.
|
|
// A move that is already as old as the bound is stuck at once and has no pulse.
|
|
func (w *Watchman) mirror(bucket, key, waited, head string, since time.Time, revision uint64) {
|
|
w.mirrorWithin(0, bucket, key, waited, head, since, revision)
|
|
}
|
|
|
|
// mirrorWithin is mirror for a move that may take bound (zero: the watchman's own bound, delivery.PulseMoveBound).
|
|
func (w *Watchman) mirrorWithin(bound time.Duration, bucket, key, waited, head string, since time.Time, revision uint64) {
|
|
pk, err := delivery.PulseKey(bucket, key)
|
|
if err != nil {
|
|
w.say("no pulse for %s %s: %v", bucket, key, err)
|
|
return
|
|
}
|
|
token := head + "|" + since.UTC().Format(time.RFC3339Nano)
|
|
cur, tracked := w.pulses[pk]
|
|
if tracked && !cur.seeded && cur.token == token {
|
|
return
|
|
}
|
|
ttl, stalled := w.ttl(w.now().Sub(since))
|
|
if bound > 0 {
|
|
ttl, stalled = bound-w.now().Sub(since), w.now().Sub(since) >= bound
|
|
}
|
|
if stalled {
|
|
if tracked {
|
|
w.deletePulse(pk)
|
|
}
|
|
w.writeStuck(bucket, key, waited, since)
|
|
return
|
|
}
|
|
value, err := delivery.EncodePulse(delivery.Pulse{Bucket: bucket, Key: key, Revision: revision, Owes: waited})
|
|
if err != nil {
|
|
w.say("no pulse for %s %s: %v", bucket, key, err)
|
|
return
|
|
}
|
|
if err := w.out.PutPulse(pk, value, ttl); err != nil {
|
|
w.say("cannot write the pulse %s: %v", pk, err)
|
|
return
|
|
}
|
|
w.pulses[pk] = &pulse{bucket: bucket, key: key, waited: waited, since: since, head: head, token: token}
|
|
}
|
|
|
|
func (w *Watchman) deletePulse(pk string) {
|
|
delete(w.pulses, pk)
|
|
if err := w.out.DeletePulse(pk); err != nil {
|
|
w.say("cannot delete the pulse %s: %v", pk, err)
|
|
}
|
|
}
|
|
|
|
// moved is the mirrored key moving: its pulse is deleted, so it never expires, and a stuck key it left is written as
|
|
// moved (never deleted).
|
|
func (w *Watchman) moved(bucket, key string) {
|
|
pk, err := delivery.PulseKey(bucket, key)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if _, ok := w.pulses[pk]; ok {
|
|
w.deletePulse(pk)
|
|
}
|
|
w.stuckMoved(bucket, key)
|
|
}
|
|
|
|
// stuckMoved writes a stuck key of the key as moved, if it stands and is not yet.
|
|
func (w *Watchman) stuckMoved(bucket, key string) {
|
|
sk, err := delivery.StuckKey(bucket, key)
|
|
if err != nil {
|
|
return
|
|
}
|
|
s, ok := w.stuck[sk]
|
|
if !ok || s.Moved {
|
|
return
|
|
}
|
|
s.Moved = true
|
|
w.putStuck(sk, s)
|
|
}
|
|
|
|
func (w *Watchman) writeStuck(bucket, key, waited string, since time.Time) {
|
|
sk, err := delivery.StuckKey(bucket, key)
|
|
if err != nil {
|
|
w.say("no stuck key for %s %s: %v", bucket, key, err)
|
|
return
|
|
}
|
|
if cur, ok := w.stuck[sk]; ok && cur.Since.Equal(since) {
|
|
return
|
|
}
|
|
w.putStuck(sk, delivery.Stuck{Bucket: bucket, Key: key, Waited: waited, Since: since})
|
|
}
|
|
|
|
func (w *Watchman) putStuck(sk string, s delivery.Stuck) {
|
|
value, err := delivery.EncodeStuck(s)
|
|
if err != nil {
|
|
w.say("cannot encode the stuck key %s: %v", sk, err)
|
|
return
|
|
}
|
|
if err := w.out.PutStuck(sk, value); err != nil {
|
|
w.say("cannot write the stuck key %s: %v", sk, err)
|
|
return
|
|
}
|
|
s.V = delivery.Version
|
|
w.stuck[sk] = s
|
|
w.stuckFinding(sk, s)
|
|
}
|
|
|
|
// Pulse is a change heard on the pulse bucket. Only the server's expiry marker of a pulse the watchman made writes a
|
|
// stuck key: the watchman's own delete, and a key it never made, write nothing.
|
|
func (w *Watchman) Pulse(key string, op Op, at time.Time) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if op != Expired {
|
|
return
|
|
}
|
|
if _, ok := w.signals[key]; ok {
|
|
w.signalExpired(key)
|
|
return
|
|
}
|
|
p, ok := w.pulses[key]
|
|
if !ok || p.seeded {
|
|
return
|
|
}
|
|
delete(w.pulses, key)
|
|
w.writeStuck(p.bucket, p.key, p.waited, p.since)
|
|
}
|
|
|
|
// SeedStuck tells the watchman a stuck key that stands in its bucket.
|
|
func (w *Watchman) SeedStuck(key string, value []byte) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if s, err := delivery.DecodeStuck(value); err == nil {
|
|
w.stuck[key] = s
|
|
w.stuckFinding(key, s)
|
|
}
|
|
}
|
|
|
|
// SeedPulse tells the watchman a pulse key that stands in its bucket at its start; Reconcile deletes it unless a
|
|
// mirrored key made it again since.
|
|
func (w *Watchman) SeedPulse(key string) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
bucket, k, err := delivery.ParsePulseKey(key)
|
|
if err != nil || bucket == signalBucket {
|
|
return
|
|
}
|
|
w.pulses[key] = &pulse{bucket: bucket, key: k, seeded: true}
|
|
}
|
|
|
|
// Reconcile ends the start: a pulse that stood and that no mirrored key has made again is a move that happened while
|
|
// the watchman was away, so it is deleted and writes no stuck key; and a stuck key whose watched key is gone is closed
|
|
// (closeGone).
|
|
func (w *Watchman) Reconcile() {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
for pk, p := range w.pulses {
|
|
if p.seeded {
|
|
w.deletePulse(pk)
|
|
}
|
|
}
|
|
w.closeGone()
|
|
w.reconcileFindings()
|
|
}
|
|
|
|
// **A stuck key whose watched key is gone is closed** (issue 774, sweep 2026-10-30): a key that ages out of its bucket
|
|
// (a stream's maximum age, 168 h for the build seats' asks) leaves no marker a watch hears, and one deleted while the
|
|
// watchman was down is not heard either, so its stuck key stood unmoved for ever: 26 stuck-key conditions of asks from
|
|
// 2026-10-21 were open on 10-30. At the start's end, every stuck key not moved whose bucket was read whole at this start
|
|
// (ReadWhole) and whose key was not read standing is written moved, which clears its finding, and said. A bucket that
|
|
// could not be read closes nothing: its keys are not known to be gone.
|
|
|
|
// ReadWhole says every key of bucket (as a stuck key names it: an ask stream for a build seat's steps, the assigner's
|
|
// wants) was read at this start; called once its watch holds what stands.
|
|
func (w *Watchman) ReadWhole(bucket string) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
w.readWhole[bucket] = true
|
|
}
|
|
|
|
// stands records a mirrored key read standing during the start.
|
|
func (w *Watchman) stands(bucket, key string) {
|
|
if w.reconciled {
|
|
return
|
|
}
|
|
if w.standing[bucket] == nil {
|
|
w.standing[bucket] = map[string]bool{}
|
|
}
|
|
w.standing[bucket][key] = true
|
|
}
|
|
|
|
func (w *Watchman) closeGone() {
|
|
for sk, s := range w.stuck {
|
|
if s.Moved || !w.readWhole[s.Bucket] || w.standing[s.Bucket][s.Key] {
|
|
continue
|
|
}
|
|
s.Moved = true
|
|
w.say("the stuck key %s is closed: its key %s is no longer in %s (it aged out of the bucket or was deleted while nobody heard), so nothing waits for it since %s",
|
|
sk, s.Key, s.Bucket, s.Since.UTC().Format(time.RFC3339))
|
|
w.putStuck(sk, s)
|
|
}
|
|
w.standing, w.readWhole, w.reconciled = map[string]map[string]bool{}, map[string]bool{}, true
|
|
}
|
|
|
|
// reconcileFindings ends the start for the findings: one read at the start that what the watchman read since did not
|
|
// observe again is deleted, unless a signal still standing keeps it (a provider's failing word, an advisory's quiet hour, a
|
|
// machine's silent tools, a silent self-check): those end by their own events or expiry.
|
|
func (w *Watchman) reconcileFindings() {
|
|
kept := map[string]bool{}
|
|
for _, due := range w.signals {
|
|
kept[due.Finding.Key()] = true
|
|
}
|
|
for key := range w.seededFindings {
|
|
delete(w.seededFindings, key)
|
|
if kept[key] || signalKind(w.findings[key]) {
|
|
continue
|
|
}
|
|
w.unfind(key)
|
|
}
|
|
}
|
|
|
|
// signalKind says a finding is one a signal row writes and ends by its own event (S8, S9, S10, S11): it outlives a
|
|
// restart, since what ends it is heard, not read.
|
|
func signalKind(value []byte) bool {
|
|
f, err := DecodeFinding(value)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
switch f.Source {
|
|
case "provisioner.failing", "S8", "S9", "S10", "S11":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func describe(format string, args ...any) string { return fmt.Sprintf(format, args...) }
|
|
|
|
// StepBound is how long a build step may run, from its start on the node, before it is stuck: the asker's bound on a step
|
|
// (mesh-sdk go/delivery/buildrun Asks.Bound, 30 minutes for a build). Nothing rewrites a running step's key, so its pulse
|
|
// is bounded by this and not by the 120 s of a move that is heartbeat (issue 1908).
|
|
const StepBound = 30 * time.Minute
|
|
|
|
// SetStepBound sets how long a build step may run; it is StepBound unless a test shortens it.
|
|
func (w *Watchman) SetStepBound(d time.Duration) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
w.stepBound = d
|
|
}
|
|
|
|
// SetBound sets how long a move, and a running step, may take; it is delivery.PulseMoveBound unless a test shortens it.
|
|
func (w *Watchman) SetBound(d time.Duration) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
w.bound, w.stepBound = d, d
|
|
}
|
|
|
|
// ttl is the time to live of a pulse for a move of the given age, and whether the move is stalled already
|
|
// (delivery.PulseTTL, with the bound this watchman keeps).
|
|
func (w *Watchman) ttl(age time.Duration) (time.Duration, bool) {
|
|
if w.bound == delivery.PulseMoveBound {
|
|
return delivery.PulseTTL(age)
|
|
}
|
|
if age >= w.bound {
|
|
return 0, true
|
|
}
|
|
return w.bound - age, false
|
|
}
|