Files
mesh-delivery/modules/mesh-delivery-watchman/internal/watch/mirror.go
T
jschoubben 3eca094c5c A-197: the watchman reads no node-engine state: have and health go, running ends a want's pulse
Removed with the reads of node-engine.have and mesh-host.health: the findings module-unhealthy (module <module>.<node>, token
unhealthy, urgent after four hours), kept-as-it-runs (module <module>.<node>, token kept), apply-failing from a have that failed
(machine <node>; the provisioner clients' half stays), machine-network, network-rewritten, network-unreachable, machine-units,
relogin-needed, directory-used-as-found, node-account-fallback, and the urgency of a silent control node. A want's pulse now ends on
the node-engine's running event (consumes node-engine.running).

Builds: design 47 G A-197, A-192, §541

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 05:58:55 +01:00

210 lines
7.3 KiB
Go

package watch
import (
"encoding/json"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)
// A build ask taken by a node is a move (novox/hq ADR 0525 §4, issues 1887 A8b, 1908, 1909; design 47 G §483, §496, §602,
// A-136): the holder of the build seat says `ask-taken` when it takes any ask, `step-progress` once a minute while it runs
// it, and `step-ended` when it ends it. The pulse of the move is the pulse of the bucket named as the seat's stream
// (buildstep.AskStream), under the ask's id, and it begins at the latest of the ask's taking and its progress, so a step
// that runs long but says its progress raises nothing, and one whose holder stops is stuck within the progress bound.
//
// **An ask still waiting in the stream is a backlog, not a stall** (issue 1908): no pulse is made for a waiting ask, so none
// can expire into a false stuck row; the view shows the backlog and the oldest waiting ask's age (delivery view, A8a).
// ProgressBound is how long a running ask may say nothing of its progress (design 47 G §75: five minutes).
const ProgressBound = 5 * time.Minute
// languageOfSeat is the build language whose seat is seat, and whether it is a build seat.
func languageOfSeat(seat string) (string, bool) {
for _, lang := range buildstep.AskLanguages {
if s, err := buildstep.BuildSeat(lang); err == nil && s == seat {
return lang, true
}
}
return "", false
}
// AskTaken hears a build seat's `ask-taken` (node: the subject's last token): the ask is taken, so its clock begins at the
// event's time.
func (w *Watchman) AskTaken(seat, node string, body []byte) {
var e nodebuildgo.AskTakenEvent
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
w.say("an ask-taken of %s does not read, left: %v", seat, err)
return
}
w.askClock(seat, node, e.Ask, e.At)
}
// StepProgress hears a build seat's `step-progress`: the running ask says it lives, so its clock begins again at the
// event's time.
func (w *Watchman) StepProgress(seat, node string, body []byte) {
var e nodebuildgo.StepProgressEvent
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
w.say("a step-progress of %s does not read, left: %v", seat, err)
return
}
w.askClock(seat, node, e.Ask, e.At)
}
// AskEnded hears a build seat's `step-ended`, whatever its asker: the ask's clock ends.
func (w *Watchman) AskEnded(seat string, body []byte) {
lang, ok := languageOfSeat(seat)
if !ok {
return
}
bucket, err := buildstep.AskStream(lang)
if err != nil {
return
}
var e nodebuildgo.StepEndedEvent
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
w.say("a step-ended of %s does not read, left: %v", seat, err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
w.moved(bucket, e.Ask)
}
func (w *Watchman) askClock(seat, node, ask string, at time.Time) {
lang, ok := languageOfSeat(seat)
if !ok {
return
}
bucket, err := buildstep.AskStream(lang)
if err != nil {
w.say("no pulse for the ask %s: %v", ask, err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
if at.IsZero() {
at = w.now()
}
w.mirrorWithin(w.progressBound, bucket, ask, describe("the ask %s to end on %s", ask, node), node, at, 0)
}
// A want is a move too (novox/hq ADR 0539 §19, issue 1906): the assigner writes `want.<node>.<module>` in its wants bucket,
// and the node's agent owes the node-engine's have naming the want's fingerprint. The pulse is the pulse of that bucket under
// the want's key, owed by the node's agent, and it begins when the assigner wrote the want. The have naming the fingerprint
// ends it; a want that is removed, or a key deleted, ends it as well. Nothing renews it: the bound is how long a node may
// take to apply a want, the same as a build step's (StepBound). Keys of that bucket that are not a want of a module (a hold,
// and the node's own key) are left.
// Want is a change of a key of the assigner's wants bucket, read from the watch (at the start the keys that stand, then each
// change), at the time the entry was written.
func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
node, module, err := meshwantsclient.ParseWantKey(key)
if err != nil {
return
}
w.mu.Lock()
defer w.mu.Unlock()
switch op {
case Expired:
return
case Delete:
delete(w.wants, key)
delete(w.held, key)
w.moved(AssignerWantsBucket, key)
return
}
w.stands(AssignerWantsBucket, key)
want, err := meshwantsclient.DecodeWant(value)
if err != nil {
w.say("the want %s cannot be read, and is left: %v", key, err)
return
}
if want.Removed {
delete(w.wants, key)
delete(w.held, key)
w.moved(AssignerWantsBucket, key)
return
}
w.wants[key] = want.Fp
if want.Newest != "" || want.Rule != 0 {
// A want held by a rule (ADR 0603 §8) waits on the rule, not on the node: its wait is the hold's, from the time the
// assigner wrote it, and the node running what the want names does not end it.
if at.IsZero() {
at = w.now()
}
w.held[key] = true
w.mirrorKept(AssignerWantsBucket, key,
describe("the hold on %s on %s at %s (rule %d) to end, bound %s: %s", module, node, shortFP(want.Fp), want.Rule, w.stepBound, want.Words),
"held|"+want.Fp+"|"+want.Newest, at)
return
}
if w.held[key] {
delete(w.held, key)
if pk, err := meshwatchdogsclient.PulseKey(AssignerWantsBucket, key); err == nil {
delete(w.starts, pk)
}
w.stuckMoved(AssignerWantsBucket, key) // the hold ended: its finding is cleared
}
if w.have(node, module) == want.Fp {
w.moved(AssignerWantsBucket, key)
return
}
if at.IsZero() {
at = w.now()
}
w.mirrorWithin(w.stepBound, AssignerWantsBucket, key,
describe("the node agent of %s to run %s at %s", node, module, shortFP(want.Fp)), want.Fp, at, 0)
}
// EngineRunning hears the node-engine of node saying `running` for module (design 47 G §541, A-192, A-197): the fingerprint it runs, or
// that it runs none (stopped). One that names the fingerprint of the node's want for the module ends the want's pulse.
func (w *Watchman) EngineRunning(node, module string, body []byte) {
var r nodeengine.Running
if err := json.Unmarshal(body, &r); err != nil {
w.say("a running of the node-engine of %s for %s does not read, left: %v", node, module, err)
return
}
key, err := nodeengineclient.HaveKey(node, module)
if err != nil {
return
}
w.mu.Lock()
defer w.mu.Unlock()
if r.Stopped || r.Fingerprint == "" {
delete(w.haves, key)
return
}
w.haves[key] = r.Fingerprint
wk, err := meshwantsclient.WantKey(node, module)
if err != nil {
return
}
if fp, ok := w.wants[wk]; ok && fp == r.Fingerprint {
w.moved(AssignerWantsBucket, wk)
}
}
func (w *Watchman) have(node, module string) string {
key, err := nodeengineclient.HaveKey(node, module)
if err != nil {
return ""
}
return w.haves[key]
}
func shortFP(fp string) string {
hex := strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
if len(hex) > 12 {
hex = hex[:12]
}
return hex
}