Removed with the reads of node-engine.have and mesh-host.health: the findings module-unhealthy (module <module>.<node>, token unhealthy, urgent after four hours), kept-as-it-runs (module <module>.<node>, token kept), apply-failing from a have that failed (machine <node>; the provisioner clients' half stays), machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed, directory-used-as-found, node-account-fallback, and the urgency of a silent control node. A want's pulse now ends on the node-engine's running event (consumes node-engine.running). Builds: design 47 G A-197, A-192, §541 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
210 lines
7.3 KiB
Go
210 lines
7.3 KiB
Go
package watch
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
|
)
|
|
|
|
// A build ask taken by a node is a move (novox/hq ADR 0525 §4, issues 1887 A8b, 1908, 1909; design 47 G §483, §496, §602,
|
|
// A-136): the holder of the build seat says `ask-taken` when it takes any ask, `step-progress` once a minute while it runs
|
|
// it, and `step-ended` when it ends it. The pulse of the move is the pulse of the bucket named as the seat's stream
|
|
// (buildstep.AskStream), under the ask's id, and it begins at the latest of the ask's taking and its progress, so a step
|
|
// that runs long but says its progress raises nothing, and one whose holder stops is stuck within the progress bound.
|
|
//
|
|
// **An ask still waiting in the stream is a backlog, not a stall** (issue 1908): no pulse is made for a waiting ask, so none
|
|
// can expire into a false stuck row; the view shows the backlog and the oldest waiting ask's age (delivery view, A8a).
|
|
|
|
// ProgressBound is how long a running ask may say nothing of its progress (design 47 G §75: five minutes).
|
|
const ProgressBound = 5 * time.Minute
|
|
|
|
// languageOfSeat is the build language whose seat is seat, and whether it is a build seat.
|
|
func languageOfSeat(seat string) (string, bool) {
|
|
for _, lang := range buildstep.AskLanguages {
|
|
if s, err := buildstep.BuildSeat(lang); err == nil && s == seat {
|
|
return lang, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
// AskTaken hears a build seat's `ask-taken` (node: the subject's last token): the ask is taken, so its clock begins at the
|
|
// event's time.
|
|
func (w *Watchman) AskTaken(seat, node string, body []byte) {
|
|
var e nodebuildgo.AskTakenEvent
|
|
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
|
|
w.say("an ask-taken of %s does not read, left: %v", seat, err)
|
|
return
|
|
}
|
|
w.askClock(seat, node, e.Ask, e.At)
|
|
}
|
|
|
|
// StepProgress hears a build seat's `step-progress`: the running ask says it lives, so its clock begins again at the
|
|
// event's time.
|
|
func (w *Watchman) StepProgress(seat, node string, body []byte) {
|
|
var e nodebuildgo.StepProgressEvent
|
|
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
|
|
w.say("a step-progress of %s does not read, left: %v", seat, err)
|
|
return
|
|
}
|
|
w.askClock(seat, node, e.Ask, e.At)
|
|
}
|
|
|
|
// AskEnded hears a build seat's `step-ended`, whatever its asker: the ask's clock ends.
|
|
func (w *Watchman) AskEnded(seat string, body []byte) {
|
|
lang, ok := languageOfSeat(seat)
|
|
if !ok {
|
|
return
|
|
}
|
|
bucket, err := buildstep.AskStream(lang)
|
|
if err != nil {
|
|
return
|
|
}
|
|
var e nodebuildgo.StepEndedEvent
|
|
if err := json.Unmarshal(body, &e); err != nil || e.Ask == "" {
|
|
w.say("a step-ended of %s does not read, left: %v", seat, err)
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
w.moved(bucket, e.Ask)
|
|
}
|
|
|
|
func (w *Watchman) askClock(seat, node, ask string, at time.Time) {
|
|
lang, ok := languageOfSeat(seat)
|
|
if !ok {
|
|
return
|
|
}
|
|
bucket, err := buildstep.AskStream(lang)
|
|
if err != nil {
|
|
w.say("no pulse for the ask %s: %v", ask, err)
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if at.IsZero() {
|
|
at = w.now()
|
|
}
|
|
w.mirrorWithin(w.progressBound, bucket, ask, describe("the ask %s to end on %s", ask, node), node, at, 0)
|
|
}
|
|
|
|
// A want is a move too (novox/hq ADR 0539 §19, issue 1906): the assigner writes `want.<node>.<module>` in its wants bucket,
|
|
// and the node's agent owes the node-engine's have naming the want's fingerprint. The pulse is the pulse of that bucket under
|
|
// the want's key, owed by the node's agent, and it begins when the assigner wrote the want. The have naming the fingerprint
|
|
// ends it; a want that is removed, or a key deleted, ends it as well. Nothing renews it: the bound is how long a node may
|
|
// take to apply a want, the same as a build step's (StepBound). Keys of that bucket that are not a want of a module (a hold,
|
|
// and the node's own key) are left.
|
|
|
|
// Want is a change of a key of the assigner's wants bucket, read from the watch (at the start the keys that stand, then each
|
|
// change), at the time the entry was written.
|
|
func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
|
|
node, module, err := meshwantsclient.ParseWantKey(key)
|
|
if err != nil {
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
switch op {
|
|
case Expired:
|
|
return
|
|
case Delete:
|
|
delete(w.wants, key)
|
|
delete(w.held, key)
|
|
w.moved(AssignerWantsBucket, key)
|
|
return
|
|
}
|
|
w.stands(AssignerWantsBucket, key)
|
|
want, err := meshwantsclient.DecodeWant(value)
|
|
if err != nil {
|
|
w.say("the want %s cannot be read, and is left: %v", key, err)
|
|
return
|
|
}
|
|
if want.Removed {
|
|
delete(w.wants, key)
|
|
delete(w.held, key)
|
|
w.moved(AssignerWantsBucket, key)
|
|
return
|
|
}
|
|
w.wants[key] = want.Fp
|
|
if want.Newest != "" || want.Rule != 0 {
|
|
// A want held by a rule (ADR 0603 §8) waits on the rule, not on the node: its wait is the hold's, from the time the
|
|
// assigner wrote it, and the node running what the want names does not end it.
|
|
if at.IsZero() {
|
|
at = w.now()
|
|
}
|
|
w.held[key] = true
|
|
w.mirrorKept(AssignerWantsBucket, key,
|
|
describe("the hold on %s on %s at %s (rule %d) to end, bound %s: %s", module, node, shortFP(want.Fp), want.Rule, w.stepBound, want.Words),
|
|
"held|"+want.Fp+"|"+want.Newest, at)
|
|
return
|
|
}
|
|
if w.held[key] {
|
|
delete(w.held, key)
|
|
if pk, err := meshwatchdogsclient.PulseKey(AssignerWantsBucket, key); err == nil {
|
|
delete(w.starts, pk)
|
|
}
|
|
w.stuckMoved(AssignerWantsBucket, key) // the hold ended: its finding is cleared
|
|
}
|
|
if w.have(node, module) == want.Fp {
|
|
w.moved(AssignerWantsBucket, key)
|
|
return
|
|
}
|
|
if at.IsZero() {
|
|
at = w.now()
|
|
}
|
|
w.mirrorWithin(w.stepBound, AssignerWantsBucket, key,
|
|
describe("the node agent of %s to run %s at %s", node, module, shortFP(want.Fp)), want.Fp, at, 0)
|
|
}
|
|
|
|
// EngineRunning hears the node-engine of node saying `running` for module (design 47 G §541, A-192, A-197): the fingerprint it runs, or
|
|
// that it runs none (stopped). One that names the fingerprint of the node's want for the module ends the want's pulse.
|
|
func (w *Watchman) EngineRunning(node, module string, body []byte) {
|
|
var r nodeengine.Running
|
|
if err := json.Unmarshal(body, &r); err != nil {
|
|
w.say("a running of the node-engine of %s for %s does not read, left: %v", node, module, err)
|
|
return
|
|
}
|
|
key, err := nodeengineclient.HaveKey(node, module)
|
|
if err != nil {
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
if r.Stopped || r.Fingerprint == "" {
|
|
delete(w.haves, key)
|
|
return
|
|
}
|
|
w.haves[key] = r.Fingerprint
|
|
wk, err := meshwantsclient.WantKey(node, module)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if fp, ok := w.wants[wk]; ok && fp == r.Fingerprint {
|
|
w.moved(AssignerWantsBucket, wk)
|
|
}
|
|
}
|
|
|
|
func (w *Watchman) have(node, module string) string {
|
|
key, err := nodeengineclient.HaveKey(node, module)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return w.haves[key]
|
|
}
|
|
|
|
func shortFP(fp string) string {
|
|
hex := strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2)
|
|
if len(hex) > 12 {
|
|
hex = hex[:12]
|
|
}
|
|
return hex
|
|
}
|