Merge pull request 'The controller takes what the watchman says: key-ttl buckets, the checks running readable, stuck keys raised, silence from presence (design 47 R2b)' (#388) from feat/delivery-watch into main

cmd/mesh-controller/apply_failing.go

@@ -33,7 +33,7 @@ const packageManagerKey = "the package manager ("
 func watchApplyFailing(f *signalFacts) []conditions.Observation {
 	heard := map[string]bool{}
 	for _, m := range f.machines {
-		if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
+		if !m.asleep() && !m.lastHeard.IsZero() && !m.silent(f.now) {
 			heard[m.name] = true
 		}
 	}

cmd/mesh-controller/doctor.go

@@ -120,6 +120,11 @@ var probeRegistry = []probe{
 	{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
 		"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
 		Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
+	// What the watchman says stopped moving (ADR 0433 §2, issue 1156): each key of its stuck bucket not moved, but a
+	// node's presence, which the silent check takes.
+	{ID: probeStuckID, Asserts: "no key the watchman says stopped moving is left unsaid: each key of its stuck bucket " +
+		"not moved is a condition of the delivery scope until it says moved", From: "ADR 0433 §2, issue 1156",
+		Kind: kindStuck, Phase: 3, run: probeStuck},
 	// A first merge into a repository the mesh builds nothing from, kept waiting for mesh-delivery's word on the
 	// operator's Approve (novox/hq ADR 0432, issue 1146): said past a bound, never only in a log line (ADR 0426 §3, §8).
 	{ID: probeFirstBuildsID, Asserts: "no first merge into a repository the mesh builds nothing from waits more than ten " +
@@ -183,6 +188,10 @@ var probeRegistry = []probe{
 	{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
 		"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
 		run: probeBackupHeld},
+	// A plan's send key the bus refused (ADR 0433 §4, issue 1157): in place of the timer that stood in for it.
+	{ID: sendKeyRefusedID, Asserts: "no delivery plan's key that watches a machine for its report was refused by the " +
+		"bus at the newest look", From: "ADR 0433 §4, issue 1157", Kind: kindSendKeyRefused, Phase: 1,
+		run: probeSendKeyRefused},
 }
 
 // probeVerdict is one probe's outcome in a run.

cmd/mesh-controller/plain_words.go

@@ -610,6 +610,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
 				"did not answer in ten minutes, so nothing of the delivery is built or sent.",
 			Resolved: "Resolved: the backup is done and the delivery goes on"}
 	}),
+	kindStuck: worded(func(o conditions.Observation) words {
+		return words{Headline: "Something in a delivery has stopped moving",
+			Explanation: "The watchman noticed a part of the delivery that did not move when it should have: a build " +
+				"asked and not answered, a check that stopped, or a part that went offline. Nothing is lost, and it is " +
+				"said again until it moves.",
+			Resolved: "Resolved: it moves again"}
+	}),
+	kindSendKeyRefused: worded(func(o conditions.Observation) words {
+		return words{Headline: "A delivery cannot watch a machine for its report",
+			Explanation: "The bus refused the note that makes the mesh notice a machine that never reports on a send. " +
+				"The delivery goes on, and the note is written again at its next look.",
+			Resolved: "Resolved: the delivery watches the machine for its report again"}
+	}),
 	kindConsumerBehind: worded(func(o conditions.Observation) words {
 		return words{Headline: "Messages pile up for a listener",
 			Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +

cmd/mesh-controller/plan_looks.go

@@ -400,34 +400,39 @@ func syncSilentKeys(ctx context.Context, open *stores) error {
 }
 
 // keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys
-// cannot be read, one cannot be put — the wait is looked at by a timer of its own, once, and the error is given back so
-// the look is retried.
+// cannot be read, one cannot be put — the refusal is raised as a condition naming the plan and the machine, never
+// stood in for by a timer, and the error is given back so the look is retried and writes the key again (novox/hq ADR
+// 0433 §4). The set of refusals is what this look found: a key written at the next look is no longer in it.
 func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error {
-	byTimer := func() {
-		for w, sent := range want {
-			silentByTimer(w, sent, silentTTL(sent, now))
+	refused := func(err error) map[silentWait]error {
+		out := map[silentWait]error{}
+		for w := range want {
+			out[w] = err
 		}
+		return out
 	}
 	if c == nil {
-		byTimer()
+		noteRefusedSendKeys(refused(errors.New("the bus's clock for the waits is not there")))
 		return nil
 	}
 	c = boundedClock{c}
 	held, err := c.Held(ctx)
 	if err != nil {
-		byTimer()
+		noteRefusedSendKeys(refused(err))
 		return err
 	}
 	arm, drop := silentDiff(want, held)
 	var first error
+	notTaken := map[silentWait]error{}
 	for _, w := range arm {
 		if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
-			silentByTimer(w, want[w], silentTTL(want[w], now))
+			notTaken[w] = err
 			if first == nil {
 				first = err
 			}
 		}
 	}
+	noteRefusedSendKeys(notTaken)
 	for _, w := range drop {
 		if err := c.Reported(ctx, w); err != nil && first == nil {
 			first = err
@@ -481,31 +486,6 @@ func (b boundedClock) Reported(ctx context.Context, w silentWait) error {
 	return err
 }
 
-var silentTimers = struct {
-	mu     sync.Mutex
-	timers map[silentWait]time.Time
-}{timers: map[silentWait]time.Time{}}
-
-// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key
-// (DegradableGrants says why that can be): never silently never. Once for each send.
-func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) {
-	silentTimers.mu.Lock()
-	defer silentTimers.mu.Unlock()
-	if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) {
-		return
-	}
-	silentTimers.timers[w] = sent
-	// retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once.
-	time.AfterFunc(ttl, func() {
-		silentTimers.mu.Lock()
-		if silentTimers.timers[w].Equal(sent) {
-			delete(silentTimers.timers, w)
-		}
-		silentTimers.mu.Unlock()
-		askToLook()
-	})
-}
-
 // startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
 // controller is the lease holder here, the only one that writes it. The watcher asks for a look when a key expires.
 func startRolloutSilent(ctx context.Context, js *broker.JetStream) error {

cmd/mesh-controller/plan_looks_test.go

@@ -3,10 +3,12 @@ package main
 import (
 	"context"
 	"errors"
+	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -159,22 +161,76 @@ func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
 	return nil, errors.New("the bus is down")
 }
 
-// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own
-// and the error is given back, so the look is marked failed and retried (issue 974, review 1).
-func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
+// 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by
+// nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error
+// is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4).
+func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1
 	w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
 	sent := time.Now().UTC()
-	t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
-	err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent)
-	if err == nil {
-		t.Fatal("no error given back")
+	for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{},
+		"no clock": nil} {
+		t.Run(name, func(t *testing.T) {
+			noteRefusedSendKeys(nil)
+			t.Cleanup(func() { noteRefusedSendKeys(nil) })
+			err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent)
+			if clock != nil && err == nil {
+				t.Fatal("no error given back")
+			}
+			got := observeRefusedSendKeys()
+			if len(got) != 1 {
+				t.Fatalf("observations %+v", got)
+			}
+			o := got[0]
+			if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused ||
+				!strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") {
+				t.Fatalf("the condition does not name the plan and the machine: %+v", o)
+			}
+		})
 	}
-	silentTimers.mu.Lock()
-	armed := silentTimers.timers[w].Equal(sent)
-	silentTimers.mu.Unlock()
-	if !armed {
-		t.Error("no timer of its own watches the wait")
+}
+
+// 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is.
+func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2
+	w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
+	sent := time.Now().UTC()
+	noteRefusedSendKeys(nil)
+	t.Cleanup(func() { noteRefusedSendKeys(nil) })
+	want := map[silentWait]time.Time{w: sent}
+	if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 {
+		t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys())
 	}
+	clock := &keptClock{}
+	if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil {
+		t.Fatal(err)
+	}
+	if !clock.armed[w].Equal(sent) {
+		t.Fatalf("the next look did not write the key again: %v", clock.armed)
+	}
+	if got := observeRefusedSendKeys(); len(got) != 0 {
+		t.Fatalf("the condition stays after the key was written: %+v", got)
+	}
+}
+
+type armRefusingClock struct{ silentClock }
+
+func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
+func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error {
+	return errors.New("the bus refuses the key")
+}
+
+// keptClock is a clock that takes every key.
+type keptClock struct {
+	silentClock
+	armed map[silentWait]time.Time
+}
+
+func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
+func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error {
+	if c.armed == nil {
+		c.armed = map[silentWait]time.Time{}
+	}
+	c.armed[w] = sent
+	return nil
 }
 
 // A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
@@ -189,14 +245,13 @@ func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) {
 }
 
 // issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans
-// and every node for good. Every call to the clock has a bound, and the wait is looked at by a timer of its own.
+// and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition.
 func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) {
 	was := silentCallBound
 	silentCallBound = 50 * time.Millisecond
 	t.Cleanup(func() { silentCallBound = was })
 	w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"}
 	sent := time.Now().UTC()
-	t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
 	done := make(chan error, 1)
 	go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }()
 	select {

cmd/mesh-controller/probes.go

@@ -629,7 +629,7 @@ func heardMachines(d *doctor) map[string]bool {
 		return out
 	}
 	for _, m := range f.machines {
-		if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
+		if !m.lastHeard.IsZero() && !m.silent(f.now) && !m.asleep() {
 			out[m.name] = true
 		}
 	}

cmd/mesh-controller/rollout_silent_bus_integration_test.go

@@ -42,7 +42,7 @@ func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) {
 	return s, heard
 }
 
-func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) {
+func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { // 1157/AC3: the send keys stay until release 6 (ADR 0433 §5)
 	s, heard := silentOnABus(t)
 	want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
 	if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {

cmd/mesh-controller/send_key_refused.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"sync"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+)
+
+// A plan's send key the bus refuses (novox/hq ADR 0433 §4, issue 1157): the look that could not write it says which, and
+// a probe of the controller's self-check raises each as a condition naming the plan and the machine. The next look writes
+// the key again; the look that writes it leaves it out of the set, and the condition clears.
+const (
+	kindSendKeyRefused = "send-key-refused"
+	sendKeyRefusedID   = "DL"
+)
+
+var refusedSendKeys = struct {
+	mu sync.Mutex
+	by map[silentWait]error
+}{}
+
+// noteRefusedSendKeys sets the keys the newest look could not write, replacing what the look before found.
+func noteRefusedSendKeys(refused map[silentWait]error) {
+	refusedSendKeys.mu.Lock()
+	defer refusedSendKeys.mu.Unlock()
+	refusedSendKeys.by = refused
+}
+
+// observeRefusedSendKeys is one condition per plan and machine whose send key was refused.
+func observeRefusedSendKeys() []conditions.Observation {
+	refusedSendKeys.mu.Lock()
+	waits := make([]silentWait, 0, len(refusedSendKeys.by))
+	why := map[silentWait]error{}
+	for w, err := range refusedSendKeys.by {
+		waits = append(waits, w)
+		why[w] = err
+	}
+	refusedSendKeys.mu.Unlock()
+	sort.Slice(waits, func(i, j int) bool { return silentKey(waits[i]) < silentKey(waits[j]) })
+	seen := map[[2]string]bool{}
+	var out []conditions.Observation
+	for _, w := range waits {
+		if seen[[2]string{w.Plan, w.Node}] {
+			continue // a plan sends a machine one tier at a time; one condition says it
+		}
+		seen[[2]string{w.Plan, w.Node}] = true
+		out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.Plan, Token: kindSendKeyRefused + "-" + w.Node,
+			Kind: kindSendKeyRefused, Machine: w.Node, Severity: conditions.Warning,
+			Summary: fmt.Sprintf("the bus refused the key that watches the delivery %s for a report from %s, so nothing "+
+				"names the machine silent if it never reports; the plan writes the key again at its next look", w.Plan, w.Node),
+			Said: fmt.Sprintf("%s: %v", silentKey(w), why[w])})
+	}
+	return out
+}
+
+// probeSendKeyRefused reads what the newest look of the plans could not write.
+func probeSendKeyRefused(context.Context, *doctor) ([]conditions.Observation, error) {
+	return observeRefusedSendKeys(), nil
+}

cmd/mesh-controller/signals.go

@@ -335,7 +335,7 @@ func watchHeartbeats(f *signalFacts) []conditions.Observation {
 		}
 		bound := heartbeatBound(m.every)
 		silent := f.now.Sub(m.lastHeard)
-		if silent <= bound {
+		if !m.silent(f.now) {
 			continue
 		}
 		severity := conditions.Warning
@@ -357,7 +357,7 @@ func watchReports(f *signalFacts) []conditions.Observation {
 		if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
 			continue
 		}
-		if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
+		if m.silent(f.now) {
 			continue // silent: S1 says it, and a silent machine reports nothing
 		}
 		bound := max(reportAtLeast, 3*m.lastApply)

cmd/mesh-controller/stuck_probe.go

@@ -0,0 +1,228 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"time"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+)
+
+// What stopped moving (novox/hq ADR 0433 §2, §3, issue 1156). The watchman writes `stuck.<bucket>.<key>` in its own
+// bucket for each key whose pulse expired, and again as moved when the key moves; it never deletes one. The controller
+// reads that bucket in two places and holds no timer for either: a probe of the self-check raises each key not moved as
+// a condition of the delivery scope, and the check that a machine is silent takes a node's presence key as its verdict.
+const (
+	kindStuck    = "stuck"
+	probeStuckID = "DM"
+)
+
+// presenceVerdict is the watchman's word on whether a machine is online, from its stuck presence key.
+type presenceVerdict int
+
+const (
+	// presenceUnsaid: the watchman has written nothing for the machine; the heartbeat rule judges it, as before.
+	presenceUnsaid presenceVerdict = iota
+	// presenceHeard: the key says moved, the machine is back online.
+	presenceHeard
+	// presenceSilent: the key says not moved, the machine was offline past its bound.
+	presenceSilent
+)
+
+// presenceEntry is presence.<node> as the watchman wrote it: whether the machine is online and when the entry was made.
+// The watchman writes it only when it changes, so for an online machine the time is when it connected.
+type presenceEntry struct {
+	Online bool
+	At     time.Time
+}
+
+// applyPresence sets what the watchman says of a machine: the verdict of its stuck presence key (nil when there is none).
+// A presence entry that says online counts the machine as heard now; otherwise last heard is the later of the store's
+// last_seen and the entry's time (the time it went offline).
+func applyPresence(m *machineFacts, stuck *sdkdelivery.Stuck, entry *presenceEntry, now time.Time) {
+	switch {
+	case stuck == nil:
+		m.presence = presenceUnsaid
+	case stuck.Moved:
+		m.presence = presenceHeard
+	default:
+		m.presence = presenceSilent
+	}
+	switch {
+	case entry == nil:
+	case entry.Online:
+		m.lastHeard = now
+	case entry.At.After(m.lastHeard):
+		m.lastHeard = entry.At
+	}
+}
+
+// silent says the machine is silent: the watchman's verdict where it gave one, else the heartbeat rule. A machine never
+// heard from is not silent, it has not joined.
+func (m machineFacts) silent(now time.Time) bool {
+	switch m.presence {
+	case presenceSilent:
+		return true
+	case presenceHeard:
+		return false
+	}
+	return !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every)
+}
+
+// stuckObservations are the conditions of what stopped moving: one for each key not moved, in the delivery scope, kind
+// stuck, naming the key and what it waited for. A node's presence key is left to the silent check; a moved key raises
+// nothing, so its condition clears.
+func stuckObservations(entries map[string]sdkdelivery.Stuck) []conditions.Observation {
+	keys := make([]string, 0, len(entries))
+	for k := range entries {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	var out []conditions.Observation
+	for _, k := range keys {
+		s := entries[k]
+		if s.Moved {
+			continue
+		}
+		if s.Bucket == sdkdelivery.PresenceBucket {
+			if node, _, _, err := sdkdelivery.ParsePresenceKey(s.Key); err == nil && node != "" {
+				continue
+			}
+		}
+		waited := s.Waited
+		if waited == "" {
+			waited = "a move"
+		}
+		out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: k, Kind: kindStuck,
+			Severity: conditions.Warning,
+			Summary: fmt.Sprintf("%s has not moved since %s: it waited for %s", k, s.Since.UTC().Format("2006-01-02 15:04 MST"),
+				waited),
+			Said: fmt.Sprintf("%s: waiting for %s since %s", k, waited, s.Since.UTC().Format(time.RFC3339))})
+	}
+	return out
+}
+
+// readStuck is every key of the stuck bucket and its value; none when the bucket is not there yet (the watchman not
+// delivered). A value of another version is left out, said.
+func readStuck(ctx context.Context, conn *nats.Conn) (map[string]sdkdelivery.Stuck, error) {
+	api, err := jetstream.New(conn)
+	if err != nil {
+		return nil, err
+	}
+	kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
+	if errors.Is(err, jetstream.ErrBucketNotFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("the watchman's stuck bucket: %w", err)
+	}
+	keys, err := boundedCall(ctx, "listing the stuck keys", func(ctx context.Context) ([]string, error) { return kv.Keys(ctx) })
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	out := map[string]sdkdelivery.Stuck{}
+	for _, key := range keys {
+		e, err := kv.Get(ctx, key)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		s, err := sdkdelivery.DecodeStuck(e.Value())
+		if err != nil {
+			fmt.Printf("the stuck key %s is not read: %v\n", key, err)
+			continue
+		}
+		out[key] = s
+	}
+	return out, nil
+}
+
+// readPresence is, by node, presence.<node>'s entry; a node without an entry is left out.
+func readPresence(ctx context.Context, conn *nats.Conn, nodes []string) (map[string]presenceEntry, error) {
+	api, err := jetstream.New(conn)
+	if err != nil {
+		return nil, err
+	}
+	kv, err := api.KeyValue(ctx, sdkdelivery.PresenceBucket)
+	if errors.Is(err, jetstream.ErrBucketNotFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("the watchman's presence bucket: %w", err)
+	}
+	out := map[string]presenceEntry{}
+	for _, node := range nodes {
+		key, err := sdkdelivery.PresenceNodeKey(node)
+		if err != nil {
+			continue
+		}
+		e, err := kv.Get(ctx, key)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		p, err := sdkdelivery.DecodePresence(e.Value())
+		if err != nil {
+			fmt.Printf("the presence key %s is not read: %v\n", key, err)
+			continue
+		}
+		out[node] = presenceEntry{Online: p.Online, At: e.Created()}
+	}
+	return out, nil
+}
+
+// withPresence says of each machine what the watchman says: its stuck presence key and its presence entry's time.
+func withPresence(ctx context.Context, conn *nats.Conn, machines []machineFacts, now time.Time) error {
+	stuck, err := readStuck(ctx, conn)
+	if err != nil {
+		return err
+	}
+	names := make([]string, len(machines))
+	for i, m := range machines {
+		names[i] = m.name
+	}
+	entries, err := readPresence(ctx, conn, names)
+	if err != nil {
+		return err
+	}
+	for i := range machines {
+		var said *sdkdelivery.Stuck
+		if key, err := sdkdelivery.StuckKey(sdkdelivery.PresenceBucket, machines[i].name); err == nil {
+			if s, ok := stuck[key]; ok {
+				said = &s
+			}
+		}
+		var entry *presenceEntry
+		if e, ok := entries[machines[i].name]; ok {
+			entry = &e
+		}
+		applyPresence(&machines[i], said, entry, now)
+	}
+	return nil
+}
+
+// probeStuck is DM: every key the watchman says stopped moving is raised, but a node's presence, which the silent check
+// takes.
+func probeStuck(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
+	if d.js == nil {
+		return nil, nil
+	}
+	entries, err := readStuck(ctx, d.js.Conn())
+	if err != nil {
+		return nil, err
+	}
+	return stuckObservations(entries), nil
+}

cmd/mesh-controller/stuck_probe_integration_test.go

@@ -0,0 +1,82 @@
+//go:build integration
+
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// 1156/AC4: on the embedded bus with the watchman's buckets, a stuck key written raises a condition and written as moved
+// clears it; a node's presence key not moved says the machine silent through the facts, and moved says it heard.
+func TestAStuckKeyWrittenAndThenMovedRaisesAndClearsACondition(t *testing.T) { // 1156/AC4
+	js, err := broker.Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	if _, err := broker.RaiseBuckets(js, []broker.Bucket{
+		{Module: sdkdelivery.WatchmanModule, Name: "stuck"},
+		{Module: sdkdelivery.WatchmanModule, Name: "presence", History: sdkdelivery.PresenceHistory},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	write := func(bucket, key string, moved bool) string {
+		t.Helper()
+		stuckKey, err := sdkdelivery.StuckKey(bucket, key)
+		if err != nil {
+			t.Fatal(err)
+		}
+		body, err := sdkdelivery.EncodeStuck(sdkdelivery.Stuck{Bucket: bucket, Key: key, Waited: "an answer", Since: time.Now().UTC(), Moved: moved})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := kv.Put(ctx, stuckKey, body); err != nil {
+			t.Fatal(err)
+		}
+		return stuckKey
+	}
+	d := &doctor{js: js}
+	stuckKey := write(sdkdelivery.PulseBucket, "builds.x", false)
+	write(sdkdelivery.PresenceBucket, "ace", false)
+	got, err := probeStuck(ctx, d)
+	if err != nil || len(got) != 1 || got[0].ID != stuckKey || got[0].Kind != kindStuck {
+		t.Fatalf("a stuck key written: %+v %v", got, err)
+	}
+	machines := []machineFacts{{name: "ace", lastHeard: time.Now()}}
+	if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	if !machines[0].silent(time.Now()) {
+		t.Fatal("a presence key not moved did not say the machine silent")
+	}
+	write(sdkdelivery.PulseBucket, "builds.x", true)
+	write(sdkdelivery.PresenceBucket, "ace", true)
+	if got, err := probeStuck(ctx, d); err != nil || len(got) != 0 {
+		t.Fatalf("a stuck key written as moved: %+v %v", got, err)
+	}
+	machines = []machineFacts{{name: "ace", lastHeard: time.Now().Add(-time.Hour), every: time.Minute}}
+	if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	if machines[0].silent(time.Now()) {
+		t.Fatal("a presence key that moved still says the machine silent")
+	}
+}

cmd/mesh-controller/stuck_probe_test.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+func aStuckKey(t *testing.T, bucket, key string) string {
+	t.Helper()
+	k, err := sdkdelivery.StuckKey(bucket, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return k
+}
+
+// 1156/AC1: each not-moved stuck key that is not a node's presence is an observation of the delivery scope, kind stuck,
+// naming the key and what it waited for; a moved key raises nothing, so its condition clears.
+func TestEachStuckKeyThatHasNotMovedIsAConditionOfTheDeliveryScope(t *testing.T) { // 1156/AC1
+	since := time.Date(2026, 10, 18, 19, 0, 0, 0, time.UTC)
+	pulse := aStuckKey(t, sdkdelivery.PulseBucket, "x")
+	moved := aStuckKey(t, sdkdelivery.HaveBucket, "ace.zsh")
+	node := aStuckKey(t, sdkdelivery.PresenceBucket, "ace")
+	part := aStuckKey(t, sdkdelivery.PresenceBucket, "part.mesh-controller.one")
+	entries := map[string]sdkdelivery.Stuck{
+		pulse: {Bucket: sdkdelivery.PulseBucket, Key: "x", Waited: "the build of zsh to be answered", Since: since},
+		moved: {Bucket: sdkdelivery.HaveBucket, Key: "ace.zsh", Waited: "a want", Since: since, Moved: true},
+		node:  {Bucket: sdkdelivery.PresenceBucket, Key: "ace", Waited: "its return", Since: since},
+		part:  {Bucket: sdkdelivery.PresenceBucket, Key: "part.mesh-controller.one", Waited: "its return", Since: since},
+	}
+	got := stuckObservations(entries)
+	if len(got) != 2 {
+		t.Fatalf("observations %+v, want the pulse key and the part's presence", got)
+	}
+	byID := map[string]conditions.Observation{}
+	for _, o := range got {
+		if o.Scope != conditions.ScopeDelivery || o.Kind != kindStuck {
+			t.Errorf("%+v is not a stuck condition of the delivery scope", o)
+		}
+		byID[o.ID] = o
+	}
+	o, ok := byID[pulse]
+	if !ok || !strings.Contains(o.Summary, pulse) || !strings.Contains(o.Summary, "the build of zsh to be answered") {
+		t.Fatalf("the condition does not name the key and what it waited for: %+v", got)
+	}
+	if _, ok := byID[part]; !ok {
+		t.Errorf("a part's presence is left out: %v", byID)
+	}
+	// Moved again: the same key, written as moved, raises nothing.
+	entries[pulse] = sdkdelivery.Stuck{Bucket: sdkdelivery.PulseBucket, Key: "x", Since: since, Moved: true}
+	if got := stuckObservations(entries); len(got) != 1 {
+		t.Fatalf("a moved key still raises: %+v", got)
+	}
+}
+
+func aMachineHeardAt(now time.Time, ago time.Duration) machineFacts {
+	return machineFacts{name: "laptop", lastHeard: now.Add(-ago), every: time.Minute}
+}
+
+// 1156/AC2: a node's stuck presence key not moved makes the machine silent, unless it sleeps; moved, it is heard; no
+// timer is armed (the verdict is the key's, whatever the time of the last word).
+func TestAStuckPresenceKeyIsTheSilentVerdict(t *testing.T) { // 1156/AC2
+	now := time.Now()
+	f := calm(now)
+	f.machines = append(f.machines, aMachineHeardAt(now, 10*time.Second))
+	notMoved := sdkdelivery.Stuck{Bucket: sdkdelivery.PresenceBucket, Key: "laptop", Waited: "its return", Since: now.Add(-3 * time.Minute)}
+	applyPresence(&f.machines[1], &notMoved, nil, now)
+	got := watchHeartbeats(f)
+	if len(got) != 1 || got[0].Machine != "laptop" || got[0].Kind != "silent" {
+		t.Fatalf("a presence key not moved: %+v", got)
+	}
+	// Asleep: not silent.
+	f.machines[1].power = link.PowerState{State: "sleeping", At: now.Add(-time.Hour)}
+	if got := watchHeartbeats(f); len(got) != 0 {
+		t.Fatalf("a sleeping machine raised %+v", got)
+	}
+	// Moved: heard, whatever the store's last word says.
+	f.machines[1].power = link.PowerState{}
+	f.machines[1].lastHeard = now.Add(-2 * time.Hour)
+	moved := notMoved
+	moved.Moved = true
+	applyPresence(&f.machines[1], &moved, nil, now)
+	if got := watchHeartbeats(f); len(got) != 0 {
+		t.Fatalf("a presence key that moved raised %+v", got)
+	}
+}
+
+// 1156/AC3: with a presence entry that says offline, last heard is the later of last_seen and the entry's time; with none,
+// last_seen. An entry that says online counts as heard now: the watchman writes it only when it changes, so its time is
+// when the machine connected, and a machine online for hours with no stuck key is not silent, whatever last_seen says.
+func TestLastHeardFollowsThePresenceEntry(t *testing.T) { // 1156/AC3
+	now := time.Now()
+	seen := now.Add(-time.Hour)
+	m := machineFacts{name: "laptop", lastHeard: seen, every: time.Minute}
+	applyPresence(&m, nil, nil, now)
+	if !m.lastHeard.Equal(seen) {
+		t.Fatalf("with no presence entry last heard is %v, want last_seen %v", m.lastHeard, seen)
+	}
+	offlineAt := now.Add(-5 * time.Minute)
+	applyPresence(&m, nil, &presenceEntry{Online: false, At: offlineAt}, now)
+	if !m.lastHeard.Equal(offlineAt) {
+		t.Fatalf("offline since %v: last heard is %v", offlineAt, m.lastHeard)
+	}
+	m = machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute), every: time.Minute}
+	applyPresence(&m, nil, &presenceEntry{Online: false, At: now.Add(-time.Hour)}, now)
+	if !m.lastHeard.Equal(now.Add(-time.Minute)) {
+		t.Fatalf("an older entry replaced a newer last_seen: %v", m.lastHeard)
+	}
+	for _, lastSeen := range []time.Duration{10 * time.Second, 5 * time.Hour} {
+		m := machineFacts{name: "laptop", lastHeard: now.Add(-lastSeen), every: time.Minute}
+		applyPresence(&m, nil, &presenceEntry{Online: true, At: now.Add(-6 * time.Hour)}, now)
+		if m.silent(now) {
+			t.Fatalf("a machine online since hours ago (last_seen %v ago) with no stuck key is silent", lastSeen)
+		}
+	}
+}

cmd/mesh-controller/watchdogs.go

@@ -155,6 +155,8 @@ type machineFacts struct {
 	tools      bool
 	toolsHeard time.Time
 	toolsEvery time.Duration
+	// presence is the watchman's word on whether it is online (novox/hq ADR 0433 §3); unsaid, the heartbeat rule judges.
+	presence presenceVerdict
 }
 
 // asleep says the machine said it would be away and has not said it is back (ADR 0211).
@@ -470,7 +472,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
 			}
 			m.reportedCurrent = r.Current
 		}
-		if !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every) {
+		if m.silent(now) {
 			anyPast = true
 		}
 		if m.tools && now.Sub(later(m.toolsHeard, link.ToolsBeats.Started())) > heartbeatBound(m.toolsEvery) {
@@ -478,6 +480,19 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
 		}
 		out = append(out, m)
 	}
+	// What the watchman says of each machine: its stuck presence key is the silent verdict, and presence.<node>'s time when
+	// it was last heard (ADR 0433 §3). Unreadable is said, and the heartbeat rule judges as before.
+	if w.js != nil {
+		if err := withPresence(ctx, w.js.Conn(), out, now); err != nil {
+			fmt.Printf("what the watchman says of the machines' presence cannot be read: %v\n", err)
+		} else {
+			for _, m := range out {
+				if m.silent(now) {
+					anyPast = true
+				}
+			}
+		}
+	}
 	// What a machine past its bound last said of its power, read only then: a machine that said it
 	// is asleep is not lost (ADR 0211). Unreadable is said: a sleeping laptop is then called silent,
 	// which is the louder mistake and the right one to make.