From 8a658a4290c965cc09ecb2e5322020da27f62234 Mon Sep 17 00:00:00 2001
From: jochen <jschoubben@gmail.com>
Date: Sat, 31 Oct 2026 22:01:59 +0100
Subject: [PATCH] Watchman: an unhealthy finding read back at the start is kept
 as it was (issue 1952)

At a restart a standing finding is observed again by the replayed failed have: kept, not confirmed again, and its urgent
signal runs on from the finding's Since; an urgent one stays urgent with no signal restarted.
Quick: skip the confirm when a finding stands. Thorough, chosen: the same, plus urgency read from the finding itself.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
---
 .../internal/watch/findings_test.go           | 31 +++++++++++++++++++
 .../internal/watch/signals.go                 | 17 ++++++++--
 2 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go
index 3e8c5f0..4466789 100644
--- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go
+++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go
@@ -158,6 +158,29 @@ func TestTheWatchmansFindings(t *testing.T) {
 			expire(w, c, signalKey("unhealthy", "web.ace"), UnhealthyUrgentAfter)
 			w.Have("ace.mail", haveOf(t, delivery.PhaseRunning, delivery.HealthWaits, "a part waits for the secret token"), Put) // needs-operator is not observed here
 		}, map[string]string{"module.web.ace.unhealthy": Urgent}},
+		{"WM3: a warning unhealthy finding read back at the start is kept as it was, urgent when its four hours from Since are out", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
+			seedUnhealthy(t, w, o, Warning, t0.Add(-time.Hour))
+			w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put)
+			w.Reconcile()
+			if o.findings["module.web.ace.unhealthy"].Severity != Warning {
+				t.Fatalf("cleared: findings %v", o.findings)
+			}
+			if ttl := o.ttls[signalKey("unhealthy", "web.ace")]; ttl != UnhealthyUrgentAfter-time.Hour {
+				t.Fatalf("urgent signal restarted from zero: %v", ttl)
+			}
+			if _, ok := w.signals[signalKey("unhealthy-confirm", "web.ace")]; ok {
+				t.Fatal("a standing finding is confirmed again")
+			}
+			expire(w, c, signalKey("unhealthy", "web.ace"), UnhealthyUrgentAfter-time.Hour)
+		}, map[string]string{"module.web.ace.unhealthy": Urgent}},
+		{"WM3: an urgent unhealthy finding read back at the start stays urgent, with no signal restarted", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
+			seedUnhealthy(t, w, o, Urgent, t0.Add(-5*time.Hour))
+			w.Have("ace.web", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "x"), Put)
+			w.Reconcile()
+			if o.findings["module.web.ace.unhealthy"].Severity != Urgent || len(w.signals) != 0 {
+				t.Fatalf("findings %v signals %v", o.findings, w.signals)
+			}
+		}, map[string]string{"module.web.ace.unhealthy": Urgent}},
 		{"WM3: a restart replays a have already failed: it is confirmed after a minute like a new one", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
 			w.Have("novox.fail2ban", haveOf(t, delivery.PhaseRunning, delivery.HealthFailed, "down since 19:40"), Put)
 			expire(w, c, signalKey("unhealthy-confirm", "fail2ban.novox"), UnhealthyConfirmAfter)
@@ -370,3 +393,11 @@ func TestTheWatchmansFindings(t *testing.T) {
 		})
 	}
 }
+
+func seedUnhealthy(t *testing.T, w *Watchman, o *fakeOut, sev string, since time.Time) {
+	t.Helper()
+	f := Finding{Scope: ScopeModule, ID: "web.ace", Token: "unhealthy", Kind: KindUnhealthy, Machine: "ace", Severity: sev, Source: "health", Since: since, Summary: "web on ace is not healthy: x"}
+	v, _ := EncodeFinding(f)
+	o.findings[f.Key()], _ = DecodeFinding(v)
+	w.SeedFinding(f.Key(), v)
+}
diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go
index 3f3ace0..e28ec1e 100644
--- a/modules/mesh-delivery-watchman/internal/watch/signals.go
+++ b/modules/mesh-delivery-watchman/internal/watch/signals.go
@@ -504,12 +504,23 @@ func (w *Watchman) haveFindings(key string, node, module string, h *delivery.Hav
 				Said:     firstLine(orWords(h.Detail, "failed")),
 				Headline: fmt.Sprintf("%s not working on %s", module, node)}
 			id := module + "." + node
-			if _, ok := w.signals[signalKey("unhealthy-confirm", id)]; !ok {
+			urgentIn := UnhealthyUrgentAfter
+			if w.seededFindings[unhealthyKey] {
+				// Read back at the start: it stands, so it is observed again, kept as it was (not cleared and raised again), and
+				// its four hours run on from its own Since, not from zero; one that already went urgent needs no signal.
+				delete(w.seededFindings, unhealthyKey)
+				old, err := DecodeFinding(w.findings[unhealthyKey])
+				if err == nil && old.Severity == Urgent {
+					urgentIn = 0
+				} else if err == nil && !old.Since.IsZero() {
+					urgentIn = max(time.Second, UnhealthyUrgentAfter-now.Sub(old.Since))
+				}
+			} else if _, ok := w.signals[signalKey("unhealthy-confirm", id)]; !ok {
 				w.signal("unhealthy-confirm", id, UnhealthyConfirmAfter, signalDue{Do: dueFind, While: key, Finding: f})
 			}
-			if _, ok := w.signals[signalKey("unhealthy", id)]; !ok {
+			if _, ok := w.signals[signalKey("unhealthy", id)]; !ok && urgentIn > 0 {
 				f.Severity = Urgent
-				w.signal("unhealthy", id, UnhealthyUrgentAfter, signalDue{Do: dueFind, While: key, Finding: f})
+				w.signal("unhealthy", id, urgentIn, signalDue{Do: dueFind, While: key, Finding: f})
 			}
 		}
 	} else {