Judge S21 by time since the first failing report while the machine is heard, not by how many reports said it (issue 233 review)

cmd/mesh-controller/apply_failing.go

@@ -25,20 +25,21 @@ const packageManagerKey = "the package manager ("
 // watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
 // to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
 // declaration for an hour and a half because its package manager did not answer, and the mesh's only word
-// on it was a line in `status`. Raised once its reports have said the same failure for longer than one
-// apply interval, over at least two reports — a laptop shut for a week has had one attempt, not a week of
-// them — naming the refusal's or the failure's words; cleared by a clean apply. Not while the machine is
-// silent or asleep: S1 says silent, and asleep is not lost.
+// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
+// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
+// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
+// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
+// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
 func watchApplyFailing(f *signalFacts) []conditions.Observation {
-	quiet := map[string]bool{}
+	heard := map[string]bool{}
 	for _, m := range f.machines {
-		if m.asleep() || (!m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every)) {
-			quiet[m.name] = true
+		if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
+			heard[m.name] = true
 		}
 	}
 	var out []conditions.Observation
 	for _, d := range f.doing {
-		if !d.Wrong() || d.Since == nil || d.Times < 2 || f.now.Sub(*d.Since) <= applyEvery || quiet[d.Node] {
+		if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
 			continue
 		}
 		out = append(out, applyFailing(d, f.now))
@@ -57,14 +58,13 @@ func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
 	if d.Outcome != inventory.OutcomeRefused {
 		how = "applied its declaration only in part"
 	}
-	o.Summary = fmt.Sprintf("%s has %s on %d reports in a row since %s (bound %s): %s", m, how, d.Times, since,
-		ago(applyEvery), failedNames(d))
+	o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
 	o.Headline = m + " is not applying its update"
 	o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
 		m, plainHow(d), humanDuration(now.Sub(*d.Since)))
 	if tool := unansweringTool(d); tool != "" {
 		o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
-			"is applied, on %d reports in a row since %s (bound %s)", m, tool, d.Times, since, ago(applyEvery))
+			"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
 		o.Headline = fmt.Sprintf("%s cannot install packages", m)
 		o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
 			"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+

cmd/mesh-controller/apply_failing_test.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	"context"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -37,8 +39,8 @@ func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
 	if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
 		t.Errorf("the condition is not the machine's: %s", o.Key())
 	}
-	if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "18 reports") {
-		t.Errorf("the summary does not name the tool and how long: %s", o.Summary)
+	if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") {
+		t.Errorf("the summary does not name the tool and since when: %s", o.Summary)
 	}
 	if strings.Contains(o.Summary, "/etc/") {
 		t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
@@ -66,18 +68,56 @@ func TestARefusalThatGoesOnIsRaised(t *testing.T) {
 	}
 }
 
-// **One attempt is not a fault that goes on, and a silent or sleeping machine is S1's**: a laptop shut for a
-// week comes back with one report of an old failure; a machine not heard from says silent, not this.
-func TestOneReportOrAQuietMachineIsNotRaised(t *testing.T) {
+// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233):
+// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one
+// younger is not yet; and a silent or sleeping machine is S1's.
+func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) {
 	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
 	f := calm(now)
-	f.doing = []inventory.Doing{unansweringPackageManager(now, 7*24*time.Hour, 1)}
-	if got := watchApplyFailing(f); len(got) != 0 {
-		t.Errorf("one report was raised as a failure that goes on: %+v", got)
+	f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)}
+	if got := watchApplyFailing(f); len(got) != 1 {
+		t.Errorf("one report of a failure six minutes old was not raised: %+v", got)
 	}
-	f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 12)}
+	f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)}
+	if got := watchApplyFailing(f); len(got) != 0 {
+		t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got)
+	}
+	f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
 	f.machines[0].lastHeard = now.Add(-time.Hour)
 	if got := watchApplyFailing(f); len(got) != 0 {
 		t.Errorf("a silent machine was raised for its last report: %+v", got)
 	}
+	f = calm(now)
+	f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
+	f.doing[0].Node = "a-machine-the-mesh-does-not-know"
+	if got := watchApplyFailing(f); len(got) != 0 {
+		t.Errorf("a machine not heard of was raised: %+v", got)
+	}
+}
+
+// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says
+// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised
+// is cleared through the keeper on the next tick.
+func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) {
+	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
+	store := conditions.NewInMemory()
+	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
+		Teller: &conditions.Told{}, Now: func() time.Time { return now }})
+	defer k.Close(context.Background())
+	w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
+
+	failing := calm(now)
+	failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)}
+	w.see(t.Context(), failing)
+	open, err := k.Open(t.Context())
+	if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing {
+		t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err)
+	}
+
+	clean := calm(now.Add(5 * time.Minute))
+	clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine
+	w.see(t.Context(), clean)
+	if open, _ := k.Open(t.Context()); len(open) != 0 {
+		t.Fatalf("the condition stayed open after the clean re-apply: %+v", open)
+	}
 }

cmd/mesh-controller/signals.go

@@ -255,9 +255,10 @@ var signalsTable = []signalRow{
 			return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since })
 		}},
 	{Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)",
-		Bound: "one apply interval (5 min) past the first report of the same failure, over at least two reports; not " +
-			"while the machine is silent or asleep; naming the refusal, or the package manager that does not answer " +
-			"with its words; cleared by a clean apply",
+		Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " +
+			"is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " +
+			"asleep; naming the refusal, or the package manager that does not answer with its words; cleared by the " +
+			"clean apply it then says",
 		Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2,
 		needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing,
 		newest: func(f *signalFacts) time.Time {

cmd/mesh-controller/signals_test.go

@@ -186,12 +186,12 @@ var suppressions = map[string]suppression{
 		},
 		past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} },
 	},
-	// A machine whose package manager does not answer (novox/hq issue 233): said once the same failure has
-	// outlasted one apply interval over two reports. Inside: a second report, only five minutes after the first.
+	// A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the
+	// failure is older than one apply interval while it is heard. Inside: exactly five minutes old.
 	"S21": {
-		inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 2)} },
+		inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 1)} },
 		past: func(f *signalFacts) {
-			f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 2)}
+			f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 1)}
 		},
 	},
 	// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.