Keep apply-failing raised while any failure stands, so a failure said again in new words never clears it (issue 774)

machine.novox.apply-failing cleared at 16:56 while the vault refused every secret there: the provisioner
client says a failure again whenever its words change, with a new time, and S21 measured each module's
failure from that newest time, so every new wording hid the failure for one apply interval (raised again
17:01). The same holds for the node-engine's own failed apply, whose since restarts with new words.

The bound now gates raising only: once the condition is open (read from the keeper's open conditions on
the bus), it stays while any module's apply fails on that machine, and clears when none does.

Quick fix: this hysteresis on the open condition in the controller. Thorough fix: the provisioner client
carries the start of a failure run (failing-since) in its report, so every reader measures from the true
start; that is mesh-catalog's mesh-provisioner-client, routed to the lead. Chose the quick fix here,
because the check moves to mesh-watchdogs (ADR 0564) and the condition's own state already proves the run.

Research 072 piece: milestone 3 (health leaves the controller): the judgement reads bus state, the open
condition, rather than a report's time.

cmd/mesh-controller/apply_failing.go

@@ -30,6 +30,11 @@ const packageManagerKey = "the package manager ("
 // says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
 // Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
 // the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
+//
+// **The bound gates raising, never keeping** (sweep 2026-10-30, issue 774). A report says a failure again whenever its
+// words change, with a new time: on novox the vault's refusal of every secret replaced a bus refusal at 16:56, and the
+// condition cleared for five minutes while every module still failed. Once raised, it stays raised while any failure
+// stands on that machine, judged by the open condition the keeper holds on the bus, not by a report's time.
 func watchApplyFailing(f *signalFacts) []conditions.Observation {
 	heard := map[string]bool{}
 	for _, m := range f.machines {
@@ -40,7 +45,7 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
 	var out []conditions.Observation
 	said := map[string]bool{}
 	for _, d := range f.doing {
-		if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
+		if !d.Wrong() || d.Since == nil || !heard[d.Node] || (f.now.Sub(*d.Since) <= applyEvery && !f.applyFailingOpen[d.Node]) {
 			continue
 		}
 		out = append(out, applyFailing(d, f.now))
@@ -51,7 +56,8 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
 	// on a machine whose node-engine applied what it was sent.
 	failed := map[string][]inventory.ProvReport{}
 	for _, r := range f.provFailed {
-		if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
+		if r.Status != inventory.ProvFailed || !heard[r.Node] || said[r.Node] ||
+			(f.now.Sub(r.At) <= applyEvery && !f.applyFailingOpen[r.Node]) {
 			continue
 		}
 		failed[r.Node] = append(failed[r.Node], r)
@@ -62,6 +68,17 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
 	return out
 }
 
+// applyFailingOpen are the machines whose S21 condition is open among the open conditions.
+func applyFailingOpen(open []conditions.Condition) map[string]bool {
+	out := map[string]bool{}
+	for _, c := range open {
+		if c.Kind == kindApplyFailing && c.Subject.Machine != "" {
+			out[c.Subject.Machine] = true
+		}
+	}
+	return out
+}
+
 // provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
 // place.
 func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {

cmd/mesh-controller/provisioned_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -72,3 +73,39 @@ func TestAFailedPlacingRaisesApplyFailing(t *testing.T) {
 		t.Fatalf("a failed placing is not raised as anchor's apply-failing: %+v", got)
 	}
 }
+
+// Sweep 2026-10-30 (issue 774): a failure said again in new words carries a new time, and the condition it raised stays
+// raised while it stands; only a machine with no open condition waits out the bound, and a cleared failure clears it.
+func TestApplyFailingStaysRaisedWhileAnyFailureStands(t *testing.T) {
+	now := time.Date(2026, 10, 30, 16, 57, 0, 0, time.UTC)
+	since := now.Add(-time.Minute)
+	for name, c := range map[string]struct {
+		open   bool
+		prov   string
+		doing  bool
+		raised bool
+	}{
+		"a fresh placing failure, nothing open":   {false, inventory.ProvFailed, false, false},
+		"a fresh placing failure, condition open": {true, inventory.ProvFailed, false, true},
+		"placing applied again, condition open":   {true, inventory.ProvApplied, false, false},
+		"a fresh failed apply, condition open":    {true, "", true, true},
+		"a fresh failed apply, nothing open":      {false, "", true, false},
+	} {
+		f := calm(now)
+		if c.open {
+			f.applyFailingOpen = applyFailingOpen([]conditions.Condition{{Kind: kindApplyFailing,
+				Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}}})
+		}
+		if c.prov != "" {
+			f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "postgres", Status: c.prov,
+				Reason: "the vault refused it", At: now.Add(-time.Minute)}}
+		}
+		if c.doing {
+			f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Refused: "no", At: since,
+				Since: &since}}
+		}
+		if got := watchApplyFailing(f); (len(got) == 1) != c.raised || len(got) > 1 {
+			t.Errorf("%s: %d observation(s), raised want %v", name, len(got), c.raised)
+		}
+	}
+}

cmd/mesh-controller/watchdogs.go

@@ -58,6 +58,9 @@ type signalFacts struct {
 	// provFailed is every module whose files a machine's provisioner client last said it failed to place (S21,
 	// novox/hq ADR 0362 §3): that machine's failed apply.
 	provFailed []inventory.ProvReport
+	// applyFailingOpen are the machines whose S21 condition is open: kept raised while any failure stands, whatever
+	// the newest report's time (sweep 2026-10-30).
+	applyFailingOpen map[string]bool
 
 	loop    loopFacts
 	loopErr error
@@ -317,6 +320,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 	} else {
 		f.standings = providerStandings(open)
 		f.providerWords = providerConditions(open)
+		f.applyFailingOpen = applyFailingOpen(open)
 	}
 	f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
 	f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)