diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go index 5f76ef80..b9698b10 100644 --- a/cmd/mesh-controller/apply_failing.go +++ b/cmd/mesh-controller/apply_failing.go @@ -30,6 +30,11 @@ const packageManagerKey = "the package manager (" // says a re-apply only when how it went changed, so the same failure is one report however long it lasts. // Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while // the machine is silent, asleep or not known: S1 says silent, and asleep is not lost. +// +// **The bound gates raising, never keeping** (sweep 2026-10-30, issue 774). A report says a failure again whenever its +// words change, with a new time: on novox the vault's refusal of every secret replaced a bus refusal at 16:56, and the +// condition cleared for five minutes while every module still failed. Once raised, it stays raised while any failure +// stands on that machine, judged by the open condition the keeper holds on the bus, not by a report's time. func watchApplyFailing(f *signalFacts) []conditions.Observation { heard := map[string]bool{} for _, m := range f.machines { @@ -40,7 +45,7 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation { var out []conditions.Observation said := map[string]bool{} for _, d := range f.doing { - if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] { + if !d.Wrong() || d.Since == nil || !heard[d.Node] || (f.now.Sub(*d.Since) <= applyEvery && !f.applyFailingOpen[d.Node]) { continue } out = append(out, applyFailing(d, f.now)) @@ -51,7 +56,8 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation { // on a machine whose node-engine applied what it was sent. failed := map[string][]inventory.ProvReport{} for _, r := range f.provFailed { - if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] { + if r.Status != inventory.ProvFailed || !heard[r.Node] || said[r.Node] || + (f.now.Sub(r.At) <= applyEvery && !f.applyFailingOpen[r.Node]) { continue } failed[r.Node] = append(failed[r.Node], r) @@ -62,6 +68,17 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation { return out } +// applyFailingOpen are the machines whose S21 condition is open among the open conditions. +func applyFailingOpen(open []conditions.Condition) map[string]bool { + out := map[string]bool{} + for _, c := range open { + if c.Kind == kindApplyFailing && c.Subject.Machine != "" { + out[c.Subject.Machine] = true + } + } + return out +} + // provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to // place. func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation { diff --git a/cmd/mesh-controller/provisioned_test.go b/cmd/mesh-controller/provisioned_test.go index 71ecac88..67539647 100644 --- a/cmd/mesh-controller/provisioned_test.go +++ b/cmd/mesh-controller/provisioned_test.go @@ -7,6 +7,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) @@ -72,3 +73,39 @@ func TestAFailedPlacingRaisesApplyFailing(t *testing.T) { t.Fatalf("a failed placing is not raised as anchor's apply-failing: %+v", got) } } + +// Sweep 2026-10-30 (issue 774): a failure said again in new words carries a new time, and the condition it raised stays +// raised while it stands; only a machine with no open condition waits out the bound, and a cleared failure clears it. +func TestApplyFailingStaysRaisedWhileAnyFailureStands(t *testing.T) { + now := time.Date(2026, 10, 30, 16, 57, 0, 0, time.UTC) + since := now.Add(-time.Minute) + for name, c := range map[string]struct { + open bool + prov string + doing bool + raised bool + }{ + "a fresh placing failure, nothing open": {false, inventory.ProvFailed, false, false}, + "a fresh placing failure, condition open": {true, inventory.ProvFailed, false, true}, + "placing applied again, condition open": {true, inventory.ProvApplied, false, false}, + "a fresh failed apply, condition open": {true, "", true, true}, + "a fresh failed apply, nothing open": {false, "", true, false}, + } { + f := calm(now) + if c.open { + f.applyFailingOpen = applyFailingOpen([]conditions.Condition{{Kind: kindApplyFailing, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}}}) + } + if c.prov != "" { + f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "postgres", Status: c.prov, + Reason: "the vault refused it", At: now.Add(-time.Minute)}} + } + if c.doing { + f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Refused: "no", At: since, + Since: &since}} + } + if got := watchApplyFailing(f); (len(got) == 1) != c.raised || len(got) > 1 { + t.Errorf("%s: %d observation(s), raised want %v", name, len(got), c.raised) + } + } +} diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index c2207c16..52b0d038 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -58,6 +58,9 @@ type signalFacts struct { // provFailed is every module whose files a machine's provisioner client last said it failed to place (S21, // novox/hq ADR 0362 ยง3): that machine's failed apply. provFailed []inventory.ProvReport + // applyFailingOpen are the machines whose S21 condition is open: kept raised while any failure stands, whatever + // the newest report's time (sweep 2026-10-30). + applyFailingOpen map[string]bool loop loopFacts loopErr error @@ -317,6 +320,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } else { f.standings = providerStandings(open) f.providerWords = providerConditions(open) + f.applyFailingOpen = applyFailingOpen(open) } f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet)) f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)