Remove the leftover merges consumer behind the false deaf alarm; apply-failing stays raised while a failure stands (issue 774) #484

Closed
mesh-tools wants to merge 14 commits from fix/sweep-2026-10-30 into main
3 changed files with 60 additions and 2 deletions
Showing only changes of commit 6ffcea4f1d - Show all commits
+19 -2
View File
@@ -30,6 +30,11 @@ const packageManagerKey = "the package manager ("
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
//
// **The bound gates raising, never keeping** (sweep 2026-10-30, issue 774). A report says a failure again whenever its
// words change, with a new time: on novox the vault's refusal of every secret replaced a bus refusal at 16:56, and the
// condition cleared for five minutes while every module still failed. Once raised, it stays raised while any failure
// stands on that machine, judged by the open condition the keeper holds on the bus, not by a report's time.
func watchApplyFailing(f *signalFacts) []conditions.Observation {
heard := map[string]bool{}
for _, m := range f.machines {
@@ -40,7 +45,7 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
said := map[string]bool{}
for _, d := range f.doing {
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
if !d.Wrong() || d.Since == nil || !heard[d.Node] || (f.now.Sub(*d.Since) <= applyEvery && !f.applyFailingOpen[d.Node]) {
continue
}
out = append(out, applyFailing(d, f.now))
@@ -51,7 +56,8 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
// on a machine whose node-engine applied what it was sent.
failed := map[string][]inventory.ProvReport{}
for _, r := range f.provFailed {
if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
if r.Status != inventory.ProvFailed || !heard[r.Node] || said[r.Node] ||
(f.now.Sub(r.At) <= applyEvery && !f.applyFailingOpen[r.Node]) {
continue
}
failed[r.Node] = append(failed[r.Node], r)
@@ -62,6 +68,17 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
return out
}
// applyFailingOpen are the machines whose S21 condition is open among the open conditions.
func applyFailingOpen(open []conditions.Condition) map[string]bool {
out := map[string]bool{}
for _, c := range open {
if c.Kind == kindApplyFailing && c.Subject.Machine != "" {
out[c.Subject.Machine] = true
}
}
return out
}
// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
// place.
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
+37
View File
@@ -7,6 +7,7 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
@@ -72,3 +73,39 @@ func TestAFailedPlacingRaisesApplyFailing(t *testing.T) {
t.Fatalf("a failed placing is not raised as anchor's apply-failing: %+v", got)
}
}
// Sweep 2026-10-30 (issue 774): a failure said again in new words carries a new time, and the condition it raised stays
// raised while it stands; only a machine with no open condition waits out the bound, and a cleared failure clears it.
func TestApplyFailingStaysRaisedWhileAnyFailureStands(t *testing.T) {
now := time.Date(2026, 10, 30, 16, 57, 0, 0, time.UTC)
since := now.Add(-time.Minute)
for name, c := range map[string]struct {
open bool
prov string
doing bool
raised bool
}{
"a fresh placing failure, nothing open": {false, inventory.ProvFailed, false, false},
"a fresh placing failure, condition open": {true, inventory.ProvFailed, false, true},
"placing applied again, condition open": {true, inventory.ProvApplied, false, false},
"a fresh failed apply, condition open": {true, "", true, true},
"a fresh failed apply, nothing open": {false, "", true, false},
} {
f := calm(now)
if c.open {
f.applyFailingOpen = applyFailingOpen([]conditions.Condition{{Kind: kindApplyFailing,
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}}})
}
if c.prov != "" {
f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "postgres", Status: c.prov,
Reason: "the vault refused it", At: now.Add(-time.Minute)}}
}
if c.doing {
f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Refused: "no", At: since,
Since: &since}}
}
if got := watchApplyFailing(f); (len(got) == 1) != c.raised || len(got) > 1 {
t.Errorf("%s: %d observation(s), raised want %v", name, len(got), c.raised)
}
}
}
+4
View File
@@ -58,6 +58,9 @@ type signalFacts struct {
// provFailed is every module whose files a machine's provisioner client last said it failed to place (S21,
// novox/hq ADR 0362 §3): that machine's failed apply.
provFailed []inventory.ProvReport
// applyFailingOpen are the machines whose S21 condition is open: kept raised while any failure stands, whatever
// the newest report's time (sweep 2026-10-30).
applyFailingOpen map[string]bool
loop loopFacts
loopErr error
@@ -317,6 +320,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
} else {
f.standings = providerStandings(open)
f.providerWords = providerConditions(open)
f.applyFailingOpen = applyFailingOpen(open)
}
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)