Archived
Remove the leftover merges consumer behind the false deaf alarm; apply-failing stays raised while a failure stands (issue 774) #484
@@ -30,6 +30,11 @@ const packageManagerKey = "the package manager ("
|
||||
// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
|
||||
// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
|
||||
// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
|
||||
//
|
||||
// **The bound gates raising, never keeping** (sweep 2026-10-30, issue 774). A report says a failure again whenever its
|
||||
// words change, with a new time: on novox the vault's refusal of every secret replaced a bus refusal at 16:56, and the
|
||||
// condition cleared for five minutes while every module still failed. Once raised, it stays raised while any failure
|
||||
// stands on that machine, judged by the open condition the keeper holds on the bus, not by a report's time.
|
||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
heard := map[string]bool{}
|
||||
for _, m := range f.machines {
|
||||
@@ -40,7 +45,7 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
said := map[string]bool{}
|
||||
for _, d := range f.doing {
|
||||
if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
|
||||
if !d.Wrong() || d.Since == nil || !heard[d.Node] || (f.now.Sub(*d.Since) <= applyEvery && !f.applyFailingOpen[d.Node]) {
|
||||
continue
|
||||
}
|
||||
out = append(out, applyFailing(d, f.now))
|
||||
@@ -51,7 +56,8 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
// on a machine whose node-engine applied what it was sent.
|
||||
failed := map[string][]inventory.ProvReport{}
|
||||
for _, r := range f.provFailed {
|
||||
if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] {
|
||||
if r.Status != inventory.ProvFailed || !heard[r.Node] || said[r.Node] ||
|
||||
(f.now.Sub(r.At) <= applyEvery && !f.applyFailingOpen[r.Node]) {
|
||||
continue
|
||||
}
|
||||
failed[r.Node] = append(failed[r.Node], r)
|
||||
@@ -62,6 +68,17 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
return out
|
||||
}
|
||||
|
||||
// applyFailingOpen are the machines whose S21 condition is open among the open conditions.
|
||||
func applyFailingOpen(open []conditions.Condition) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, c := range open {
|
||||
if c.Kind == kindApplyFailing && c.Subject.Machine != "" {
|
||||
out[c.Subject.Machine] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to
|
||||
// place.
|
||||
func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
@@ -72,3 +73,39 @@ func TestAFailedPlacingRaisesApplyFailing(t *testing.T) {
|
||||
t.Fatalf("a failed placing is not raised as anchor's apply-failing: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Sweep 2026-10-30 (issue 774): a failure said again in new words carries a new time, and the condition it raised stays
|
||||
// raised while it stands; only a machine with no open condition waits out the bound, and a cleared failure clears it.
|
||||
func TestApplyFailingStaysRaisedWhileAnyFailureStands(t *testing.T) {
|
||||
now := time.Date(2026, 10, 30, 16, 57, 0, 0, time.UTC)
|
||||
since := now.Add(-time.Minute)
|
||||
for name, c := range map[string]struct {
|
||||
open bool
|
||||
prov string
|
||||
doing bool
|
||||
raised bool
|
||||
}{
|
||||
"a fresh placing failure, nothing open": {false, inventory.ProvFailed, false, false},
|
||||
"a fresh placing failure, condition open": {true, inventory.ProvFailed, false, true},
|
||||
"placing applied again, condition open": {true, inventory.ProvApplied, false, false},
|
||||
"a fresh failed apply, condition open": {true, "", true, true},
|
||||
"a fresh failed apply, nothing open": {false, "", true, false},
|
||||
} {
|
||||
f := calm(now)
|
||||
if c.open {
|
||||
f.applyFailingOpen = applyFailingOpen([]conditions.Condition{{Kind: kindApplyFailing,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}}})
|
||||
}
|
||||
if c.prov != "" {
|
||||
f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "postgres", Status: c.prov,
|
||||
Reason: "the vault refused it", At: now.Add(-time.Minute)}}
|
||||
}
|
||||
if c.doing {
|
||||
f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Refused: "no", At: since,
|
||||
Since: &since}}
|
||||
}
|
||||
if got := watchApplyFailing(f); (len(got) == 1) != c.raised || len(got) > 1 {
|
||||
t.Errorf("%s: %d observation(s), raised want %v", name, len(got), c.raised)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +58,9 @@ type signalFacts struct {
|
||||
// provFailed is every module whose files a machine's provisioner client last said it failed to place (S21,
|
||||
// novox/hq ADR 0362 §3): that machine's failed apply.
|
||||
provFailed []inventory.ProvReport
|
||||
// applyFailingOpen are the machines whose S21 condition is open: kept raised while any failure stands, whatever
|
||||
// the newest report's time (sweep 2026-10-30).
|
||||
applyFailingOpen map[string]bool
|
||||
|
||||
loop loopFacts
|
||||
loopErr error
|
||||
@@ -317,6 +320,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
} else {
|
||||
f.standings = providerStandings(open)
|
||||
f.providerWords = providerConditions(open)
|
||||
f.applyFailingOpen = applyFailingOpen(open)
|
||||
}
|
||||
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||
|
||||
Reference in New Issue
Block a user