Clear a condition whose check no longer exists, so a removed row or probe leaves no alarm open for good (issue 774)
About 35 conditions from 10-20 to 10-27 still showed as current. Read on the bus at 18:06: S23's three failed plans (last observed 10-24) and D14's four stalled deliveries (last observed 10-24) come from a signals row and a probe this controller no longer has; nothing reconciles their source, so nothing clears them. The other old ones are observed every tick and are not stale by the controller: S9's dead letters are still held, S15's hand acts are still kept, and DM's 26 stuck keys are the watchman's (it never writes a key whose watched key aged out as moved) — routed to mesh-delivery. Each watchdogs tick now clears every open condition whose source has a row's or a probe's form (S12, D4, DW, D-root) and is in neither the signals table nor the probe registry, saying the check was removed. Sources of other forms are left to their own word. Quick fix: clear the seven by hand. Thorough fix: the controller judges its own retired checks on every tick, so the next removed row or probe clears what it raised. Chose thorough: rows and probes are being moved out of the controller one by one (ADR 0564), each removal would leave this again. Research 072 piece: milestone 3 (health leaves the controller): the conditions keeper drops what no check of its own observes, so the move to mesh-conditions starts from a bucket of live conditions.
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A condition whose check no longer exists is cleared** (sweep 2026-10-30, issue 774). A signals row or a self-check
|
||||
// probe raises and clears its conditions by reconciling what it sees; one removed from the controller reconciles
|
||||
// nothing again, so what it raised stayed open for good and read as current: S23's failed plans (the plans went with
|
||||
// ADR 0539) and D14's stalled deliveries, from 2026-10-20 on. Each watchdogs tick clears every open condition whose
|
||||
// source is a row's or a probe's name that neither the signals table nor the probe registry holds any more, saying so.
|
||||
// A source of another form (an event, a module's health, a verb) is never judged here: its own word clears it.
|
||||
|
||||
// checkSource is the form of a row's (S1) or a probe's (D4, DW, D-root) name.
|
||||
var checkSource = regexp.MustCompile(`^(S[0-9]+|D[0-9]+|D[A-Z]+|D-[a-z]+)$`)
|
||||
|
||||
// liveChecks are the sources this controller still reconciles: its signals rows and its self-check's probes.
|
||||
func liveChecks() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range signalsTable {
|
||||
out[r.Row] = true
|
||||
}
|
||||
for _, p := range probeRegistry {
|
||||
out[p.ID] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// retiredCheckConditions are the open conditions raised by a row or a probe the controller no longer has.
|
||||
func retiredCheckConditions(open []conditions.Condition, live map[string]bool) []conditions.Condition {
|
||||
var out []conditions.Condition
|
||||
for _, c := range open {
|
||||
if checkSource.MatchString(c.Source) && !live[c.Source] {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// clearRetiredChecks clears each of them, saying why.
|
||||
func clearRetiredChecks(ctx context.Context, k *conditions.Keeper, retired []conditions.Condition) []string {
|
||||
var problems []string
|
||||
for _, c := range retired {
|
||||
why := fmt.Sprintf("%s, which raised it, is no longer a check of the controller", c.Source)
|
||||
if _, err := k.ClearSaying(ctx, c.Key, why, "No longer checked: the check that raised this was removed"); err != nil {
|
||||
problems = append(problems, c.Key+": "+err.Error())
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// Sweep 2026-10-30 (issue 774): an open condition raised by a row or a probe the controller no longer has is cleared;
|
||||
// one raised by a check it still has, or by a source of another form, is left to its own word.
|
||||
func TestAConditionWhoseCheckIsGoneIsCleared(t *testing.T) {
|
||||
live := liveChecks()
|
||||
for source, gone := range map[string]bool{
|
||||
"S23": true,
|
||||
"D14": true,
|
||||
"S21": false,
|
||||
"S9": false,
|
||||
"D-root": false,
|
||||
"DM": false,
|
||||
"DB": false,
|
||||
"D4": false,
|
||||
"health": false,
|
||||
"provisioner.failing": false,
|
||||
"done keys": false,
|
||||
} {
|
||||
got := retiredCheckConditions([]conditions.Condition{{Key: "k." + source, Source: source}}, live)
|
||||
if (len(got) == 1) != gone {
|
||||
t.Errorf("%s: judged retired %v, want %v", source, len(got) == 1, gone)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,8 @@ type signalFacts struct {
|
||||
// applyFailingOpen are the machines whose S21 condition is open: kept raised while any failure stands, whatever
|
||||
// the newest report's time (sweep 2026-10-30).
|
||||
applyFailingOpen map[string]bool
|
||||
// retiredChecks are the open conditions raised by a row or a probe the controller no longer has (sweep 2026-10-30).
|
||||
retiredChecks []conditions.Condition
|
||||
|
||||
loop loopFacts
|
||||
loopErr error
|
||||
@@ -257,6 +259,7 @@ func (w *watchdogs) see(running context.Context, f *signalFacts) {
|
||||
problems = append(problems, "S8: "+err.Error())
|
||||
}
|
||||
}
|
||||
problems = append(problems, clearRetiredChecks(running, w.keeper, f.retiredChecks)...)
|
||||
if err := w.keeper.EndSilences(running); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
@@ -321,6 +324,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
|
||||
f.standings = providerStandings(open)
|
||||
f.providerWords = providerConditions(open)
|
||||
f.applyFailingOpen = applyFailingOpen(open)
|
||||
f.retiredChecks = retiredCheckConditions(open, liveChecks())
|
||||
}
|
||||
f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet))
|
||||
f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)
|
||||
|
||||
Reference in New Issue
Block a user