From d63eb83ed3111781d1334fa56106db6b587eba8e Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 18:10:29 +0100 Subject: [PATCH] Clear a condition whose check no longer exists, so a removed row or probe leaves no alarm open for good (issue 774) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit About 35 conditions from 10-20 to 10-27 still showed as current. Read on the bus at 18:06: S23's three failed plans (last observed 10-24) and D14's four stalled deliveries (last observed 10-24) come from a signals row and a probe this controller no longer has; nothing reconciles their source, so nothing clears them. The other old ones are observed every tick and are not stale by the controller: S9's dead letters are still held, S15's hand acts are still kept, and DM's 26 stuck keys are the watchman's (it never writes a key whose watched key aged out as moved) — routed to mesh-delivery. Each watchdogs tick now clears every open condition whose source has a row's or a probe's form (S12, D4, DW, D-root) and is in neither the signals table nor the probe registry, saying the check was removed. Sources of other forms are left to their own word. Quick fix: clear the seven by hand. Thorough fix: the controller judges its own retired checks on every tick, so the next removed row or probe clears what it raised. Chose thorough: rows and probes are being moved out of the controller one by one (ADR 0564), each removal would leave this again. Research 072 piece: milestone 3 (health leaves the controller): the conditions keeper drops what no check of its own observes, so the move to mesh-conditions starts from a bucket of live conditions. --- cmd/mesh-controller/retired_sources.go | 54 +++++++++++++++++++++ cmd/mesh-controller/retired_sources_test.go | 31 ++++++++++++ cmd/mesh-controller/watchdogs.go | 4 ++ 3 files changed, 89 insertions(+) create mode 100644 cmd/mesh-controller/retired_sources.go create mode 100644 cmd/mesh-controller/retired_sources_test.go diff --git a/cmd/mesh-controller/retired_sources.go b/cmd/mesh-controller/retired_sources.go new file mode 100644 index 00000000..174da63d --- /dev/null +++ b/cmd/mesh-controller/retired_sources.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "fmt" + "regexp" + + "github.com/novox/mesh-controller/internal/conditions" +) + +// **A condition whose check no longer exists is cleared** (sweep 2026-10-30, issue 774). A signals row or a self-check +// probe raises and clears its conditions by reconciling what it sees; one removed from the controller reconciles +// nothing again, so what it raised stayed open for good and read as current: S23's failed plans (the plans went with +// ADR 0539) and D14's stalled deliveries, from 2026-10-20 on. Each watchdogs tick clears every open condition whose +// source is a row's or a probe's name that neither the signals table nor the probe registry holds any more, saying so. +// A source of another form (an event, a module's health, a verb) is never judged here: its own word clears it. + +// checkSource is the form of a row's (S1) or a probe's (D4, DW, D-root) name. +var checkSource = regexp.MustCompile(`^(S[0-9]+|D[0-9]+|D[A-Z]+|D-[a-z]+)$`) + +// liveChecks are the sources this controller still reconciles: its signals rows and its self-check's probes. +func liveChecks() map[string]bool { + out := map[string]bool{} + for _, r := range signalsTable { + out[r.Row] = true + } + for _, p := range probeRegistry { + out[p.ID] = true + } + return out +} + +// retiredCheckConditions are the open conditions raised by a row or a probe the controller no longer has. +func retiredCheckConditions(open []conditions.Condition, live map[string]bool) []conditions.Condition { + var out []conditions.Condition + for _, c := range open { + if checkSource.MatchString(c.Source) && !live[c.Source] { + out = append(out, c) + } + } + return out +} + +// clearRetiredChecks clears each of them, saying why. +func clearRetiredChecks(ctx context.Context, k *conditions.Keeper, retired []conditions.Condition) []string { + var problems []string + for _, c := range retired { + why := fmt.Sprintf("%s, which raised it, is no longer a check of the controller", c.Source) + if _, err := k.ClearSaying(ctx, c.Key, why, "No longer checked: the check that raised this was removed"); err != nil { + problems = append(problems, c.Key+": "+err.Error()) + } + } + return problems +} diff --git a/cmd/mesh-controller/retired_sources_test.go b/cmd/mesh-controller/retired_sources_test.go new file mode 100644 index 00000000..9eb7aaed --- /dev/null +++ b/cmd/mesh-controller/retired_sources_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/conditions" +) + +// Sweep 2026-10-30 (issue 774): an open condition raised by a row or a probe the controller no longer has is cleared; +// one raised by a check it still has, or by a source of another form, is left to its own word. +func TestAConditionWhoseCheckIsGoneIsCleared(t *testing.T) { + live := liveChecks() + for source, gone := range map[string]bool{ + "S23": true, + "D14": true, + "S21": false, + "S9": false, + "D-root": false, + "DM": false, + "DB": false, + "D4": false, + "health": false, + "provisioner.failing": false, + "done keys": false, + } { + got := retiredCheckConditions([]conditions.Condition{{Key: "k." + source, Source: source}}, live) + if (len(got) == 1) != gone { + t.Errorf("%s: judged retired %v, want %v", source, len(got) == 1, gone) + } + } +} diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 52b0d038..c80f3a74 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -61,6 +61,8 @@ type signalFacts struct { // applyFailingOpen are the machines whose S21 condition is open: kept raised while any failure stands, whatever // the newest report's time (sweep 2026-10-30). applyFailingOpen map[string]bool + // retiredChecks are the open conditions raised by a row or a probe the controller no longer has (sweep 2026-10-30). + retiredChecks []conditions.Condition loop loopFacts loopErr error @@ -257,6 +259,7 @@ func (w *watchdogs) see(running context.Context, f *signalFacts) { problems = append(problems, "S8: "+err.Error()) } } + problems = append(problems, clearRetiredChecks(running, w.keeper, f.retiredChecks)...) if err := w.keeper.EndSilences(running); err != nil { problems = append(problems, err.Error()) } @@ -321,6 +324,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { f.standings = providerStandings(open) f.providerWords = providerConditions(open) f.applyFailingOpen = applyFailingOpen(open) + f.retiredChecks = retiredCheckConditions(open, liveChecks()) } f.advisories = link.Advisories.Since(now.Add(-advisoryQuiet)) f.lostConsumers, f.advisoriesErr = w.lostConsumers(ctx, f.advisories)