diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go index 5e9230e9..6daf960d 100644 --- a/cmd/mesh-controller/apply_failing.go +++ b/cmd/mesh-controller/apply_failing.go @@ -25,20 +25,21 @@ const packageManagerKey = "the package manager (" // watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only // to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every // declaration for an hour and a half because its package manager did not answer, and the mesh's only word -// on it was a line in `status`. Raised once its reports have said the same failure for longer than one -// apply interval, over at least two reports — a laptop shut for a week has had one attempt, not a week of -// them — naming the refusal's or the failure's words; cleared by a clean apply. Not while the machine is -// silent or asleep: S1 says silent, and asleep is not lost. +// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply +// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine +// says a re-apply only when how it went changed, so the same failure is one report however long it lasts. +// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while +// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost. func watchApplyFailing(f *signalFacts) []conditions.Observation { - quiet := map[string]bool{} + heard := map[string]bool{} for _, m := range f.machines { - if m.asleep() || (!m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every)) { - quiet[m.name] = true + if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) { + heard[m.name] = true } } var out []conditions.Observation for _, d := range f.doing { - if !d.Wrong() || d.Since == nil || d.Times < 2 || f.now.Sub(*d.Since) <= applyEvery || quiet[d.Node] { + if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] { continue } out = append(out, applyFailing(d, f.now)) @@ -57,14 +58,13 @@ func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { if d.Outcome != inventory.OutcomeRefused { how = "applied its declaration only in part" } - o.Summary = fmt.Sprintf("%s has %s on %d reports in a row since %s (bound %s): %s", m, how, d.Times, since, - ago(applyEvery), failedNames(d)) + o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d)) o.Headline = m + " is not applying its update" o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.", m, plainHow(d), humanDuration(now.Sub(*d.Since))) if tool := unansweringTool(d); tool != "" { o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+ - "is applied, on %d reports in a row since %s (bound %s)", m, tool, d.Times, since, ago(applyEvery)) + "is applied, since %s (bound %s)", m, tool, since, ago(applyEvery)) o.Headline = fmt.Sprintf("%s cannot install packages", m) o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+ "it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+ diff --git a/cmd/mesh-controller/apply_failing_test.go b/cmd/mesh-controller/apply_failing_test.go index 36248b3f..bb76f910 100644 --- a/cmd/mesh-controller/apply_failing_test.go +++ b/cmd/mesh-controller/apply_failing_test.go @@ -1,10 +1,12 @@ package main import ( + "context" "strings" "testing" "time" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) @@ -37,8 +39,8 @@ func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) { if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" { t.Errorf("the condition is not the machine's: %s", o.Key()) } - if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "18 reports") { - t.Errorf("the summary does not name the tool and how long: %s", o.Summary) + if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") { + t.Errorf("the summary does not name the tool and since when: %s", o.Summary) } if strings.Contains(o.Summary, "/etc/") { t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary) @@ -66,18 +68,56 @@ func TestARefusalThatGoesOnIsRaised(t *testing.T) { } } -// **One attempt is not a fault that goes on, and a silent or sleeping machine is S1's**: a laptop shut for a -// week comes back with one report of an old failure; a machine not heard from says silent, not this. -func TestOneReportOrAQuietMachineIsNotRaised(t *testing.T) { +// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233): +// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one +// younger is not yet; and a silent or sleeping machine is S1's. +func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) { now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) f := calm(now) - f.doing = []inventory.Doing{unansweringPackageManager(now, 7*24*time.Hour, 1)} - if got := watchApplyFailing(f); len(got) != 0 { - t.Errorf("one report was raised as a failure that goes on: %+v", got) + f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)} + if got := watchApplyFailing(f); len(got) != 1 { + t.Errorf("one report of a failure six minutes old was not raised: %+v", got) } - f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 12)} + f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)} + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got) + } + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} f.machines[0].lastHeard = now.Add(-time.Hour) if got := watchApplyFailing(f); len(got) != 0 { t.Errorf("a silent machine was raised for its last report: %+v", got) } + f = calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} + f.doing[0].Node = "a-machine-the-mesh-does-not-know" + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a machine not heard of was raised: %+v", got) + } +} + +// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says +// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised +// is cleared through the keeper on the next tick. +func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, + Teller: &conditions.Told{}, Now: func() time.Time { return now }}) + defer k.Close(context.Background()) + w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} + + failing := calm(now) + failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)} + w.see(t.Context(), failing) + open, err := k.Open(t.Context()) + if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing { + t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err) + } + + clean := calm(now.Add(5 * time.Minute)) + clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine + w.see(t.Context(), clean) + if open, _ := k.Open(t.Context()); len(open) != 0 { + t.Fatalf("the condition stayed open after the clean re-apply: %+v", open) + } } diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index c855e2b7..e2cbc387 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -255,9 +255,10 @@ var signalsTable = []signalRow{ return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since }) }}, {Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)", - Bound: "one apply interval (5 min) past the first report of the same failure, over at least two reports; not " + - "while the machine is silent or asleep; naming the refusal, or the package manager that does not answer " + - "with its words; cleared by a clean apply", + Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " + + "is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " + + "asleep; naming the refusal, or the package manager that does not answer with its words; cleared by the " + + "clean apply it then says", Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2, needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing, newest: func(f *signalFacts) time.Time { diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 64c4becc..6398c5ae 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -186,12 +186,12 @@ var suppressions = map[string]suppression{ }, past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, }, - // A machine whose package manager does not answer (novox/hq issue 233): said once the same failure has - // outlasted one apply interval over two reports. Inside: a second report, only five minutes after the first. + // A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the + // failure is older than one apply interval while it is heard. Inside: exactly five minutes old. "S21": { - inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 2)} }, + inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 1)} }, past: func(f *signalFacts) { - f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 2)} + f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 1)} }, }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.