From 3e10ad1535b7570e3505a8c2ca65fd3230788e9f Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 14:49:33 +0200 Subject: [PATCH 1/2] Raise a machine that refuses or only partly applies its declarations past one apply interval, naming the package manager and its words, so the mesh says it rather than only status (issue 233) --- .gitignore | 2 + cmd/mesh-controller/apply_failing.go | 118 ++++++++++++++++++++++ cmd/mesh-controller/apply_failing_test.go | 83 +++++++++++++++ cmd/mesh-controller/plain_words.go | 7 ++ cmd/mesh-controller/signals.go | 9 ++ cmd/mesh-controller/signals_test.go | 8 ++ cmd/mesh-controller/watchdogs.go | 5 + 7 files changed, 232 insertions(+) create mode 100644 cmd/mesh-controller/apply_failing.go create mode 100644 cmd/mesh-controller/apply_failing_test.go diff --git a/.gitignore b/.gitignore index 7f67f6c5..ff196481 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ /mesh-controller /postgres-provisioner /redis-provisioner +# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233). +/cmd/mesh-controller/mesh-controller diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go new file mode 100644 index 00000000..5e9230e9 --- /dev/null +++ b/cmd/mesh-controller/apply_failing.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part, +// for longer than one apply interval (novox/hq issue 233). +const kindApplyFailing = "apply-failing" + +// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its +// ReconcileEvery. A failure still there after one more of them is not one the next apply clears. +const applyEvery = 5 * time.Minute + +// packageManagerKey is how a node-engine names a package manager that does not answer among what failed +// (mesh-host's probe.go): "the package manager (pacman)". +const packageManagerKey = "the package manager (" + +// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only +// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every +// declaration for an hour and a half because its package manager did not answer, and the mesh's only word +// on it was a line in `status`. Raised once its reports have said the same failure for longer than one +// apply interval, over at least two reports — a laptop shut for a week has had one attempt, not a week of +// them — naming the refusal's or the failure's words; cleared by a clean apply. Not while the machine is +// silent or asleep: S1 says silent, and asleep is not lost. +func watchApplyFailing(f *signalFacts) []conditions.Observation { + quiet := map[string]bool{} + for _, m := range f.machines { + if m.asleep() || (!m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every)) { + quiet[m.name] = true + } + } + var out []conditions.Observation + for _, d := range f.doing { + if !d.Wrong() || d.Since == nil || d.Times < 2 || f.now.Sub(*d.Since) <= applyEvery || quiet[d.Node] { + continue + } + out = append(out, applyFailing(d, f.now)) + } + return out +} + +// applyFailing is one machine's S21 condition. +func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { + m := d.Node + since := d.Since.UTC().Format("2006-01-02 15:04 MST") + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, + Severity: conditions.Warning, Said: doingWords(d), + Resolved: m + " applies what it is sent again"} + how := "refused its declaration whole" + if d.Outcome != inventory.OutcomeRefused { + how = "applied its declaration only in part" + } + o.Summary = fmt.Sprintf("%s has %s on %d reports in a row since %s (bound %s): %s", m, how, d.Times, since, + ago(applyEvery), failedNames(d)) + o.Headline = m + " is not applying its update" + o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.", + m, plainHow(d), humanDuration(now.Sub(*d.Since))) + if tool := unansweringTool(d); tool != "" { + o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+ + "is applied, on %d reports in a row since %s (bound %s)", m, tool, d.Times, since, ago(applyEvery)) + o.Headline = fmt.Sprintf("%s cannot install packages", m) + o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+ + "it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+ + "look on the machine.", m, tool, humanDuration(now.Sub(*d.Since))) + } + return o +} + +// plainHow is how a machine failed, as the operator reads it. +func plainHow(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused every update" + } + return "applied its updates only in part" +} + +// unansweringTool is the package manager a machine said does not answer, or "". +func unansweringTool(d inventory.Doing) string { + for _, r := range d.Failed { + if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok { + return strings.TrimSuffix(rest, ")") + } + } + return "" +} + +// failedNames is what failed, by name: the refusal, or the failed resources' ids. +func failedNames(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused" + } + ids := make([]string, 0, len(d.Failed)) + for _, r := range d.Failed { + ids = append(ids, r.ID) + } + sort.Strings(ids) + return "failed " + strings.Join(ids, ", ") +} + +// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh. +func doingWords(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused: " + oneLine(d.Refused) + } + var parts []string + for _, r := range d.Failed { + parts = append(parts, r.ID+": "+oneLine(r.Error)) + } + sort.Strings(parts) + return strings.Join(parts, "; ") +} diff --git a/cmd/mesh-controller/apply_failing_test.go b/cmd/mesh-controller/apply_failing_test.go new file mode 100644 index 00000000..36248b3f --- /dev/null +++ b/cmd/mesh-controller/apply_failing_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its +// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer, +// the same failure on times reports since for before now. +func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing { + since := now.Add(-forHowLong) + return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second), + Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{ + {ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` + + "pacman does not answer here. Either this machine is not Arch, or its package database is broken: " + + "pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"}, + {ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " + + "declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " + + "file /etc/pacman.conf could not be read: No such file or directory"}}} +} + +// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the +// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary. +func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)} + got := watchApplyFailing(f) + if len(got) != 1 { + t.Fatalf("raised %d conditions, want one: %+v", len(got), got) + } + o := got[0] + if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" { + t.Errorf("the condition is not the machine's: %s", o.Key()) + } + if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "18 reports") { + t.Errorf("the summary does not name the tool and how long: %s", o.Summary) + } + if strings.Contains(o.Summary, "/etc/") { + t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary) + } + if !strings.Contains(o.Said, "could not be read") { + t.Errorf("the evidence does not carry the probe's words: %s", o.Said) + } + if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" { + t.Errorf("the plain words do not say it: %+v", o) + } +} + +// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the +// node-engine applied the rest. +func TestARefusalThatGoesOnIsRaised(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + since := now.Add(-time.Hour) + f := calm(now) + f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12, + Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}} + got := watchApplyFailing(f) + if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") || + !strings.Contains(got[0].Said, "pacman does not answer") { + t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got) + } +} + +// **One attempt is not a fault that goes on, and a silent or sleeping machine is S1's**: a laptop shut for a +// week comes back with one report of an old failure; a machine not heard from says silent, not this. +func TestOneReportOrAQuietMachineIsNotRaised(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, 7*24*time.Hour, 1)} + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("one report was raised as a failure that goes on: %+v", got) + } + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 12)} + f.machines[0].lastHeard = now.Add(-time.Hour) + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a silent machine was raised for its last report: %+v", got) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 35846e47..7cd4f3b6 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -81,6 +81,13 @@ var plainWordings = map[string]func(conditions.Observation) words{ "It usually catches up by itself.", m, m), Resolved: m + " applied the change"} }), + kindApplyFailing: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " is not applying its update", + Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+ + "The details say what fails.", conditions.Capital(m)), + Resolved: m + " applies what it is sent again"} + }), "tools-silent": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Tools on " + m + " are not answering", diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 34a6781c..c855e2b7 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -254,6 +254,15 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since }) }}, + {Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)", + Bound: "one apply interval (5 min) past the first report of the same failure, over at least two reports; not " + + "while the machine is silent or asleep; naming the refusal, or the package manager that does not answer " + + "with its words; cleared by a clean apply", + Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2, + needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing, + newest: func(f *signalFacts) time.Time { + return newestOf(f.doing, func(d inventory.Doing) time.Time { return d.At }) + }}, } // watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 9abf6ce8..64c4becc 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -186,6 +186,14 @@ var suppressions = map[string]suppression{ }, past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, }, + // A machine whose package manager does not answer (novox/hq issue 233): said once the same failure has + // outlasted one apply interval over two reports. Inside: a second report, only five minutes after the first. + "S21": { + inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 2)} }, + past: func(f *signalFacts) { + f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 2)} + }, + }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not. "S15": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 58692163..fc3fd58c 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -59,6 +59,10 @@ type signalFacts struct { // from, each naming its sender (S20, novox/hq issue 234). refusedSends []inventory.Send refusedSendsErr error + // doing is every machine whose last report was not a clean apply, with since when it has said the same + // failure (S21, novox/hq issue 233). + doing []inventory.Doing + doingErr error loop loopFacts loopErr error @@ -369,6 +373,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } f.handActs, f.handActsErr = w.gatherHandActs(ctx, now) f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid)) + f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx) f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last() return f } From d4de4543ccbf803ca10d3c706c5640a95b485b42 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 15:39:08 +0200 Subject: [PATCH 2/2] Judge S21 by time since the first failing report while the machine is heard, not by how many reports said it (issue 233 review) --- cmd/mesh-controller/apply_failing.go | 22 ++++----- cmd/mesh-controller/apply_failing_test.go | 58 +++++++++++++++++++---- cmd/mesh-controller/signals.go | 7 +-- cmd/mesh-controller/signals_test.go | 8 ++-- 4 files changed, 68 insertions(+), 27 deletions(-) diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go index 5e9230e9..6daf960d 100644 --- a/cmd/mesh-controller/apply_failing.go +++ b/cmd/mesh-controller/apply_failing.go @@ -25,20 +25,21 @@ const packageManagerKey = "the package manager (" // watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only // to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every // declaration for an hour and a half because its package manager did not answer, and the mesh's only word -// on it was a line in `status`. Raised once its reports have said the same failure for longer than one -// apply interval, over at least two reports — a laptop shut for a week has had one attempt, not a week of -// them — naming the refusal's or the failure's words; cleared by a clean apply. Not while the machine is -// silent or asleep: S1 says silent, and asleep is not lost. +// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply +// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine +// says a re-apply only when how it went changed, so the same failure is one report however long it lasts. +// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while +// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost. func watchApplyFailing(f *signalFacts) []conditions.Observation { - quiet := map[string]bool{} + heard := map[string]bool{} for _, m := range f.machines { - if m.asleep() || (!m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every)) { - quiet[m.name] = true + if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) { + heard[m.name] = true } } var out []conditions.Observation for _, d := range f.doing { - if !d.Wrong() || d.Since == nil || d.Times < 2 || f.now.Sub(*d.Since) <= applyEvery || quiet[d.Node] { + if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] { continue } out = append(out, applyFailing(d, f.now)) @@ -57,14 +58,13 @@ func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { if d.Outcome != inventory.OutcomeRefused { how = "applied its declaration only in part" } - o.Summary = fmt.Sprintf("%s has %s on %d reports in a row since %s (bound %s): %s", m, how, d.Times, since, - ago(applyEvery), failedNames(d)) + o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d)) o.Headline = m + " is not applying its update" o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.", m, plainHow(d), humanDuration(now.Sub(*d.Since))) if tool := unansweringTool(d); tool != "" { o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+ - "is applied, on %d reports in a row since %s (bound %s)", m, tool, d.Times, since, ago(applyEvery)) + "is applied, since %s (bound %s)", m, tool, since, ago(applyEvery)) o.Headline = fmt.Sprintf("%s cannot install packages", m) o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+ "it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+ diff --git a/cmd/mesh-controller/apply_failing_test.go b/cmd/mesh-controller/apply_failing_test.go index 36248b3f..bb76f910 100644 --- a/cmd/mesh-controller/apply_failing_test.go +++ b/cmd/mesh-controller/apply_failing_test.go @@ -1,10 +1,12 @@ package main import ( + "context" "strings" "testing" "time" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) @@ -37,8 +39,8 @@ func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) { if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" { t.Errorf("the condition is not the machine's: %s", o.Key()) } - if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "18 reports") { - t.Errorf("the summary does not name the tool and how long: %s", o.Summary) + if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") { + t.Errorf("the summary does not name the tool and since when: %s", o.Summary) } if strings.Contains(o.Summary, "/etc/") { t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary) @@ -66,18 +68,56 @@ func TestARefusalThatGoesOnIsRaised(t *testing.T) { } } -// **One attempt is not a fault that goes on, and a silent or sleeping machine is S1's**: a laptop shut for a -// week comes back with one report of an old failure; a machine not heard from says silent, not this. -func TestOneReportOrAQuietMachineIsNotRaised(t *testing.T) { +// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233): +// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one +// younger is not yet; and a silent or sleeping machine is S1's. +func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) { now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) f := calm(now) - f.doing = []inventory.Doing{unansweringPackageManager(now, 7*24*time.Hour, 1)} - if got := watchApplyFailing(f); len(got) != 0 { - t.Errorf("one report was raised as a failure that goes on: %+v", got) + f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)} + if got := watchApplyFailing(f); len(got) != 1 { + t.Errorf("one report of a failure six minutes old was not raised: %+v", got) } - f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 12)} + f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)} + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got) + } + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} f.machines[0].lastHeard = now.Add(-time.Hour) if got := watchApplyFailing(f); len(got) != 0 { t.Errorf("a silent machine was raised for its last report: %+v", got) } + f = calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} + f.doing[0].Node = "a-machine-the-mesh-does-not-know" + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a machine not heard of was raised: %+v", got) + } +} + +// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says +// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised +// is cleared through the keeper on the next tick. +func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, + Teller: &conditions.Told{}, Now: func() time.Time { return now }}) + defer k.Close(context.Background()) + w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} + + failing := calm(now) + failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)} + w.see(t.Context(), failing) + open, err := k.Open(t.Context()) + if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing { + t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err) + } + + clean := calm(now.Add(5 * time.Minute)) + clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine + w.see(t.Context(), clean) + if open, _ := k.Open(t.Context()); len(open) != 0 { + t.Fatalf("the condition stayed open after the clean re-apply: %+v", open) + } } diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index c855e2b7..e2cbc387 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -255,9 +255,10 @@ var signalsTable = []signalRow{ return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since }) }}, {Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)", - Bound: "one apply interval (5 min) past the first report of the same failure, over at least two reports; not " + - "while the machine is silent or asleep; naming the refusal, or the package manager that does not answer " + - "with its words; cleared by a clean apply", + Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " + + "is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " + + "asleep; naming the refusal, or the package manager that does not answer with its words; cleared by the " + + "clean apply it then says", Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2, needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing, newest: func(f *signalFacts) time.Time { diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 64c4becc..6398c5ae 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -186,12 +186,12 @@ var suppressions = map[string]suppression{ }, past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, }, - // A machine whose package manager does not answer (novox/hq issue 233): said once the same failure has - // outlasted one apply interval over two reports. Inside: a second report, only five minutes after the first. + // A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the + // failure is older than one apply interval while it is heard. Inside: exactly five minutes old. "S21": { - inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 2)} }, + inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 1)} }, past: func(f *signalFacts) { - f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 2)} + f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 1)} }, }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.