diff --git a/.gitignore b/.gitignore index 7f67f6c5..ff196481 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ /mesh-controller /postgres-provisioner /redis-provisioner +# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233). +/cmd/mesh-controller/mesh-controller diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go new file mode 100644 index 00000000..5e9230e9 --- /dev/null +++ b/cmd/mesh-controller/apply_failing.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part, +// for longer than one apply interval (novox/hq issue 233). +const kindApplyFailing = "apply-failing" + +// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its +// ReconcileEvery. A failure still there after one more of them is not one the next apply clears. +const applyEvery = 5 * time.Minute + +// packageManagerKey is how a node-engine names a package manager that does not answer among what failed +// (mesh-host's probe.go): "the package manager (pacman)". +const packageManagerKey = "the package manager (" + +// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only +// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every +// declaration for an hour and a half because its package manager did not answer, and the mesh's only word +// on it was a line in `status`. Raised once its reports have said the same failure for longer than one +// apply interval, over at least two reports — a laptop shut for a week has had one attempt, not a week of +// them — naming the refusal's or the failure's words; cleared by a clean apply. Not while the machine is +// silent or asleep: S1 says silent, and asleep is not lost. +func watchApplyFailing(f *signalFacts) []conditions.Observation { + quiet := map[string]bool{} + for _, m := range f.machines { + if m.asleep() || (!m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every)) { + quiet[m.name] = true + } + } + var out []conditions.Observation + for _, d := range f.doing { + if !d.Wrong() || d.Since == nil || d.Times < 2 || f.now.Sub(*d.Since) <= applyEvery || quiet[d.Node] { + continue + } + out = append(out, applyFailing(d, f.now)) + } + return out +} + +// applyFailing is one machine's S21 condition. +func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { + m := d.Node + since := d.Since.UTC().Format("2006-01-02 15:04 MST") + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, + Severity: conditions.Warning, Said: doingWords(d), + Resolved: m + " applies what it is sent again"} + how := "refused its declaration whole" + if d.Outcome != inventory.OutcomeRefused { + how = "applied its declaration only in part" + } + o.Summary = fmt.Sprintf("%s has %s on %d reports in a row since %s (bound %s): %s", m, how, d.Times, since, + ago(applyEvery), failedNames(d)) + o.Headline = m + " is not applying its update" + o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.", + m, plainHow(d), humanDuration(now.Sub(*d.Since))) + if tool := unansweringTool(d); tool != "" { + o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+ + "is applied, on %d reports in a row since %s (bound %s)", m, tool, d.Times, since, ago(applyEvery)) + o.Headline = fmt.Sprintf("%s cannot install packages", m) + o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+ + "it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+ + "look on the machine.", m, tool, humanDuration(now.Sub(*d.Since))) + } + return o +} + +// plainHow is how a machine failed, as the operator reads it. +func plainHow(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused every update" + } + return "applied its updates only in part" +} + +// unansweringTool is the package manager a machine said does not answer, or "". +func unansweringTool(d inventory.Doing) string { + for _, r := range d.Failed { + if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok { + return strings.TrimSuffix(rest, ")") + } + } + return "" +} + +// failedNames is what failed, by name: the refusal, or the failed resources' ids. +func failedNames(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused" + } + ids := make([]string, 0, len(d.Failed)) + for _, r := range d.Failed { + ids = append(ids, r.ID) + } + sort.Strings(ids) + return "failed " + strings.Join(ids, ", ") +} + +// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh. +func doingWords(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused: " + oneLine(d.Refused) + } + var parts []string + for _, r := range d.Failed { + parts = append(parts, r.ID+": "+oneLine(r.Error)) + } + sort.Strings(parts) + return strings.Join(parts, "; ") +} diff --git a/cmd/mesh-controller/apply_failing_test.go b/cmd/mesh-controller/apply_failing_test.go new file mode 100644 index 00000000..36248b3f --- /dev/null +++ b/cmd/mesh-controller/apply_failing_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its +// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer, +// the same failure on times reports since for before now. +func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing { + since := now.Add(-forHowLong) + return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second), + Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{ + {ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` + + "pacman does not answer here. Either this machine is not Arch, or its package database is broken: " + + "pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"}, + {ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " + + "declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " + + "file /etc/pacman.conf could not be read: No such file or directory"}}} +} + +// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the +// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary. +func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)} + got := watchApplyFailing(f) + if len(got) != 1 { + t.Fatalf("raised %d conditions, want one: %+v", len(got), got) + } + o := got[0] + if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" { + t.Errorf("the condition is not the machine's: %s", o.Key()) + } + if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "18 reports") { + t.Errorf("the summary does not name the tool and how long: %s", o.Summary) + } + if strings.Contains(o.Summary, "/etc/") { + t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary) + } + if !strings.Contains(o.Said, "could not be read") { + t.Errorf("the evidence does not carry the probe's words: %s", o.Said) + } + if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" { + t.Errorf("the plain words do not say it: %+v", o) + } +} + +// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the +// node-engine applied the rest. +func TestARefusalThatGoesOnIsRaised(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + since := now.Add(-time.Hour) + f := calm(now) + f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12, + Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}} + got := watchApplyFailing(f) + if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") || + !strings.Contains(got[0].Said, "pacman does not answer") { + t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got) + } +} + +// **One attempt is not a fault that goes on, and a silent or sleeping machine is S1's**: a laptop shut for a +// week comes back with one report of an old failure; a machine not heard from says silent, not this. +func TestOneReportOrAQuietMachineIsNotRaised(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, 7*24*time.Hour, 1)} + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("one report was raised as a failure that goes on: %+v", got) + } + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 12)} + f.machines[0].lastHeard = now.Add(-time.Hour) + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a silent machine was raised for its last report: %+v", got) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 35846e47..7cd4f3b6 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -81,6 +81,13 @@ var plainWordings = map[string]func(conditions.Observation) words{ "It usually catches up by itself.", m, m), Resolved: m + " applied the change"} }), + kindApplyFailing: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " is not applying its update", + Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+ + "The details say what fails.", conditions.Capital(m)), + Resolved: m + " applies what it is sent again"} + }), "tools-silent": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Tools on " + m + " are not answering", diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 34a6781c..c855e2b7 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -254,6 +254,15 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since }) }}, + {Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)", + Bound: "one apply interval (5 min) past the first report of the same failure, over at least two reports; not " + + "while the machine is silent or asleep; naming the refusal, or the package manager that does not answer " + + "with its words; cleared by a clean apply", + Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2, + needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing, + newest: func(f *signalFacts) time.Time { + return newestOf(f.doing, func(d inventory.Doing) time.Time { return d.At }) + }}, } // watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 9abf6ce8..64c4becc 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -186,6 +186,14 @@ var suppressions = map[string]suppression{ }, past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, }, + // A machine whose package manager does not answer (novox/hq issue 233): said once the same failure has + // outlasted one apply interval over two reports. Inside: a second report, only five minutes after the first. + "S21": { + inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 2)} }, + past: func(f *signalFacts) { + f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 2)} + }, + }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not. "S15": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 58692163..fc3fd58c 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -59,6 +59,10 @@ type signalFacts struct { // from, each naming its sender (S20, novox/hq issue 234). refusedSends []inventory.Send refusedSendsErr error + // doing is every machine whose last report was not a clean apply, with since when it has said the same + // failure (S21, novox/hq issue 233). + doing []inventory.Doing + doingErr error loop loopFacts loopErr error @@ -369,6 +373,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } f.handActs, f.handActsErr = w.gatherHandActs(ctx, now) f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid)) + f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx) f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last() return f }