diff --git a/.gitignore b/.gitignore index 7f67f6c5..ff196481 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ /mesh-controller /postgres-provisioner /redis-provisioner +# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233). +/cmd/mesh-controller/mesh-controller diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go new file mode 100644 index 00000000..6daf960d --- /dev/null +++ b/cmd/mesh-controller/apply_failing.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part, +// for longer than one apply interval (novox/hq issue 233). +const kindApplyFailing = "apply-failing" + +// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its +// ReconcileEvery. A failure still there after one more of them is not one the next apply clears. +const applyEvery = 5 * time.Minute + +// packageManagerKey is how a node-engine names a package manager that does not answer among what failed +// (mesh-host's probe.go): "the package manager (pacman)". +const packageManagerKey = "the package manager (" + +// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only +// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every +// declaration for an hour and a half because its package manager did not answer, and the mesh's only word +// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply +// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine +// says a re-apply only when how it went changed, so the same failure is one report however long it lasts. +// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while +// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost. +func watchApplyFailing(f *signalFacts) []conditions.Observation { + heard := map[string]bool{} + for _, m := range f.machines { + if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) { + heard[m.name] = true + } + } + var out []conditions.Observation + for _, d := range f.doing { + if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] { + continue + } + out = append(out, applyFailing(d, f.now)) + } + return out +} + +// applyFailing is one machine's S21 condition. +func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { + m := d.Node + since := d.Since.UTC().Format("2006-01-02 15:04 MST") + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, + Severity: conditions.Warning, Said: doingWords(d), + Resolved: m + " applies what it is sent again"} + how := "refused its declaration whole" + if d.Outcome != inventory.OutcomeRefused { + how = "applied its declaration only in part" + } + o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d)) + o.Headline = m + " is not applying its update" + o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.", + m, plainHow(d), humanDuration(now.Sub(*d.Since))) + if tool := unansweringTool(d); tool != "" { + o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+ + "is applied, since %s (bound %s)", m, tool, since, ago(applyEvery)) + o.Headline = fmt.Sprintf("%s cannot install packages", m) + o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+ + "it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+ + "look on the machine.", m, tool, humanDuration(now.Sub(*d.Since))) + } + return o +} + +// plainHow is how a machine failed, as the operator reads it. +func plainHow(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused every update" + } + return "applied its updates only in part" +} + +// unansweringTool is the package manager a machine said does not answer, or "". +func unansweringTool(d inventory.Doing) string { + for _, r := range d.Failed { + if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok { + return strings.TrimSuffix(rest, ")") + } + } + return "" +} + +// failedNames is what failed, by name: the refusal, or the failed resources' ids. +func failedNames(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused" + } + ids := make([]string, 0, len(d.Failed)) + for _, r := range d.Failed { + ids = append(ids, r.ID) + } + sort.Strings(ids) + return "failed " + strings.Join(ids, ", ") +} + +// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh. +func doingWords(d inventory.Doing) string { + if d.Outcome == inventory.OutcomeRefused { + return "refused: " + oneLine(d.Refused) + } + var parts []string + for _, r := range d.Failed { + parts = append(parts, r.ID+": "+oneLine(r.Error)) + } + sort.Strings(parts) + return strings.Join(parts, "; ") +} diff --git a/cmd/mesh-controller/apply_failing_test.go b/cmd/mesh-controller/apply_failing_test.go new file mode 100644 index 00000000..bb76f910 --- /dev/null +++ b/cmd/mesh-controller/apply_failing_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its +// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer, +// the same failure on times reports since for before now. +func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing { + since := now.Add(-forHowLong) + return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second), + Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{ + {ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` + + "pacman does not answer here. Either this machine is not Arch, or its package database is broken: " + + "pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"}, + {ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " + + "declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " + + "file /etc/pacman.conf could not be read: No such file or directory"}}} +} + +// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the +// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary. +func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)} + got := watchApplyFailing(f) + if len(got) != 1 { + t.Fatalf("raised %d conditions, want one: %+v", len(got), got) + } + o := got[0] + if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" { + t.Errorf("the condition is not the machine's: %s", o.Key()) + } + if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") { + t.Errorf("the summary does not name the tool and since when: %s", o.Summary) + } + if strings.Contains(o.Summary, "/etc/") { + t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary) + } + if !strings.Contains(o.Said, "could not be read") { + t.Errorf("the evidence does not carry the probe's words: %s", o.Said) + } + if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" { + t.Errorf("the plain words do not say it: %+v", o) + } +} + +// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the +// node-engine applied the rest. +func TestARefusalThatGoesOnIsRaised(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + since := now.Add(-time.Hour) + f := calm(now) + f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12, + Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}} + got := watchApplyFailing(f) + if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") || + !strings.Contains(got[0].Said, "pacman does not answer") { + t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got) + } +} + +// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233): +// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one +// younger is not yet; and a silent or sleeping machine is S1's. +func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)} + if got := watchApplyFailing(f); len(got) != 1 { + t.Errorf("one report of a failure six minutes old was not raised: %+v", got) + } + f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)} + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got) + } + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} + f.machines[0].lastHeard = now.Add(-time.Hour) + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a silent machine was raised for its last report: %+v", got) + } + f = calm(now) + f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)} + f.doing[0].Node = "a-machine-the-mesh-does-not-know" + if got := watchApplyFailing(f); len(got) != 0 { + t.Errorf("a machine not heard of was raised: %+v", got) + } +} + +// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says +// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised +// is cleared through the keeper on the next tick. +func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, + Teller: &conditions.Told{}, Now: func() time.Time { return now }}) + defer k.Close(context.Background()) + w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} + + failing := calm(now) + failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)} + w.see(t.Context(), failing) + open, err := k.Open(t.Context()) + if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing { + t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err) + } + + clean := calm(now.Add(5 * time.Minute)) + clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine + w.see(t.Context(), clean) + if open, _ := k.Open(t.Context()); len(open) != 0 { + t.Fatalf("the condition stayed open after the clean re-apply: %+v", open) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 35846e47..7cd4f3b6 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -81,6 +81,13 @@ var plainWordings = map[string]func(conditions.Observation) words{ "It usually catches up by itself.", m, m), Resolved: m + " applied the change"} }), + kindApplyFailing: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: m + " is not applying its update", + Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+ + "The details say what fails.", conditions.Capital(m)), + Resolved: m + " applies what it is sent again"} + }), "tools-silent": worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Tools on " + m + " are not answering", diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 34a6781c..e2cbc387 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -254,6 +254,16 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since }) }}, + {Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)", + Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " + + "is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " + + "asleep; naming the refusal, or the package manager that does not answer with its words; cleared by the " + + "clean apply it then says", + Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2, + needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing, + newest: func(f *signalFacts) time.Time { + return newestOf(f.doing, func(d inventory.Doing) time.Time { return d.At }) + }}, } // watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 9abf6ce8..6398c5ae 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -186,6 +186,14 @@ var suppressions = map[string]suppression{ }, past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, }, + // A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the + // failure is older than one apply interval while it is heard. Inside: exactly five minutes old. + "S21": { + inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 1)} }, + past: func(f *signalFacts) { + f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 1)} + }, + }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not. "S15": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 58692163..fc3fd58c 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -59,6 +59,10 @@ type signalFacts struct { // from, each naming its sender (S20, novox/hq issue 234). refusedSends []inventory.Send refusedSendsErr error + // doing is every machine whose last report was not a clean apply, with since when it has said the same + // failure (S21, novox/hq issue 233). + doing []inventory.Doing + doingErr error loop loopFacts loopErr error @@ -369,6 +373,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } f.handActs, f.handActsErr = w.gatherHandActs(ctx, now) f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid)) + f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx) f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last() return f }