Raise a machine that refuses or only partly applies its declarations past one apply interval, naming the package manager and its words, so the mesh says it rather than only status (issue 233)

cmd/mesh-controller/apply_failing.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
+// for longer than one apply interval (novox/hq issue 233).
+const kindApplyFailing = "apply-failing"
+
+// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
+// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
+const applyEvery = 5 * time.Minute
+
+// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
+// (mesh-host's probe.go): "the package manager (pacman)".
+const packageManagerKey = "the package manager ("
+
+// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
+// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
+// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
+// on it was a line in `status`. Raised once its reports have said the same failure for longer than one
+// apply interval, over at least two reports — a laptop shut for a week has had one attempt, not a week of
+// them — naming the refusal's or the failure's words; cleared by a clean apply. Not while the machine is
+// silent or asleep: S1 says silent, and asleep is not lost.
+func watchApplyFailing(f *signalFacts) []conditions.Observation {
+	quiet := map[string]bool{}
+	for _, m := range f.machines {
+		if m.asleep() || (!m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every)) {
+			quiet[m.name] = true
+		}
+	}
+	var out []conditions.Observation
+	for _, d := range f.doing {
+		if !d.Wrong() || d.Since == nil || d.Times < 2 || f.now.Sub(*d.Since) <= applyEvery || quiet[d.Node] {
+			continue
+		}
+		out = append(out, applyFailing(d, f.now))
+	}
+	return out
+}
+
+// applyFailing is one machine's S21 condition.
+func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
+	m := d.Node
+	since := d.Since.UTC().Format("2006-01-02 15:04 MST")
+	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
+		Severity: conditions.Warning, Said: doingWords(d),
+		Resolved: m + " applies what it is sent again"}
+	how := "refused its declaration whole"
+	if d.Outcome != inventory.OutcomeRefused {
+		how = "applied its declaration only in part"
+	}
+	o.Summary = fmt.Sprintf("%s has %s on %d reports in a row since %s (bound %s): %s", m, how, d.Times, since,
+		ago(applyEvery), failedNames(d))
+	o.Headline = m + " is not applying its update"
+	o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
+		m, plainHow(d), humanDuration(now.Sub(*d.Since)))
+	if tool := unansweringTool(d); tool != "" {
+		o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
+			"is applied, on %d reports in a row since %s (bound %s)", m, tool, d.Times, since, ago(applyEvery))
+		o.Headline = fmt.Sprintf("%s cannot install packages", m)
+		o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
+			"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
+			"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
+	}
+	return o
+}
+
+// plainHow is how a machine failed, as the operator reads it.
+func plainHow(d inventory.Doing) string {
+	if d.Outcome == inventory.OutcomeRefused {
+		return "refused every update"
+	}
+	return "applied its updates only in part"
+}
+
+// unansweringTool is the package manager a machine said does not answer, or "".
+func unansweringTool(d inventory.Doing) string {
+	for _, r := range d.Failed {
+		if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
+			return strings.TrimSuffix(rest, ")")
+		}
+	}
+	return ""
+}
+
+// failedNames is what failed, by name: the refusal, or the failed resources' ids.
+func failedNames(d inventory.Doing) string {
+	if d.Outcome == inventory.OutcomeRefused {
+		return "refused"
+	}
+	ids := make([]string, 0, len(d.Failed))
+	for _, r := range d.Failed {
+		ids = append(ids, r.ID)
+	}
+	sort.Strings(ids)
+	return "failed " + strings.Join(ids, ", ")
+}
+
+// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
+func doingWords(d inventory.Doing) string {
+	if d.Outcome == inventory.OutcomeRefused {
+		return "refused: " + oneLine(d.Refused)
+	}
+	var parts []string
+	for _, r := range d.Failed {
+		parts = append(parts, r.ID+": "+oneLine(r.Error))
+	}
+	sort.Strings(parts)
+	return strings.Join(parts, "; ")
+}