Merge pull request 'Raise a machine that refuses or only partly applies its declarations past one apply interval (issue 233)' (#263) from fix/233-a-failed-probe-fails-only-what-needs-the-tool-2 into main

.gitignore

@@ -6,3 +6,5 @@
 /mesh-controller
 /postgres-provisioner
 /redis-provisioner
+# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
+/cmd/mesh-controller/mesh-controller

cmd/mesh-controller/apply_failing.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
+// for longer than one apply interval (novox/hq issue 233).
+const kindApplyFailing = "apply-failing"
+
+// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
+// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
+const applyEvery = 5 * time.Minute
+
+// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
+// (mesh-host's probe.go): "the package manager (pacman)".
+const packageManagerKey = "the package manager ("
+
+// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
+// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
+// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
+// on it was a line in `status`. Raised once the machine's first report of this failure is older than one apply
+// interval **while the machine is heard** — judged by time, not by how many reports said it: a node-engine
+// says a re-apply only when how it went changed, so the same failure is one report however long it lasts.
+// Names the refusal's or the failure's words; cleared by the clean apply the node-engine then says. Not while
+// the machine is silent, asleep or not known: S1 says silent, and asleep is not lost.
+func watchApplyFailing(f *signalFacts) []conditions.Observation {
+	heard := map[string]bool{}
+	for _, m := range f.machines {
+		if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
+			heard[m.name] = true
+		}
+	}
+	var out []conditions.Observation
+	for _, d := range f.doing {
+		if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] {
+			continue
+		}
+		out = append(out, applyFailing(d, f.now))
+	}
+	return out
+}
+
+// applyFailing is one machine's S21 condition.
+func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
+	m := d.Node
+	since := d.Since.UTC().Format("2006-01-02 15:04 MST")
+	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
+		Severity: conditions.Warning, Said: doingWords(d),
+		Resolved: m + " applies what it is sent again"}
+	how := "refused its declaration whole"
+	if d.Outcome != inventory.OutcomeRefused {
+		how = "applied its declaration only in part"
+	}
+	o.Summary = fmt.Sprintf("%s has %s since %s (bound %s): %s", m, how, since, ago(applyEvery), failedNames(d))
+	o.Headline = m + " is not applying its update"
+	o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
+		m, plainHow(d), humanDuration(now.Sub(*d.Since)))
+	if tool := unansweringTool(d); tool != "" {
+		o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
+			"is applied, since %s (bound %s)", m, tool, since, ago(applyEvery))
+		o.Headline = fmt.Sprintf("%s cannot install packages", m)
+		o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
+			"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
+			"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
+	}
+	return o
+}
+
+// plainHow is how a machine failed, as the operator reads it.
+func plainHow(d inventory.Doing) string {
+	if d.Outcome == inventory.OutcomeRefused {
+		return "refused every update"
+	}
+	return "applied its updates only in part"
+}
+
+// unansweringTool is the package manager a machine said does not answer, or "".
+func unansweringTool(d inventory.Doing) string {
+	for _, r := range d.Failed {
+		if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
+			return strings.TrimSuffix(rest, ")")
+		}
+	}
+	return ""
+}
+
+// failedNames is what failed, by name: the refusal, or the failed resources' ids.
+func failedNames(d inventory.Doing) string {
+	if d.Outcome == inventory.OutcomeRefused {
+		return "refused"
+	}
+	ids := make([]string, 0, len(d.Failed))
+	for _, r := range d.Failed {
+		ids = append(ids, r.ID)
+	}
+	sort.Strings(ids)
+	return "failed " + strings.Join(ids, ", ")
+}
+
+// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
+func doingWords(d inventory.Doing) string {
+	if d.Outcome == inventory.OutcomeRefused {
+		return "refused: " + oneLine(d.Refused)
+	}
+	var parts []string
+	for _, r := range d.Failed {
+		parts = append(parts, r.ID+": "+oneLine(r.Error))
+	}
+	sort.Strings(parts)
+	return strings.Join(parts, "; ")
+}

cmd/mesh-controller/apply_failing_test.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its
+// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer,
+// the same failure on times reports since for before now.
+func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing {
+	since := now.Add(-forHowLong)
+	return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second),
+		Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{
+			{ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` +
+				"pacman does not answer here. Either this machine is not Arch, or its package database is broken: " +
+				"pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"},
+			{ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " +
+				"declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " +
+				"file /etc/pacman.conf could not be read: No such file or directory"}}}
+}
+
+// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the
+// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary.
+func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
+	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
+	f := calm(now)
+	f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)}
+	got := watchApplyFailing(f)
+	if len(got) != 1 {
+		t.Fatalf("raised %d conditions, want one: %+v", len(got), got)
+	}
+	o := got[0]
+	if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
+		t.Errorf("the condition is not the machine's: %s", o.Key())
+	}
+	if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "since 2026-10-04 12:30") {
+		t.Errorf("the summary does not name the tool and since when: %s", o.Summary)
+	}
+	if strings.Contains(o.Summary, "/etc/") {
+		t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
+	}
+	if !strings.Contains(o.Said, "could not be read") {
+		t.Errorf("the evidence does not carry the probe's words: %s", o.Said)
+	}
+	if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" {
+		t.Errorf("the plain words do not say it: %+v", o)
+	}
+}
+
+// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the
+// node-engine applied the rest.
+func TestARefusalThatGoesOnIsRaised(t *testing.T) {
+	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
+	since := now.Add(-time.Hour)
+	f := calm(now)
+	f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12,
+		Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}}
+	got := watchApplyFailing(f)
+	if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") ||
+		!strings.Contains(got[0].Said, "pacman does not answer") {
+		t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got)
+	}
+}
+
+// **Judged by time while the machine is heard, not by how many reports said it** (review of novox/hq issue 233):
+// a node-engine says the same failure once, so one report an apply interval old is a fault that goes on; one
+// younger is not yet; and a silent or sleeping machine is S1's.
+func TestOneReportOlderThanAnApplyIsRaisedAndAQuietMachineIsNot(t *testing.T) {
+	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
+	f := calm(now)
+	f.doing = []inventory.Doing{unansweringPackageManager(now, 6*time.Minute, 1)}
+	if got := watchApplyFailing(f); len(got) != 1 {
+		t.Errorf("one report of a failure six minutes old was not raised: %+v", got)
+	}
+	f.doing = []inventory.Doing{unansweringPackageManager(now, 4*time.Minute, 1)}
+	if got := watchApplyFailing(f); len(got) != 0 {
+		t.Errorf("a failure four minutes old was raised before one apply could clear it: %+v", got)
+	}
+	f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
+	f.machines[0].lastHeard = now.Add(-time.Hour)
+	if got := watchApplyFailing(f); len(got) != 0 {
+		t.Errorf("a silent machine was raised for its last report: %+v", got)
+	}
+	f = calm(now)
+	f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 1)}
+	f.doing[0].Node = "a-machine-the-mesh-does-not-know"
+	if got := watchApplyFailing(f); len(got) != 0 {
+		t.Errorf("a machine not heard of was raised: %+v", got)
+	}
+}
+
+// **A failed re-apply followed by a clean one clears S21** (review of novox/hq issue 233): the node-engine says
+// the clean re-apply, the store keeps it as applied with no failure since, and the condition the failure raised
+// is cleared through the keeper on the next tick.
+func TestACleanReapplyAfterAFailedOneClearsTheCondition(t *testing.T) {
+	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
+	store := conditions.NewInMemory()
+	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
+		Teller: &conditions.Told{}, Now: func() time.Time { return now }})
+	defer k.Close(context.Background())
+	w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
+
+	failing := calm(now)
+	failing.doing = []inventory.Doing{unansweringPackageManager(now, 20*time.Minute, 1)}
+	w.see(t.Context(), failing)
+	open, err := k.Open(t.Context())
+	if err != nil || len(open) != 1 || open[0].Kind != kindApplyFailing {
+		t.Fatalf("the failed re-apply raised %+v (%v), want apply-failing", open, err)
+	}
+
+	clean := calm(now.Add(5 * time.Minute))
+	clean.doing = nil // the clean re-apply recorded: NotDoingWhatTheyWereTold no longer lists the machine
+	w.see(t.Context(), clean)
+	if open, _ := k.Open(t.Context()); len(open) != 0 {
+		t.Fatalf("the condition stayed open after the clean re-apply: %+v", open)
+	}
+}

cmd/mesh-controller/plain_words.go

@@ -81,6 +81,13 @@ var plainWordings = map[string]func(conditions.Observation) words{
 				"It usually catches up by itself.", m, m),
 			Resolved: m + " applied the change"}
 	}),
+	kindApplyFailing: worded(func(o conditions.Observation) words {
+		m := machineOr(o, "a machine")
+		return words{Headline: m + " is not applying its update",
+			Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+
+				"The details say what fails.", conditions.Capital(m)),
+			Resolved: m + " applies what it is sent again"}
+	}),
 	"tools-silent": worded(func(o conditions.Observation) words {
 		m := machineOr(o, "a machine")
 		return words{Headline: "Tools on " + m + " are not answering",

cmd/mesh-controller/signals.go

@@ -254,6 +254,16 @@ var signalsTable = []signalRow{
 		newest: func(f *signalFacts) time.Time {
 			return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since })
 		}},
+	{Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)",
+		Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " +
+			"is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " +
+			"asleep; naming the refusal, or the package manager that does not answer with its words; cleared by the " +
+			"clean apply it then says",
+		Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2,
+		needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing,
+		newest: func(f *signalFacts) time.Time {
+			return newestOf(f.doing, func(d inventory.Doing) time.Time { return d.At })
+		}},
 }
 
 // watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since

cmd/mesh-controller/signals_test.go

@@ -186,6 +186,14 @@ var suppressions = map[string]suppression{
 		},
 		past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} },
 	},
+	// A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the
+	// failure is older than one apply interval while it is heard. Inside: exactly five minutes old.
+	"S21": {
+		inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 1)} },
+		past: func(f *signalFacts) {
+			f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 1)}
+		},
+	},
 	// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
 	"S15": {
 		inside: func(f *signalFacts) {

cmd/mesh-controller/watchdogs.go

@@ -59,6 +59,10 @@ type signalFacts struct {
 	// from, each naming its sender (S20, novox/hq issue 234).
 	refusedSends    []inventory.Send
 	refusedSendsErr error
+	// doing is every machine whose last report was not a clean apply, with since when it has said the same
+	// failure (S21, novox/hq issue 233).
+	doing    []inventory.Doing
+	doingErr error
 
 	loop    loopFacts
 	loopErr error
@@ -369,6 +373,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 	}
 	f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
 	f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
+	f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx)
 	f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
 	return f
 }