Raise a machine that refuses or only partly applies its declarations past one apply interval, naming the package manager and its words, so the mesh says it rather than only status (issue 233)

.gitignore

@@ -6,3 +6,5 @@
 /mesh-controller
 /postgres-provisioner
 /redis-provisioner
+# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
+/cmd/mesh-controller/mesh-controller

cmd/mesh-controller/apply_failing.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// kindApplyFailing is S21's kind: a machine whose declarations have been refused, or applied only in part,
+// for longer than one apply interval (novox/hq issue 233).
+const kindApplyFailing = "apply-failing"
+
+// applyEvery is how often a node-engine applies what it was last told when nothing new arrives: its
+// ReconcileEvery. A failure still there after one more of them is not one the next apply clears.
+const applyEvery = 5 * time.Minute
+
+// packageManagerKey is how a node-engine names a package manager that does not answer among what failed
+// (mesh-host's probe.go): "the package manager (pacman)".
+const packageManagerKey = "the package manager ("
+
+// watchApplyFailing is S21: **a machine that does not apply what it was sent is said by the mesh, not only
+// to someone who asks for `status`** (novox/hq issue 233, issue 187's family). A workstation refused every
+// declaration for an hour and a half because its package manager did not answer, and the mesh's only word
+// on it was a line in `status`. Raised once its reports have said the same failure for longer than one
+// apply interval, over at least two reports — a laptop shut for a week has had one attempt, not a week of
+// them — naming the refusal's or the failure's words; cleared by a clean apply. Not while the machine is
+// silent or asleep: S1 says silent, and asleep is not lost.
+func watchApplyFailing(f *signalFacts) []conditions.Observation {
+	quiet := map[string]bool{}
+	for _, m := range f.machines {
+		if m.asleep() || (!m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every)) {
+			quiet[m.name] = true
+		}
+	}
+	var out []conditions.Observation
+	for _, d := range f.doing {
+		if !d.Wrong() || d.Since == nil || d.Times < 2 || f.now.Sub(*d.Since) <= applyEvery || quiet[d.Node] {
+			continue
+		}
+		out = append(out, applyFailing(d, f.now))
+	}
+	return out
+}
+
+// applyFailing is one machine's S21 condition.
+func applyFailing(d inventory.Doing, now time.Time) conditions.Observation {
+	m := d.Node
+	since := d.Since.UTC().Format("2006-01-02 15:04 MST")
+	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m,
+		Severity: conditions.Warning, Said: doingWords(d),
+		Resolved: m + " applies what it is sent again"}
+	how := "refused its declaration whole"
+	if d.Outcome != inventory.OutcomeRefused {
+		how = "applied its declaration only in part"
+	}
+	o.Summary = fmt.Sprintf("%s has %s on %d reports in a row since %s (bound %s): %s", m, how, d.Times, since,
+		ago(applyEvery), failedNames(d))
+	o.Headline = m + " is not applying its update"
+	o.Explanation = fmt.Sprintf("%s has %s for %s, and trying again has not helped. The details say what fails.",
+		m, plainHow(d), humanDuration(now.Sub(*d.Since)))
+	if tool := unansweringTool(d); tool != "" {
+		o.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent "+
+			"is applied, on %d reports in a row since %s (bound %s)", m, tool, d.Times, since, ago(applyEvery))
+		o.Headline = fmt.Sprintf("%s cannot install packages", m)
+		o.Explanation = fmt.Sprintf("The package manager on %s (%s) has not answered for %s, so no package reaches "+
+			"it. Everything else is still applied, so an update that repairs it can land; if none does, it needs a "+
+			"look on the machine.", m, tool, humanDuration(now.Sub(*d.Since)))
+	}
+	return o
+}
+
+// plainHow is how a machine failed, as the operator reads it.
+func plainHow(d inventory.Doing) string {
+	if d.Outcome == inventory.OutcomeRefused {
+		return "refused every update"
+	}
+	return "applied its updates only in part"
+}
+
+// unansweringTool is the package manager a machine said does not answer, or "".
+func unansweringTool(d inventory.Doing) string {
+	for _, r := range d.Failed {
+		if rest, ok := strings.CutPrefix(r.ID, packageManagerKey); ok {
+			return strings.TrimSuffix(rest, ")")
+		}
+	}
+	return ""
+}
+
+// failedNames is what failed, by name: the refusal, or the failed resources' ids.
+func failedNames(d inventory.Doing) string {
+	if d.Outcome == inventory.OutcomeRefused {
+		return "refused"
+	}
+	ids := make([]string, 0, len(d.Failed))
+	for _, r := range d.Failed {
+		ids = append(ids, r.ID)
+	}
+	sort.Strings(ids)
+	return "failed " + strings.Join(ids, ", ")
+}
+
+// doingWords is the machine's own words on what failed: the evidence, kept inside the mesh.
+func doingWords(d inventory.Doing) string {
+	if d.Outcome == inventory.OutcomeRefused {
+		return "refused: " + oneLine(d.Refused)
+	}
+	var parts []string
+	for _, r := range d.Failed {
+		parts = append(parts, r.ID+": "+oneLine(r.Error))
+	}
+	sort.Strings(parts)
+	return strings.Join(parts, "; ")
+}

cmd/mesh-controller/apply_failing_test.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// unansweringPackageManager is anchor's last report as the workstation's read on 2026-10-04, once its
+// node-engine applies the rest (novox/hq issue 233): the package failed and the package manager did not answer,
+// the same failure on times reports since for before now.
+func unansweringPackageManager(now time.Time, forHowLong time.Duration, times int) inventory.Doing {
+	since := now.Add(-forHowLong)
+	return inventory.Doing{Node: "anchor", Outcome: inventory.OutcomeFailed, At: now.Add(-time.Second),
+		Since: &since, Times: times, Applied: 41, Failed: []inventory.FailedResource{
+			{ID: "apply", Error: `applying "pacman.base": nothing can be said about "base": this is the arch host and ` +
+				"pacman does not answer here. Either this machine is not Arch, or its package database is broken: " +
+				"pacman exited 1: error: config file /etc/pacman.conf could not be read: No such file or directory"},
+			{ID: "the package manager (pacman)", Error: "pacman does not answer here, so every package in this " +
+				"declaration fails and the rest of it is applied (novox/hq issue 233): pacman exited 1: error: config " +
+				"file /etc/pacman.conf could not be read: No such file or directory"}}}
+}
+
+// **A machine whose package manager does not answer is raised by the mesh itself, naming the tool and the
+// probe's words** (novox/hq issue 233): the words in the evidence, the path kept out of the summary.
+func TestAPackageManagerThatDoesNotAnswerIsRaisedNamingItsWords(t *testing.T) {
+	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
+	f := calm(now)
+	f.doing = []inventory.Doing{unansweringPackageManager(now, 90*time.Minute, 18)}
+	got := watchApplyFailing(f)
+	if len(got) != 1 {
+		t.Fatalf("raised %d conditions, want one: %+v", len(got), got)
+	}
+	o := got[0]
+	if o.Key() != "machine.anchor.apply-failing" || o.Machine != "anchor" {
+		t.Errorf("the condition is not the machine's: %s", o.Key())
+	}
+	if !strings.Contains(o.Summary, "pacman does not answer") || !strings.Contains(o.Summary, "18 reports") {
+		t.Errorf("the summary does not name the tool and how long: %s", o.Summary)
+	}
+	if strings.Contains(o.Summary, "/etc/") {
+		t.Errorf("the summary carries a path, which belongs in the evidence: %s", o.Summary)
+	}
+	if !strings.Contains(o.Said, "could not be read") {
+		t.Errorf("the evidence does not carry the probe's words: %s", o.Said)
+	}
+	if !strings.Contains(o.Headline, "cannot install packages") || o.Explanation == "" || o.Resolved == "" {
+		t.Errorf("the plain words do not say it: %+v", o)
+	}
+}
+
+// **A whole refusal is raised too**, with the node-engine's words: what the workstation's report said before the
+// node-engine applied the rest.
+func TestARefusalThatGoesOnIsRaised(t *testing.T) {
+	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
+	since := now.Add(-time.Hour)
+	f := calm(now)
+	f.doing = []inventory.Doing{{Node: "anchor", Outcome: inventory.OutcomeRefused, Since: &since, Times: 12,
+		Refused: "this is the arch host and pacman does not answer here", At: now.Add(-time.Minute)}}
+	got := watchApplyFailing(f)
+	if len(got) != 1 || !strings.Contains(got[0].Summary, "refused its declaration whole") ||
+		!strings.Contains(got[0].Said, "pacman does not answer") {
+		t.Fatalf("a refusal that went on for an hour was not raised with its words: %+v", got)
+	}
+}
+
+// **One attempt is not a fault that goes on, and a silent or sleeping machine is S1's**: a laptop shut for a
+// week comes back with one report of an old failure; a machine not heard from says silent, not this.
+func TestOneReportOrAQuietMachineIsNotRaised(t *testing.T) {
+	now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC)
+	f := calm(now)
+	f.doing = []inventory.Doing{unansweringPackageManager(now, 7*24*time.Hour, 1)}
+	if got := watchApplyFailing(f); len(got) != 0 {
+		t.Errorf("one report was raised as a failure that goes on: %+v", got)
+	}
+	f.doing = []inventory.Doing{unansweringPackageManager(now, time.Hour, 12)}
+	f.machines[0].lastHeard = now.Add(-time.Hour)
+	if got := watchApplyFailing(f); len(got) != 0 {
+		t.Errorf("a silent machine was raised for its last report: %+v", got)
+	}
+}

cmd/mesh-controller/plain_words.go

@@ -81,6 +81,13 @@ var plainWordings = map[string]func(conditions.Observation) words{
 				"It usually catches up by itself.", m, m),
 			Resolved: m + " applied the change"}
 	}),
+	kindApplyFailing: worded(func(o conditions.Observation) words {
+		m := machineOr(o, "a machine")
+		return words{Headline: m + " is not applying its update",
+			Explanation: fmt.Sprintf("%s has refused or only partly applied what the mesh sent it, again and again. "+
+				"The details say what fails.", conditions.Capital(m)),
+			Resolved: m + " applies what it is sent again"}
+	}),
 	"tools-silent": worded(func(o conditions.Observation) words {
 		m := machineOr(o, "a machine")
 		return words{Headline: "Tools on " + m + " are not answering",

cmd/mesh-controller/signals.go

@@ -254,6 +254,15 @@ var signalsTable = []signalRow{
 		newest: func(f *signalFacts) time.Time {
 			return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since })
 		}},
+	{Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)",
+		Bound: "one apply interval (5 min) past the first report of the same failure, over at least two reports; not " +
+			"while the machine is silent or asleep; naming the refusal, or the package manager that does not answer " +
+			"with its words; cleared by a clean apply",
+		Kind: kindApplyFailing, Severity: conditions.Warning, Phase: 2,
+		needs: func(f *signalFacts) error { return f.doingErr }, watch: watchApplyFailing,
+		newest: func(f *signalFacts) time.Time {
+			return newestOf(f.doing, func(d inventory.Doing) time.Time { return d.At })
+		}},
 }
 
 // watchFacts is S14: the snapshot a merge check is fed is older than its bound, or none was kept since

cmd/mesh-controller/signals_test.go

@@ -186,6 +186,14 @@ var suppressions = map[string]suppression{
 		},
 		past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} },
 	},
+	// A machine whose package manager does not answer (novox/hq issue 233): said once the same failure has
+	// outlasted one apply interval over two reports. Inside: a second report, only five minutes after the first.
+	"S21": {
+		inside: func(f *signalFacts) { f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute, 2)} },
+		past: func(f *signalFacts) {
+			f.doing = []inventory.Doing{unansweringPackageManager(f.now, 5*time.Minute+time.Second, 2)}
+		},
+	},
 	// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
 	"S15": {
 		inside: func(f *signalFacts) {

cmd/mesh-controller/watchdogs.go

@@ -59,6 +59,10 @@ type signalFacts struct {
 	// from, each naming its sender (S20, novox/hq issue 234).
 	refusedSends    []inventory.Send
 	refusedSendsErr error
+	// doing is every machine whose last report was not a clean apply, with since when it has said the same
+	// failure (S21, novox/hq issue 233).
+	doing    []inventory.Doing
+	doingErr error
 
 	loop    loopFacts
 	loopErr error
@@ -369,6 +373,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 	}
 	f.handActs, f.handActsErr = w.gatherHandActs(ctx, now)
 	f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid))
+	f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx)
 	f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last()
 	return f
 }