Raise a failed unit, the module's and the machine's (hq issue 315)

A module's failed unit now arrives among its resources and raises the
module's own condition, named by the unit. The machine's own failed
units, which no module places, are one warning for the machine listing
them, cleared when none is listed. Nothing a send moved is among them,
so the gate never holds a send on that finding. The statement's units
are kept with the machine's health (migration 0080) and node show says
them.

cmd/mesh-controller/gate.go

@@ -348,6 +348,10 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
 			}
 		case coreBehind:
 			// Not what the send moved: said nowhere against it.
+		case c.Kind == kindMachineUnits:
+			// **The machine's own failed units** (issue 315): no module places any of them, so nothing a
+			// send moved is among them — a module's failed unit is that module's own condition. Said
+			// nowhere against the send.
 		case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine:
 			// **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send
 			// did not move, or the machine cannot reach another that is down. Nothing the send did; the

cmd/mesh-controller/machine_units.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A failed unit is never silent (novox/hq issue 315).
+//
+// **A module's failed unit was never raised, and neither was the machine's.** Liveness judges what a
+// module runs long-lived; a module whose daemon is a package's unit, started by D-Bus activation, states
+// no service, and its unit failed at every start while the machine read healthy. The profile's
+// `service-manager` said `degraded`, and nothing raised that.
+//
+// The node-engine now reads every failed unit in the managers the mesh places units in, and says whose
+// each is:
+//
+//   - **a module's** — the declaration states it, writes its file, or installs the package its file
+//     belongs to — is among the module's resources, unhealthy, of kind `unit`: the module's own
+//     `module.<module>.<machine>.unhealthy`, naming the unit, raised on the second statement and held by
+//     the gate as any unhealthy resource is (ADR 0240 §4);
+//   - **the machine's** — no module places it: a mount of the machine's own table, a unit a removed
+//     package left behind — is one finding for the machine, `machine.<m>.units`, listing them, so a
+//     degraded service manager is never silent. A warning; cleared on the first statement that lists none.
+//
+// The engine applies the two-look rule itself (ADR 0241 §2), so the machine's finding is raised on the
+// statement that first says it. It is nothing a send did — no module places what it lists — so the gate
+// never holds a send on it. An engine older than this reading says nothing of its units, and nothing is
+// raised or cleared for it.
+
+// The condition a machine's own failed units raise.
+const (
+	kindMachineUnits = "machine-units"
+	sourceUnits      = "units"
+)
+
+// unitsKept is a statement's units as the inventory keeps them.
+func unitsKept(u *link.UnitsHealth) *inventory.UnitsHealth {
+	if u == nil {
+		return nil
+	}
+	out := &inventory.UnitsHealth{State: u.State, Failed: []inventory.FailedUnit{}, Unread: u.Unread}
+	for _, f := range u.Failed {
+		out.Failed = append(out.Failed, inventory.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, Result: f.Result,
+			Resource: f.Resource, Since: f.Since})
+	}
+	return out
+}
+
+// judgeUnits raises the machine's own failed units as one finding, or clears it when the statement lists
+// none. A statement that says nothing of units — an older engine — leaves it as it is.
+func judgeUnits(ctx context.Context, k *conditions.Keeper, node string, u *link.UnitsHealth) error {
+	if k == nil || u == nil {
+		return nil
+	}
+	o, raise := machineUnitsObservation(node, u)
+	if raise {
+		_, err := k.Observe(ctx, o)
+		return err
+	}
+	open, err := k.Open(ctx)
+	if err != nil {
+		return err
+	}
+	for _, c := range open {
+		if c.Kind == kindMachineUnits && c.Key == o.Key() {
+			why := fmt.Sprintf("%s's service managers list no failed unit the mesh does not place", node)
+			if u.State == link.UnitsRunning {
+				why = fmt.Sprintf("%s's service managers are running, no unit failed", node)
+			}
+			_, err := k.Clear(ctx, c.Key, why)
+			return err
+		}
+	}
+	return nil
+}
+
+// machineUnitsObservation is the machine's own failed units in one finding, and whether there are any.
+// The summary names each unit and its manager, which is what a person looks for; how each failed and
+// since when is evidence. Pure.
+func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Observation, bool) {
+	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "units", Kind: kindMachineUnits,
+		Machine: node, Severity: conditions.Warning, Source: sourceUnits}
+	if len(u.Failed) == 0 {
+		return o, false
+	}
+	var names, said []string
+	for _, f := range u.Failed {
+		where := ""
+		if f.Scope == "user" {
+			where = " (the account's own manager)"
+		}
+		names = append(names, f.Unit+where)
+		line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, orSystem(f.Scope), orNotSaid(f.Result))
+		if f.Load != "" && f.Load != "loaded" {
+			line += ", its unit " + f.Load
+		}
+		if f.Resource != "" {
+			line += ", named by the mesh's own " + f.Resource
+		}
+		line += ", since " + f.Since.UTC().Format("2006-01-02 15:04:05 MST")
+		said = append(said, line)
+	}
+	o.Summary = fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. "+
+		"Each is the machine's own: mend or remove it there, or have a module place it",
+		node, len(u.Failed), strings.Join(names, ", "))
+	o.Said = strings.Join(said, "; ")
+	return o, true
+}
+
+func orSystem(scope string) string {
+	if scope == "" {
+		return "system"
+	}
+	return scope
+}
+
+// unitsLines is what `node show` says of a machine's service managers.
+func unitsLines(h inventory.NodeHealth, had bool, _ time.Time) []string {
+	if !had || h.Units == nil {
+		return []string{"  its node-engine does not say which units failed — it is older than that reading (issue 315)"}
+	}
+	out := []string{"  its service managers: " + h.Units.State}
+	for _, f := range h.Units.Failed {
+		line := fmt.Sprintf("    failed     %s (%s) — no module places it", f.Unit, orSystem(f.Scope))
+		if f.Result != "" {
+			line += ", " + f.Result
+		}
+		out = append(out, line)
+	}
+	for _, r := range h.Resources {
+		if r.Kind == link.KindUnit {
+			out = append(out, fmt.Sprintf("    failed     %s — %s's, %s", r.Target, r.Module, r.Reason))
+		}
+	}
+	for _, w := range h.Units.Unread {
+		out = append(out, "    unread     "+w)
+	}
+	return out
+}

cmd/mesh-controller/machine_units_test.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A failed unit is never silent (novox/hq issue 315, "how it is checked"): a module's failed unit raises
+// the module's condition on the second statement, naming the unit; the machine's own failed units raise
+// one finding for the machine, listing them, cleared when none is listed; an engine that says nothing of
+// its units raises and clears nothing; and the gate never holds a send on the machine's own.
+
+// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged
+// user unit, the module's; two mounts of the machine's own table and a unit a removed package left
+// behind, the machine's.
+func desktop(at time.Time) link.Health {
+	return link.Health{Contract: link.ReadinessContract, At: at,
+		Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit,
+			Target: "openrazer-daemon.service", State: link.StateUnhealthy,
+			Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}},
+		Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{
+			{Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0},
+			{Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0},
+			{Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0},
+		}}}
+}
+
+func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv, k := open.inventory, conditionsFrom
+	openConditions := func() map[string]conditions.Condition {
+		t.Helper()
+		list, err := k.Open(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		out := map[string]conditions.Condition{}
+		for _, c := range list {
+			out[c.Key] = c
+		}
+		return out
+	}
+	say := func(at time.Time, h link.Health) {
+		t.Helper()
+		if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	say(h0, desktop(h0))
+	got := openConditions()
+	machine, raised := got["machine.laptop.units"]
+	if !raised || len(got) != 1 {
+		t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got)
+	}
+	for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} {
+		if !strings.Contains(machine.Summary, unit) {
+			t.Errorf("the finding names %s: %s", unit, machine.Summary)
+		}
+	}
+	if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning {
+		t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine)
+	}
+
+	say(h0.Add(time.Minute), desktop(h0.Add(time.Minute)))
+	got = openConditions()
+	module, raised := got["module.openrazer.laptop.unhealthy"]
+	if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") ||
+		!strings.Contains(module.Summary, "account's own service manager") {
+		t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got)
+	}
+
+	kept, had, err := inv.HealthOf(ctx, "laptop")
+	if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 {
+		t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err)
+	}
+	lines := strings.Join(unitsLines(kept, had, h0), "\n")
+	for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it",
+		"openrazer-daemon.service — openrazer's"} {
+		if !strings.Contains(lines, want) {
+			t.Errorf("node show does not say %q:\n%s", want, lines)
+		}
+	}
+
+	// An older engine says nothing of its units: nothing cleared on its word.
+	older := desktop(h0.Add(2 * time.Minute))
+	older.Units = nil
+	say(h0.Add(2*time.Minute), older)
+	if _, still := openConditions()["machine.laptop.units"]; !still {
+		t.Fatal("a statement saying nothing of units cleared the machine's finding")
+	}
+
+	// The operator mends the mounts and removes the leftover link; the module's unit still fails.
+	mended := desktop(h0.Add(3 * time.Minute))
+	mended.Units.Failed = []link.FailedUnit{}
+	say(h0.Add(3*time.Minute), mended)
+	got = openConditions()
+	if _, still := got["machine.laptop.units"]; still {
+		t.Fatalf("no failed unit of the machine's own, still open: %v", got)
+	}
+	if _, still := got["module.openrazer.laptop.unhealthy"]; !still {
+		t.Fatalf("the module's unit still fails, and its condition stands: %v", got)
+	}
+
+	// And the module's unit mended: running, nothing open.
+	well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute),
+		Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}}
+	say(h0.Add(4*time.Minute), well)
+	if got = openConditions(); len(got) != 0 {
+		t.Fatalf("a running service manager leaves nothing open: %v", got)
+	}
+}
+
+func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) {
+	since := h0
+	units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits,
+		Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
+		Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits}
+	if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true,
+		open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 {
+		t.Fatalf("the machine's own failed units held a send: %+v", w)
+	}
+}

cmd/mesh-controller/module_health.go

@@ -92,7 +92,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 		}
 	}
 	stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
-		HeardAt: now, Resources: resources, Streaks: streaks, Network: network})
+		HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)})
 	if err != nil || !stored {
 		if err == nil {
 			healthRefused.Add(1)
@@ -103,6 +103,14 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 		return nil
 	}
 	err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
+	// And the machine's own failed units (issue 315): what no module places, one finding for the machine.
+	if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil {
+		if err == nil {
+			err = uerr
+		} else {
+			err = fmt.Errorf("%w; %v", err, uerr)
+		}
+	}
 	// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
 	// machine can hold another's finding, or release it.
 	if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
@@ -235,6 +243,14 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
 func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
 	var words, said []string
 	for _, r := range rs {
+		if r.Kind == link.KindUnit {
+			// A failed unit is named by the unit, which is what a person looks for (issue 315); the
+			// resource that places it — a package, a file — is evidence.
+			words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason))
+			said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource,
+				orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
+			continue
+		}
 		words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
 		said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
 			r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,

cmd/mesh-controller/nodes.go

@@ -634,6 +634,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
 		for _, line := range networkLines(h, had, time.Now()) {
 			fmt.Println(line)
 		}
+		for _, line := range unitsLines(h, had, time.Now()) {
+			fmt.Println(line)
+		}
 	}
 
 	held, err := inv.Profile(ctx, name)