Raise a failed unit, the module's and the machine's (hq issue 315)

A module's failed unit now arrives among its resources and raises the
module's own condition, named by the unit. The machine's own failed
units, which no module places, are one warning for the machine listing
them, cleared when none is listed. Nothing a send moved is among them,
so the gate never holds a send on that finding. The statement's units
are kept with the machine's health (migration 0080) and node show says
them.

cmd/mesh-controller/gate.go

@@ -348,6 +348,10 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
 			}
 		case coreBehind:
 			// Not what the send moved: said nowhere against it.
+		case c.Kind == kindMachineUnits:
+			// **The machine's own failed units** (issue 315): no module places any of them, so nothing a
+			// send moved is among them — a module's failed unit is that module's own condition. Said
+			// nowhere against the send.
 		case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine:
 			// **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send
 			// did not move, or the machine cannot reach another that is down. Nothing the send did; the

cmd/mesh-controller/machine_units.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A failed unit is never silent (novox/hq issue 315).
+//
+// **A module's failed unit was never raised, and neither was the machine's.** Liveness judges what a
+// module runs long-lived; a module whose daemon is a package's unit, started by D-Bus activation, states
+// no service, and its unit failed at every start while the machine read healthy. The profile's
+// `service-manager` said `degraded`, and nothing raised that.
+//
+// The node-engine now reads every failed unit in the managers the mesh places units in, and says whose
+// each is:
+//
+//   - **a module's** — the declaration states it, writes its file, or installs the package its file
+//     belongs to — is among the module's resources, unhealthy, of kind `unit`: the module's own
+//     `module.<module>.<machine>.unhealthy`, naming the unit, raised on the second statement and held by
+//     the gate as any unhealthy resource is (ADR 0240 §4);
+//   - **the machine's** — no module places it: a mount of the machine's own table, a unit a removed
+//     package left behind — is one finding for the machine, `machine.<m>.units`, listing them, so a
+//     degraded service manager is never silent. A warning; cleared on the first statement that lists none.
+//
+// The engine applies the two-look rule itself (ADR 0241 §2), so the machine's finding is raised on the
+// statement that first says it. It is nothing a send did — no module places what it lists — so the gate
+// never holds a send on it. An engine older than this reading says nothing of its units, and nothing is
+// raised or cleared for it.
+
+// The condition a machine's own failed units raise.
+const (
+	kindMachineUnits = "machine-units"
+	sourceUnits      = "units"
+)
+
+// unitsKept is a statement's units as the inventory keeps them.
+func unitsKept(u *link.UnitsHealth) *inventory.UnitsHealth {
+	if u == nil {
+		return nil
+	}
+	out := &inventory.UnitsHealth{State: u.State, Failed: []inventory.FailedUnit{}, Unread: u.Unread}
+	for _, f := range u.Failed {
+		out.Failed = append(out.Failed, inventory.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, Result: f.Result,
+			Resource: f.Resource, Since: f.Since})
+	}
+	return out
+}
+
+// judgeUnits raises the machine's own failed units as one finding, or clears it when the statement lists
+// none. A statement that says nothing of units — an older engine — leaves it as it is.
+func judgeUnits(ctx context.Context, k *conditions.Keeper, node string, u *link.UnitsHealth) error {
+	if k == nil || u == nil {
+		return nil
+	}
+	o, raise := machineUnitsObservation(node, u)
+	if raise {
+		_, err := k.Observe(ctx, o)
+		return err
+	}
+	open, err := k.Open(ctx)
+	if err != nil {
+		return err
+	}
+	for _, c := range open {
+		if c.Kind == kindMachineUnits && c.Key == o.Key() {
+			why := fmt.Sprintf("%s's service managers list no failed unit the mesh does not place", node)
+			if u.State == link.UnitsRunning {
+				why = fmt.Sprintf("%s's service managers are running, no unit failed", node)
+			}
+			_, err := k.Clear(ctx, c.Key, why)
+			return err
+		}
+	}
+	return nil
+}
+
+// machineUnitsObservation is the machine's own failed units in one finding, and whether there are any.
+// The summary names each unit and its manager, which is what a person looks for; how each failed and
+// since when is evidence. Pure.
+func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Observation, bool) {
+	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "units", Kind: kindMachineUnits,
+		Machine: node, Severity: conditions.Warning, Source: sourceUnits}
+	if len(u.Failed) == 0 {
+		return o, false
+	}
+	var names, said []string
+	for _, f := range u.Failed {
+		where := ""
+		if f.Scope == "user" {
+			where = " (the account's own manager)"
+		}
+		names = append(names, f.Unit+where)
+		line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, orSystem(f.Scope), orNotSaid(f.Result))
+		if f.Load != "" && f.Load != "loaded" {
+			line += ", its unit " + f.Load
+		}
+		if f.Resource != "" {
+			line += ", named by the mesh's own " + f.Resource
+		}
+		line += ", since " + f.Since.UTC().Format("2006-01-02 15:04:05 MST")
+		said = append(said, line)
+	}
+	o.Summary = fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. "+
+		"Each is the machine's own: mend or remove it there, or have a module place it",
+		node, len(u.Failed), strings.Join(names, ", "))
+	o.Said = strings.Join(said, "; ")
+	return o, true
+}
+
+func orSystem(scope string) string {
+	if scope == "" {
+		return "system"
+	}
+	return scope
+}
+
+// unitsLines is what `node show` says of a machine's service managers.
+func unitsLines(h inventory.NodeHealth, had bool, _ time.Time) []string {
+	if !had || h.Units == nil {
+		return []string{"  its node-engine does not say which units failed — it is older than that reading (issue 315)"}
+	}
+	out := []string{"  its service managers: " + h.Units.State}
+	for _, f := range h.Units.Failed {
+		line := fmt.Sprintf("    failed     %s (%s) — no module places it", f.Unit, orSystem(f.Scope))
+		if f.Result != "" {
+			line += ", " + f.Result
+		}
+		out = append(out, line)
+	}
+	for _, r := range h.Resources {
+		if r.Kind == link.KindUnit {
+			out = append(out, fmt.Sprintf("    failed     %s — %s's, %s", r.Target, r.Module, r.Reason))
+		}
+	}
+	for _, w := range h.Units.Unread {
+		out = append(out, "    unread     "+w)
+	}
+	return out
+}

cmd/mesh-controller/machine_units_test.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A failed unit is never silent (novox/hq issue 315, "how it is checked"): a module's failed unit raises
+// the module's condition on the second statement, naming the unit; the machine's own failed units raise
+// one finding for the machine, listing them, cleared when none is listed; an engine that says nothing of
+// its units raises and clears nothing; and the gate never holds a send on the machine's own.
+
+// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged
+// user unit, the module's; two mounts of the machine's own table and a unit a removed package left
+// behind, the machine's.
+func desktop(at time.Time) link.Health {
+	return link.Health{Contract: link.ReadinessContract, At: at,
+		Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit,
+			Target: "openrazer-daemon.service", State: link.StateUnhealthy,
+			Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}},
+		Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{
+			{Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0},
+			{Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0},
+			{Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0},
+		}}}
+}
+
+func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv, k := open.inventory, conditionsFrom
+	openConditions := func() map[string]conditions.Condition {
+		t.Helper()
+		list, err := k.Open(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		out := map[string]conditions.Condition{}
+		for _, c := range list {
+			out[c.Key] = c
+		}
+		return out
+	}
+	say := func(at time.Time, h link.Health) {
+		t.Helper()
+		if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	say(h0, desktop(h0))
+	got := openConditions()
+	machine, raised := got["machine.laptop.units"]
+	if !raised || len(got) != 1 {
+		t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got)
+	}
+	for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} {
+		if !strings.Contains(machine.Summary, unit) {
+			t.Errorf("the finding names %s: %s", unit, machine.Summary)
+		}
+	}
+	if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning {
+		t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine)
+	}
+
+	say(h0.Add(time.Minute), desktop(h0.Add(time.Minute)))
+	got = openConditions()
+	module, raised := got["module.openrazer.laptop.unhealthy"]
+	if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") ||
+		!strings.Contains(module.Summary, "account's own service manager") {
+		t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got)
+	}
+
+	kept, had, err := inv.HealthOf(ctx, "laptop")
+	if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 {
+		t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err)
+	}
+	lines := strings.Join(unitsLines(kept, had, h0), "\n")
+	for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it",
+		"openrazer-daemon.service — openrazer's"} {
+		if !strings.Contains(lines, want) {
+			t.Errorf("node show does not say %q:\n%s", want, lines)
+		}
+	}
+
+	// An older engine says nothing of its units: nothing cleared on its word.
+	older := desktop(h0.Add(2 * time.Minute))
+	older.Units = nil
+	say(h0.Add(2*time.Minute), older)
+	if _, still := openConditions()["machine.laptop.units"]; !still {
+		t.Fatal("a statement saying nothing of units cleared the machine's finding")
+	}
+
+	// The operator mends the mounts and removes the leftover link; the module's unit still fails.
+	mended := desktop(h0.Add(3 * time.Minute))
+	mended.Units.Failed = []link.FailedUnit{}
+	say(h0.Add(3*time.Minute), mended)
+	got = openConditions()
+	if _, still := got["machine.laptop.units"]; still {
+		t.Fatalf("no failed unit of the machine's own, still open: %v", got)
+	}
+	if _, still := got["module.openrazer.laptop.unhealthy"]; !still {
+		t.Fatalf("the module's unit still fails, and its condition stands: %v", got)
+	}
+
+	// And the module's unit mended: running, nothing open.
+	well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute),
+		Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}}
+	say(h0.Add(4*time.Minute), well)
+	if got = openConditions(); len(got) != 0 {
+		t.Fatalf("a running service manager leaves nothing open: %v", got)
+	}
+}
+
+func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) {
+	since := h0
+	units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits,
+		Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
+		Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits}
+	if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true,
+		open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 {
+		t.Fatalf("the machine's own failed units held a send: %+v", w)
+	}
+}

cmd/mesh-controller/module_health.go

@@ -92,7 +92,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 		}
 	}
 	stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
-		HeardAt: now, Resources: resources, Streaks: streaks, Network: network})
+		HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)})
 	if err != nil || !stored {
 		if err == nil {
 			healthRefused.Add(1)
@@ -103,6 +103,14 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 		return nil
 	}
 	err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
+	// And the machine's own failed units (issue 315): what no module places, one finding for the machine.
+	if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil {
+		if err == nil {
+			err = uerr
+		} else {
+			err = fmt.Errorf("%w; %v", err, uerr)
+		}
+	}
 	// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
 	// machine can hold another's finding, or release it.
 	if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
@@ -235,6 +243,14 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
 func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
 	var words, said []string
 	for _, r := range rs {
+		if r.Kind == link.KindUnit {
+			// A failed unit is named by the unit, which is what a person looks for (issue 315); the
+			// resource that places it — a package, a file — is evidence.
+			words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason))
+			said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource,
+				orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
+			continue
+		}
 		words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
 		said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
 			r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,

cmd/mesh-controller/nodes.go

@@ -634,6 +634,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
 		for _, line := range networkLines(h, had, time.Now()) {
 			fmt.Println(line)
 		}
+		for _, line := range unitsLines(h, had, time.Now()) {
+			fmt.Println(line)
+		}
 	}
 
 	held, err := inv.Profile(ctx, name)

internal/inventory/health.go

@@ -43,6 +43,27 @@ type NodeHealth struct {
 	// Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an
 	// engine older than that judging.
 	Network *NetworkHealth
+	// Units is the machine's service managers as its engine said them (novox/hq issue 315); nil from an
+	// engine older than that reading.
+	Units *UnitsHealth
+}
+
+// UnitsHealth is a machine's service managers as its engine said them (issue 315): running, degraded or
+// unknown, and each failed unit no module places.
+type UnitsHealth struct {
+	State  string       `json:"state"`
+	Failed []FailedUnit `json:"failed"`
+	Unread []string     `json:"unread,omitempty"`
+}
+
+// FailedUnit is one failed unit no module places: the machine's own.
+type FailedUnit struct {
+	Unit     string    `json:"unit"`
+	Scope    string    `json:"scope"`
+	Load     string    `json:"load,omitempty"`
+	Result   string    `json:"result,omitempty"`
+	Resource string    `json:"resource,omitempty"`
+	Since    time.Time `json:"since"`
 }
 
 // NetworkHealth is a machine's networking as its engine said it (ADR 0241).
@@ -83,7 +104,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error)
 
 func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) {
 	rows, err := i.store.Pool().Query(ctx,
-		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network
+		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network, h.units
 		 from node_health h join node n on n.id = h.node
 		 where $1 = '' or n.name = $1`, only)
 	if err != nil {
@@ -93,8 +114,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 	out := map[string]NodeHealth{}
 	for rows.Next() {
 		var h NodeHealth
-		var resources, streaks, network []byte
-		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil {
+		var resources, streaks, network, units []byte
+		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network, &units); err != nil {
 			return nil, err
 		}
 		if err := json.Unmarshal(resources, &h.Resources); err != nil {
@@ -108,6 +129,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 				return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err)
 			}
 		}
+		if len(units) > 0 {
+			if err := json.Unmarshal(units, &h.Units); err != nil {
+				return nil, fmt.Errorf("%s's units cannot be read: %w", h.Node, err)
+			}
+		}
 		out[h.Node] = h
 	}
 	return out, rows.Err()
@@ -136,19 +162,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error
 			return false, err
 		}
 	}
+	var units []byte
+	if h.Units != nil {
+		if units, err = json.Marshal(h.Units); err != nil {
+			return false, err
+		}
+	}
 	heard := h.HeardAt
 	if heard.IsZero() {
 		heard = time.Now()
 	}
 	var node string
 	err = i.store.Pool().QueryRow(ctx,
-		`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network)
-		 select id, $2, $3, $4, $5, $6, $7 from node where name = $1
+		`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network, units)
+		 select id, $2, $3, $4, $5, $6, $7, $8 from node where name = $1
 		 on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at,
 		     heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks,
-		     network = excluded.network
+		     network = excluded.network, units = excluded.units
 		   where node_health.said_at <= excluded.said_at
-		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node)
+		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network, units).Scan(&node)
 	if errors.Is(err, pgx.ErrNoRows) {
 		if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil {
 			return false, nerr

internal/inventory/migrations/0080-a-failed-unit-is-never-silent.sql

@@ -0,0 +1,9 @@
+-- A failed unit is never silent (novox/hq issue 315).
+--
+-- Beside the state of every long-running resource and its networking, each machine's node-engine states
+-- its service managers: running, degraded or unknown, and every failed unit no module places — a mount of
+-- the machine's own table, a unit a removed package left behind. A module's failed unit is among the
+-- resources, as that module's. Kept with the machine's newest statement, replaced with it, so `node show`
+-- and a controller started again read the same word. Null for a machine whose node-engine is older than
+-- this reading: its units are not known — never healthy, never a reason to raise anything.
+alter table node_health add column units jsonb;

internal/link/protocol.go

@@ -431,6 +431,42 @@ type Health struct {
 	// Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine
 	// older than that judging, which is "not known", never healthy.
 	Network *NetworkHealth `json:"network,omitempty"`
+	// Units is the machine's service managers as its engine read them (novox/hq issue 315): whether any
+	// unit failed, and each failed unit no module places. A module's failed unit is among Resources, of
+	// kind KindUnit. Nil from an engine older than that reading: not known, never healthy.
+	Units *UnitsHealth `json:"units,omitempty"`
+}
+
+// KindUnit is a module's unit its service manager says failed (issue 315): its package's unit, a unit
+// file it writes, a service whose lifecycle is the machine's — said unhealthy while it stays failed.
+const KindUnit = "unit"
+
+// The states of a machine's service managers (issue 315).
+const (
+	UnitsRunning  = "running"
+	UnitsDegraded = "degraded"
+)
+
+// UnitsHealth is the machine's service managers in one statement (issue 315). The node-engine's own
+// (mesh-host internal/link UnitsHealth).
+type UnitsHealth struct {
+	// State is running, degraded or unknown.
+	State string `json:"state"`
+	// Failed is every unit failed on two looks in a row that no module places: the machine's own.
+	Failed []FailedUnit `json:"failed"`
+	Unread []string     `json:"unread,omitempty"`
+}
+
+// FailedUnit is one failed unit no module places.
+type FailedUnit struct {
+	Unit string `json:"unit"`
+	// Scope is system, or user: an account's own manager.
+	Scope  string `json:"scope"`
+	Load   string `json:"load,omitempty"`
+	Result string `json:"result,omitempty"`
+	// Resource is the mesh's own resource naming it, when the mesh placed it in its own right.
+	Resource string    `json:"resource,omitempty"`
+	Since    time.Time `json:"since"`
 }
 
 // NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since