Merge pull request 'Raise a failed unit, the module's and the machine's (hq issue 315)' (#137) from fix/315-a-failed-unit-is-a-condition into main

cmd/mesh-controller/gate.go

@@ -348,6 +348,10 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact
 			}
 		case coreBehind:
 			// Not what the send moved: said nowhere against it.
+		case c.Kind == kindMachineUnits:
+			// **The machine's own failed units** (issue 315): no module places any of them, so nothing a
+			// send moved is among them — a module's failed unit is that module's own condition. Said
+			// nowhere against the send.
 		case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine:
 			// **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send
 			// did not move, or the machine cannot reach another that is down. Nothing the send did; the

cmd/mesh-controller/machine_units.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A failed unit is never silent (novox/hq issue 315).
+//
+// **A module's failed unit was never raised, and neither was the machine's.** Liveness judges what a
+// module runs long-lived; a module whose daemon is a package's unit, started by D-Bus activation, states
+// no service, and its unit failed at every start while the machine read healthy. The profile's
+// `service-manager` said `degraded`, and nothing raised that.
+//
+// The node-engine now reads every failed unit in the managers the mesh places units in, and says whose
+// each is:
+//
+//   - **a module's** — the declaration states it, writes its file, or installs the package its file
+//     belongs to — is among the module's resources, unhealthy, of kind `unit`: the module's own
+//     `module.<module>.<machine>.unhealthy`, naming the unit, raised on the second statement and held by
+//     the gate as any unhealthy resource is (ADR 0240 §4);
+//   - **the machine's** — no module places it: a mount of the machine's own table, a unit a removed
+//     package left behind — is one finding for the machine, `machine.<m>.units`, listing them, so a
+//     degraded service manager is never silent. A warning; cleared on the first statement that lists none.
+//
+// The engine applies the two-look rule itself (ADR 0241 §2), so the machine's finding is raised on the
+// statement that first says it. It is nothing a send did — no module places what it lists — so the gate
+// never holds a send on it. An engine older than this reading says nothing of its units, and nothing is
+// raised or cleared for it.
+
+// The condition a machine's own failed units raise.
+const (
+	kindMachineUnits = "machine-units"
+	sourceUnits      = "units"
+)
+
+// unitsKept is a statement's units as the inventory keeps them.
+func unitsKept(u *link.UnitsHealth) *inventory.UnitsHealth {
+	if u == nil {
+		return nil
+	}
+	out := &inventory.UnitsHealth{State: u.State, Failed: []inventory.FailedUnit{}, Unread: u.Unread}
+	for _, f := range u.Failed {
+		out.Failed = append(out.Failed, inventory.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, Result: f.Result,
+			Resource: f.Resource, Since: f.Since})
+	}
+	return out
+}
+
+// judgeUnits raises the machine's own failed units as one finding, or clears it when the statement lists
+// none. A statement that says nothing of units — an older engine — leaves it as it is.
+func judgeUnits(ctx context.Context, k *conditions.Keeper, node string, u *link.UnitsHealth) error {
+	if k == nil || u == nil {
+		return nil
+	}
+	o, raise := machineUnitsObservation(node, u)
+	if raise {
+		_, err := k.Observe(ctx, o)
+		return err
+	}
+	open, err := k.Open(ctx)
+	if err != nil {
+		return err
+	}
+	for _, c := range open {
+		if c.Kind == kindMachineUnits && c.Key == o.Key() {
+			why := fmt.Sprintf("%s's service managers list no failed unit the mesh does not place", node)
+			if u.State == link.UnitsRunning {
+				why = fmt.Sprintf("%s's service managers are running, no unit failed", node)
+			}
+			_, err := k.Clear(ctx, c.Key, why)
+			return err
+		}
+	}
+	return nil
+}
+
+// machineUnitsObservation is the machine's own failed units in one finding, and whether there are any.
+// The summary names each unit and its manager, which is what a person looks for; how each failed and
+// since when is evidence. Pure.
+func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Observation, bool) {
+	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "units", Kind: kindMachineUnits,
+		Machine: node, Severity: conditions.Warning, Source: sourceUnits}
+	if len(u.Failed) == 0 {
+		return o, false
+	}
+	var names, said []string
+	for _, f := range u.Failed {
+		where := ""
+		if f.Scope == "user" {
+			where = " (the account's own manager)"
+		}
+		names = append(names, f.Unit+where)
+		line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, orSystem(f.Scope), orNotSaid(f.Result))
+		if f.Load != "" && f.Load != "loaded" {
+			line += ", its unit " + f.Load
+		}
+		if f.Resource != "" {
+			line += ", named by the mesh's own " + f.Resource
+		}
+		line += ", since " + f.Since.UTC().Format("2006-01-02 15:04:05 MST")
+		said = append(said, line)
+	}
+	o.Summary = fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. "+
+		"Each is the machine's own: mend or remove it there, or have a module place it",
+		node, len(u.Failed), strings.Join(names, ", "))
+	o.Said = strings.Join(said, "; ")
+	return o, true
+}
+
+func orSystem(scope string) string {
+	if scope == "" {
+		return "system"
+	}
+	return scope
+}
+
+// unitsLines is what `node show` says of a machine's service managers.
+func unitsLines(h inventory.NodeHealth, had bool, _ time.Time) []string {
+	if !had || h.Units == nil {
+		return []string{"  its node-engine does not say which units failed — it is older than that reading (issue 315)"}
+	}
+	out := []string{"  its service managers: " + h.Units.State}
+	for _, f := range h.Units.Failed {
+		line := fmt.Sprintf("    failed     %s (%s) — no module places it", f.Unit, orSystem(f.Scope))
+		if f.Result != "" {
+			line += ", " + f.Result
+		}
+		out = append(out, line)
+	}
+	for _, r := range h.Resources {
+		if r.Kind == link.KindUnit {
+			out = append(out, fmt.Sprintf("    failed     %s — %s's, %s", r.Target, r.Module, r.Reason))
+		}
+	}
+	for _, w := range h.Units.Unread {
+		out = append(out, "    unread     "+w)
+	}
+	return out
+}

cmd/mesh-controller/machine_units_test.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A failed unit is never silent (novox/hq issue 315, "how it is checked"): a module's failed unit raises
+// the module's condition on the second statement, naming the unit; the machine's own failed units raise
+// one finding for the machine, listing them, cleared when none is listed; an engine that says nothing of
+// its units raises and clears nothing; and the gate never holds a send on the machine's own.
+
+// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged
+// user unit, the module's; two mounts of the machine's own table and a unit a removed package left
+// behind, the machine's.
+func desktop(at time.Time) link.Health {
+	return link.Health{Contract: link.ReadinessContract, At: at,
+		Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit,
+			Target: "openrazer-daemon.service", State: link.StateUnhealthy,
+			Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}},
+		Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{
+			{Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0},
+			{Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0},
+			{Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0},
+		}}}
+}
+
+func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv, k := open.inventory, conditionsFrom
+	openConditions := func() map[string]conditions.Condition {
+		t.Helper()
+		list, err := k.Open(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		out := map[string]conditions.Condition{}
+		for _, c := range list {
+			out[c.Key] = c
+		}
+		return out
+	}
+	say := func(at time.Time, h link.Health) {
+		t.Helper()
+		if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	say(h0, desktop(h0))
+	got := openConditions()
+	machine, raised := got["machine.laptop.units"]
+	if !raised || len(got) != 1 {
+		t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got)
+	}
+	for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} {
+		if !strings.Contains(machine.Summary, unit) {
+			t.Errorf("the finding names %s: %s", unit, machine.Summary)
+		}
+	}
+	if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning {
+		t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine)
+	}
+
+	say(h0.Add(time.Minute), desktop(h0.Add(time.Minute)))
+	got = openConditions()
+	module, raised := got["module.openrazer.laptop.unhealthy"]
+	if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") ||
+		!strings.Contains(module.Summary, "account's own service manager") {
+		t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got)
+	}
+
+	kept, had, err := inv.HealthOf(ctx, "laptop")
+	if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 {
+		t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err)
+	}
+	lines := strings.Join(unitsLines(kept, had, h0), "\n")
+	for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it",
+		"openrazer-daemon.service — openrazer's"} {
+		if !strings.Contains(lines, want) {
+			t.Errorf("node show does not say %q:\n%s", want, lines)
+		}
+	}
+
+	// An older engine says nothing of its units: nothing cleared on its word.
+	older := desktop(h0.Add(2 * time.Minute))
+	older.Units = nil
+	say(h0.Add(2*time.Minute), older)
+	if _, still := openConditions()["machine.laptop.units"]; !still {
+		t.Fatal("a statement saying nothing of units cleared the machine's finding")
+	}
+
+	// The operator mends the mounts and removes the leftover link; the module's unit still fails.
+	mended := desktop(h0.Add(3 * time.Minute))
+	mended.Units.Failed = []link.FailedUnit{}
+	say(h0.Add(3*time.Minute), mended)
+	got = openConditions()
+	if _, still := got["machine.laptop.units"]; still {
+		t.Fatalf("no failed unit of the machine's own, still open: %v", got)
+	}
+	if _, still := got["module.openrazer.laptop.unhealthy"]; !still {
+		t.Fatalf("the module's unit still fails, and its condition stands: %v", got)
+	}
+
+	// And the module's unit mended: running, nothing open.
+	well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute),
+		Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}}
+	say(h0.Add(4*time.Minute), well)
+	if got = openConditions(); len(got) != 0 {
+		t.Fatalf("a running service manager leaves nothing open: %v", got)
+	}
+}
+
+func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) {
+	since := h0
+	units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits,
+		Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"},
+		Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits}
+	if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true,
+		open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 {
+		t.Fatalf("the machine's own failed units held a send: %+v", w)
+	}
+}

cmd/mesh-controller/module_health.go

@@ -92,7 +92,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 		}
 	}
 	stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
-		HeardAt: now, Resources: resources, Streaks: streaks, Network: network})
+		HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)})
 	if err != nil || !stored {
 		if err == nil {
 			healthRefused.Add(1)
@@ -103,6 +103,14 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
 		return nil
 	}
 	err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now)
+	// And the machine's own failed units (issue 315): what no module places, one finding for the machine.
+	if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil {
+		if err == nil {
+			err = uerr
+		} else {
+			err = fmt.Errorf("%w; %v", err, uerr)
+		}
+	}
 	// And every machine's network, from every machine's newest statement (ADR 0241): a statement about one
 	// machine can hold another's finding, or release it.
 	if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil {
@@ -235,6 +243,14 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata
 func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
 	var words, said []string
 	for _, r := range rs {
+		if r.Kind == link.KindUnit {
+			// A failed unit is named by the unit, which is what a person looks for (issue 315); the
+			// resource that places it — a package, a file — is evidence.
+			words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason))
+			said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource,
+				orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
+			continue
+		}
 		words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
 		said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
 			r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,

cmd/mesh-controller/nodes.go

@@ -634,6 +634,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
 		for _, line := range networkLines(h, had, time.Now()) {
 			fmt.Println(line)
 		}
+		for _, line := range unitsLines(h, had, time.Now()) {
+			fmt.Println(line)
+		}
 	}
 
 	held, err := inv.Profile(ctx, name)

internal/inventory/health.go

@@ -43,6 +43,27 @@ type NodeHealth struct {
 	// Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an
 	// engine older than that judging.
 	Network *NetworkHealth
+	// Units is the machine's service managers as its engine said them (novox/hq issue 315); nil from an
+	// engine older than that reading.
+	Units *UnitsHealth
+}
+
+// UnitsHealth is a machine's service managers as its engine said them (issue 315): running, degraded or
+// unknown, and each failed unit no module places.
+type UnitsHealth struct {
+	State  string       `json:"state"`
+	Failed []FailedUnit `json:"failed"`
+	Unread []string     `json:"unread,omitempty"`
+}
+
+// FailedUnit is one failed unit no module places: the machine's own.
+type FailedUnit struct {
+	Unit     string    `json:"unit"`
+	Scope    string    `json:"scope"`
+	Load     string    `json:"load,omitempty"`
+	Result   string    `json:"result,omitempty"`
+	Resource string    `json:"resource,omitempty"`
+	Since    time.Time `json:"since"`
 }
 
 // NetworkHealth is a machine's networking as its engine said it (ADR 0241).
@@ -83,7 +104,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error)
 
 func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) {
 	rows, err := i.store.Pool().Query(ctx,
-		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network
+		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network, h.units
 		 from node_health h join node n on n.id = h.node
 		 where $1 = '' or n.name = $1`, only)
 	if err != nil {
@@ -93,8 +114,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 	out := map[string]NodeHealth{}
 	for rows.Next() {
 		var h NodeHealth
-		var resources, streaks, network []byte
+		var resources, streaks, network, units []byte
-		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil {
+		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network, &units); err != nil {
 			return nil, err
 		}
 		if err := json.Unmarshal(resources, &h.Resources); err != nil {
@@ -108,6 +129,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 				return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err)
 			}
 		}
+		if len(units) > 0 {
+			if err := json.Unmarshal(units, &h.Units); err != nil {
+				return nil, fmt.Errorf("%s's units cannot be read: %w", h.Node, err)
+			}
+		}
 		out[h.Node] = h
 	}
 	return out, rows.Err()
@@ -136,19 +162,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error
 			return false, err
 		}
 	}
+	var units []byte
+	if h.Units != nil {
+		if units, err = json.Marshal(h.Units); err != nil {
+			return false, err
+		}
+	}
 	heard := h.HeardAt
 	if heard.IsZero() {
 		heard = time.Now()
 	}
 	var node string
 	err = i.store.Pool().QueryRow(ctx,
-		`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network)
+		`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network, units)
-		 select id, $2, $3, $4, $5, $6, $7 from node where name = $1
+		 select id, $2, $3, $4, $5, $6, $7, $8 from node where name = $1
 		 on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at,
 		     heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks,
-		     network = excluded.network
+		     network = excluded.network, units = excluded.units
 		   where node_health.said_at <= excluded.said_at
-		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node)
+		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network, units).Scan(&node)
 	if errors.Is(err, pgx.ErrNoRows) {
 		if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil {
 			return false, nerr

internal/inventory/migrations/0080-a-failed-unit-is-never-silent.sql

@@ -0,0 +1,9 @@
+-- A failed unit is never silent (novox/hq issue 315).
+--
+-- Beside the state of every long-running resource and its networking, each machine's node-engine states
+-- its service managers: running, degraded or unknown, and every failed unit no module places — a mount of
+-- the machine's own table, a unit a removed package left behind. A module's failed unit is among the
+-- resources, as that module's. Kept with the machine's newest statement, replaced with it, so `node show`
+-- and a controller started again read the same word. Null for a machine whose node-engine is older than
+-- this reading: its units are not known — never healthy, never a reason to raise anything.
+alter table node_health add column units jsonb;

internal/link/protocol.go

@@ -431,6 +431,42 @@ type Health struct {
 	// Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine
 	// older than that judging, which is "not known", never healthy.
 	Network *NetworkHealth `json:"network,omitempty"`
+	// Units is the machine's service managers as its engine read them (novox/hq issue 315): whether any
+	// unit failed, and each failed unit no module places. A module's failed unit is among Resources, of
+	// kind KindUnit. Nil from an engine older than that reading: not known, never healthy.
+	Units *UnitsHealth `json:"units,omitempty"`
+}
+
+// KindUnit is a module's unit its service manager says failed (issue 315): its package's unit, a unit
+// file it writes, a service whose lifecycle is the machine's — said unhealthy while it stays failed.
+const KindUnit = "unit"
+
+// The states of a machine's service managers (issue 315).
+const (
+	UnitsRunning  = "running"
+	UnitsDegraded = "degraded"
+)
+
+// UnitsHealth is the machine's service managers in one statement (issue 315). The node-engine's own
+// (mesh-host internal/link UnitsHealth).
+type UnitsHealth struct {
+	// State is running, degraded or unknown.
+	State string `json:"state"`
+	// Failed is every unit failed on two looks in a row that no module places: the machine's own.
+	Failed []FailedUnit `json:"failed"`
+	Unread []string     `json:"unread,omitempty"`
+}
+
+// FailedUnit is one failed unit no module places.
+type FailedUnit struct {
+	Unit string `json:"unit"`
+	// Scope is system, or user: an account's own manager.
+	Scope  string `json:"scope"`
+	Load   string `json:"load,omitempty"`
+	Result string `json:"result,omitempty"`
+	// Resource is the mesh's own resource naming it, when the mesh placed it in its own right.
+	Resource string    `json:"resource,omitempty"`
+	Since    time.Time `json:"since"`
 }
 
 // NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since