Raise a failed unit, the module's and the machine's (hq issue 315)

A module's failed unit now arrives among its resources and raises the
module's own condition, named by the unit. The machine's own failed
units, which no module places, are one warning for the machine listing
them, cleared when none is listed. Nothing a send moved is among them,
so the gate never holds a send on that finding. The statement's units
are kept with the machine's health (migration 0080) and node show says
them.

internal/inventory/health.go

@@ -43,6 +43,27 @@ type NodeHealth struct {
 	// Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an
 	// engine older than that judging.
 	Network *NetworkHealth
+	// Units is the machine's service managers as its engine said them (novox/hq issue 315); nil from an
+	// engine older than that reading.
+	Units *UnitsHealth
+}
+
+// UnitsHealth is a machine's service managers as its engine said them (issue 315): running, degraded or
+// unknown, and each failed unit no module places.
+type UnitsHealth struct {
+	State  string       `json:"state"`
+	Failed []FailedUnit `json:"failed"`
+	Unread []string     `json:"unread,omitempty"`
+}
+
+// FailedUnit is one failed unit no module places: the machine's own.
+type FailedUnit struct {
+	Unit     string    `json:"unit"`
+	Scope    string    `json:"scope"`
+	Load     string    `json:"load,omitempty"`
+	Result   string    `json:"result,omitempty"`
+	Resource string    `json:"resource,omitempty"`
+	Since    time.Time `json:"since"`
 }
 
 // NetworkHealth is a machine's networking as its engine said it (ADR 0241).
@@ -83,7 +104,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error)
 
 func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) {
 	rows, err := i.store.Pool().Query(ctx,
-		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network
+		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network, h.units
 		 from node_health h join node n on n.id = h.node
 		 where $1 = '' or n.name = $1`, only)
 	if err != nil {
@@ -93,8 +114,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 	out := map[string]NodeHealth{}
 	for rows.Next() {
 		var h NodeHealth
-		var resources, streaks, network []byte
-		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil {
+		var resources, streaks, network, units []byte
+		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network, &units); err != nil {
 			return nil, err
 		}
 		if err := json.Unmarshal(resources, &h.Resources); err != nil {
@@ -108,6 +129,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 				return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err)
 			}
 		}
+		if len(units) > 0 {
+			if err := json.Unmarshal(units, &h.Units); err != nil {
+				return nil, fmt.Errorf("%s's units cannot be read: %w", h.Node, err)
+			}
+		}
 		out[h.Node] = h
 	}
 	return out, rows.Err()
@@ -136,19 +162,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error
 			return false, err
 		}
 	}
+	var units []byte
+	if h.Units != nil {
+		if units, err = json.Marshal(h.Units); err != nil {
+			return false, err
+		}
+	}
 	heard := h.HeardAt
 	if heard.IsZero() {
 		heard = time.Now()
 	}
 	var node string
 	err = i.store.Pool().QueryRow(ctx,
-		`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network)
-		 select id, $2, $3, $4, $5, $6, $7 from node where name = $1
+		`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network, units)
+		 select id, $2, $3, $4, $5, $6, $7, $8 from node where name = $1
 		 on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at,
 		     heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks,
-		     network = excluded.network
+		     network = excluded.network, units = excluded.units
 		   where node_health.said_at <= excluded.said_at
-		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node)
+		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network, units).Scan(&node)
 	if errors.Is(err, pgx.ErrNoRows) {
 		if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil {
 			return false, nerr

internal/inventory/migrations/0080-a-failed-unit-is-never-silent.sql

@@ -0,0 +1,9 @@
+-- A failed unit is never silent (novox/hq issue 315).
+--
+-- Beside the state of every long-running resource and its networking, each machine's node-engine states
+-- its service managers: running, degraded or unknown, and every failed unit no module places — a mount of
+-- the machine's own table, a unit a removed package left behind. A module's failed unit is among the
+-- resources, as that module's. Kept with the machine's newest statement, replaced with it, so `node show`
+-- and a controller started again read the same word. Null for a machine whose node-engine is older than
+-- this reading: its units are not known — never healthy, never a reason to raise anything.
+alter table node_health add column units jsonb;