Raise a failed unit, the module's and the machine's (hq issue 315)

A module's failed unit now arrives among its resources and raises the
module's own condition, named by the unit. The machine's own failed
units, which no module places, are one warning for the machine listing
them, cleared when none is listed. Nothing a send moved is among them,
so the gate never holds a send on that finding. The statement's units
are kept with the machine's health (migration 0080) and node show says
them.

internal/inventory/health.go

@@ -43,6 +43,27 @@ type NodeHealth struct {
 	// Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an
 	// engine older than that judging.
 	Network *NetworkHealth
+	// Units is the machine's service managers as its engine said them (novox/hq issue 315); nil from an
+	// engine older than that reading.
+	Units *UnitsHealth
+}
+
+// UnitsHealth is a machine's service managers as its engine said them (issue 315): running, degraded or
+// unknown, and each failed unit no module places.
+type UnitsHealth struct {
+	State  string       `json:"state"`
+	Failed []FailedUnit `json:"failed"`
+	Unread []string     `json:"unread,omitempty"`
+}
+
+// FailedUnit is one failed unit no module places: the machine's own.
+type FailedUnit struct {
+	Unit     string    `json:"unit"`
+	Scope    string    `json:"scope"`
+	Load     string    `json:"load,omitempty"`
+	Result   string    `json:"result,omitempty"`
+	Resource string    `json:"resource,omitempty"`
+	Since    time.Time `json:"since"`
 }
 
 // NetworkHealth is a machine's networking as its engine said it (ADR 0241).
@@ -83,7 +104,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error)
 
 func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) {
 	rows, err := i.store.Pool().Query(ctx,
-		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network
+		`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network, h.units
 		 from node_health h join node n on n.id = h.node
 		 where $1 = '' or n.name = $1`, only)
 	if err != nil {
@@ -93,8 +114,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 	out := map[string]NodeHealth{}
 	for rows.Next() {
 		var h NodeHealth
-		var resources, streaks, network []byte
-		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil {
+		var resources, streaks, network, units []byte
+		if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network, &units); err != nil {
 			return nil, err
 		}
 		if err := json.Unmarshal(resources, &h.Resources); err != nil {
@@ -108,6 +129,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
 				return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err)
 			}
 		}
+		if len(units) > 0 {
+			if err := json.Unmarshal(units, &h.Units); err != nil {
+				return nil, fmt.Errorf("%s's units cannot be read: %w", h.Node, err)
+			}
+		}
 		out[h.Node] = h
 	}
 	return out, rows.Err()
@@ -136,19 +162,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error
 			return false, err
 		}
 	}
+	var units []byte
+	if h.Units != nil {
+		if units, err = json.Marshal(h.Units); err != nil {
+			return false, err
+		}
+	}
 	heard := h.HeardAt
 	if heard.IsZero() {
 		heard = time.Now()
 	}
 	var node string
 	err = i.store.Pool().QueryRow(ctx,
-		`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network)
-		 select id, $2, $3, $4, $5, $6, $7 from node where name = $1
+		`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network, units)
+		 select id, $2, $3, $4, $5, $6, $7, $8 from node where name = $1
 		 on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at,
 		     heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks,
-		     network = excluded.network
+		     network = excluded.network, units = excluded.units
 		   where node_health.said_at <= excluded.said_at
-		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node)
+		 returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network, units).Scan(&node)
 	if errors.Is(err, pgx.ErrNoRows) {
 		if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil {
 			return false, nerr

internal/inventory/migrations/0080-a-failed-unit-is-never-silent.sql

@@ -0,0 +1,9 @@
+-- A failed unit is never silent (novox/hq issue 315).
+--
+-- Beside the state of every long-running resource and its networking, each machine's node-engine states
+-- its service managers: running, degraded or unknown, and every failed unit no module places — a mount of
+-- the machine's own table, a unit a removed package left behind. A module's failed unit is among the
+-- resources, as that module's. Kept with the machine's newest statement, replaced with it, so `node show`
+-- and a controller started again read the same word. Null for a machine whose node-engine is older than
+-- this reading: its units are not known — never healthy, never a reason to raise anything.
+alter table node_health add column units jsonb;

internal/link/protocol.go

@@ -431,6 +431,42 @@ type Health struct {
 	// Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine
 	// older than that judging, which is "not known", never healthy.
 	Network *NetworkHealth `json:"network,omitempty"`
+	// Units is the machine's service managers as its engine read them (novox/hq issue 315): whether any
+	// unit failed, and each failed unit no module places. A module's failed unit is among Resources, of
+	// kind KindUnit. Nil from an engine older than that reading: not known, never healthy.
+	Units *UnitsHealth `json:"units,omitempty"`
+}
+
+// KindUnit is a module's unit its service manager says failed (issue 315): its package's unit, a unit
+// file it writes, a service whose lifecycle is the machine's — said unhealthy while it stays failed.
+const KindUnit = "unit"
+
+// The states of a machine's service managers (issue 315).
+const (
+	UnitsRunning  = "running"
+	UnitsDegraded = "degraded"
+)
+
+// UnitsHealth is the machine's service managers in one statement (issue 315). The node-engine's own
+// (mesh-host internal/link UnitsHealth).
+type UnitsHealth struct {
+	// State is running, degraded or unknown.
+	State string `json:"state"`
+	// Failed is every unit failed on two looks in a row that no module places: the machine's own.
+	Failed []FailedUnit `json:"failed"`
+	Unread []string     `json:"unread,omitempty"`
+}
+
+// FailedUnit is one failed unit no module places.
+type FailedUnit struct {
+	Unit string `json:"unit"`
+	// Scope is system, or user: an account's own manager.
+	Scope  string `json:"scope"`
+	Load   string `json:"load,omitempty"`
+	Result string `json:"result,omitempty"`
+	// Resource is the mesh's own resource naming it, when the mesh placed it in its own right.
+	Resource string    `json:"resource,omitempty"`
+	Since    time.Time `json:"since"`
 }
 
 // NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since