novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 6 Releases Wiki Activity

Merge pull request 'Raise a failed unit, the module's and the machine's (hq issue 315)' (#137) from fix/315-a-failed-unit-is-a-condition into main

Browse Source
This commit was merged in pull request #137.
This commit is contained in:
mesh-admin
2026-10-08 09:40:23 +00:00
parent 6910417090 6c98e7ea51
commit 4d337ff897
8 changed files with 382 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/inventory/health.go
+39 -7
View File
@@ -43,6 +43,27 @@ type NodeHealth struct {
// Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an
// engine older than that judging.
Network *NetworkHealth
// Units is the machine's service managers as its engine said them (novox/hq issue 315); nil from an
// engine older than that reading.
Units *UnitsHealth
}
// UnitsHealth is a machine's service managers as its engine said them (issue 315): running, degraded or
// unknown, and each failed unit no module places.
type UnitsHealth struct {
State string `json:"state"`
Failed []FailedUnit `json:"failed"`
Unread []string `json:"unread,omitempty"`
}
// FailedUnit is one failed unit no module places: the machine's own.
type FailedUnit struct {
Unit string `json:"unit"`
Scope string `json:"scope"`
Load string `json:"load,omitempty"`
Result string `json:"result,omitempty"`
Resource string `json:"resource,omitempty"`
Since time.Time `json:"since"`
}
// NetworkHealth is a machine's networking as its engine said it (ADR 0241).
@@ -83,7 +104,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error)
func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) {
rows, err := i.store.Pool().Query(ctx,
`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network
`select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network, h.units
from node_health h join node n on n.id = h.node
where $1 = '' or n.name = $1`, only)
if err != nil {
@@ -93,8 +114,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
out := map[string]NodeHealth{}
for rows.Next() {
var h NodeHealth
var resources, streaks, network []byte
if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil {
var resources, streaks, network, units []byte
if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network, &units); err != nil {
return nil, err
}
if err := json.Unmarshal(resources, &h.Resources); err != nil {
@@ -108,6 +129,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe
return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err)
}
}
if len(units) > 0 {
if err := json.Unmarshal(units, &h.Units); err != nil {
return nil, fmt.Errorf("%s's units cannot be read: %w", h.Node, err)
}
}
out[h.Node] = h
}
return out, rows.Err()
@@ -136,19 +162,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error
return false, err
}
}
var units []byte
if h.Units != nil {
if units, err = json.Marshal(h.Units); err != nil {
return false, err
}
}
heard := h.HeardAt
if heard.IsZero() {
heard = time.Now()
}
var node string
err = i.store.Pool().QueryRow(ctx,
`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network)
select id, $2, $3, $4, $5, $6, $7 from node where name = $1
`insert into node_health (node, contract, said_at, heard_at, resources, streaks, network, units)
select id, $2, $3, $4, $5, $6, $7, $8 from node where name = $1
on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at,
heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks,
network = excluded.network
network = excluded.network, units = excluded.units
where node_health.said_at <= excluded.said_at
returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node)
returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network, units).Scan(&node)
if errors.Is(err, pgx.ErrNoRows) {
if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil {
return false, nerr
internal/inventory/migrations/0080-a-failed-unit-is-never-silent.sql
+9
View File
@@ -0,0 +1,9 @@
-- A failed unit is never silent (novox/hq issue 315).
--
-- Beside the state of every long-running resource and its networking, each machine's node-engine states
-- its service managers: running, degraded or unknown, and every failed unit no module places — a mount of
-- the machine's own table, a unit a removed package left behind. A module's failed unit is among the
-- resources, as that module's. Kept with the machine's newest statement, replaced with it, so `node show`
-- and a controller started again read the same word. Null for a machine whose node-engine is older than
-- this reading: its units are not known — never healthy, never a reason to raise anything.
alter table node_health add column units jsonb;
internal/link/protocol.go
+36
View File
@@ -431,6 +431,42 @@ type Health struct {
// Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine
// older than that judging, which is "not known", never healthy.
Network *NetworkHealth `json:"network,omitempty"`
// Units is the machine's service managers as its engine read them (novox/hq issue 315): whether any
// unit failed, and each failed unit no module places. A module's failed unit is among Resources, of
// kind KindUnit. Nil from an engine older than that reading: not known, never healthy.
Units *UnitsHealth `json:"units,omitempty"`
}
// KindUnit is a module's unit its service manager says failed (issue 315): its package's unit, a unit
// file it writes, a service whose lifecycle is the machine's — said unhealthy while it stays failed.
const KindUnit = "unit"
// The states of a machine's service managers (issue 315).
const (
UnitsRunning = "running"
UnitsDegraded = "degraded"
)
// UnitsHealth is the machine's service managers in one statement (issue 315). The node-engine's own
// (mesh-host internal/link UnitsHealth).
type UnitsHealth struct {
// State is running, degraded or unknown.
State string `json:"state"`
// Failed is every unit failed on two looks in a row that no module places: the machine's own.
Failed []FailedUnit `json:"failed"`
Unread []string `json:"unread,omitempty"`
}
// FailedUnit is one failed unit no module places.
type FailedUnit struct {
Unit string `json:"unit"`
// Scope is system, or user: an account's own manager.
Scope string `json:"scope"`
Load string `json:"load,omitempty"`
Result string `json:"result,omitempty"`
// Resource is the mesh's own resource naming it, when the mesh placed it in its own right.
Resource string `json:"resource,omitempty"`
Since time.Time `json:"since"`
}
// NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since