diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 750f81fd..7f764557 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -348,6 +348,10 @@ func aboutTheMachine(machine string, moved []string, since time.Time, f gateFact } case coreBehind: // Not what the send moved: said nowhere against it. + case c.Kind == kindMachineUnits: + // **The machine's own failed units** (issue 315): no module places any of them, so nothing a + // send moved is among them — a module's failed unit is that module's own condition. Said + // nowhere against the send. case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine: // **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send // did not move, or the machine cannot reach another that is down. Nothing the send did; the diff --git a/cmd/mesh-controller/machine_units.go b/cmd/mesh-controller/machine_units.go new file mode 100644 index 00000000..2f51b57b --- /dev/null +++ b/cmd/mesh-controller/machine_units.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A failed unit is never silent (novox/hq issue 315). +// +// **A module's failed unit was never raised, and neither was the machine's.** Liveness judges what a +// module runs long-lived; a module whose daemon is a package's unit, started by D-Bus activation, states +// no service, and its unit failed at every start while the machine read healthy. The profile's +// `service-manager` said `degraded`, and nothing raised that. +// +// The node-engine now reads every failed unit in the managers the mesh places units in, and says whose +// each is: +// +// - **a module's** — the declaration states it, writes its file, or installs the package its file +// belongs to — is among the module's resources, unhealthy, of kind `unit`: the module's own +// `module...unhealthy`, naming the unit, raised on the second statement and held by +// the gate as any unhealthy resource is (ADR 0240 §4); +// - **the machine's** — no module places it: a mount of the machine's own table, a unit a removed +// package left behind — is one finding for the machine, `machine..units`, listing them, so a +// degraded service manager is never silent. A warning; cleared on the first statement that lists none. +// +// The engine applies the two-look rule itself (ADR 0241 §2), so the machine's finding is raised on the +// statement that first says it. It is nothing a send did — no module places what it lists — so the gate +// never holds a send on it. An engine older than this reading says nothing of its units, and nothing is +// raised or cleared for it. + +// The condition a machine's own failed units raise. +const ( + kindMachineUnits = "machine-units" + sourceUnits = "units" +) + +// unitsKept is a statement's units as the inventory keeps them. +func unitsKept(u *link.UnitsHealth) *inventory.UnitsHealth { + if u == nil { + return nil + } + out := &inventory.UnitsHealth{State: u.State, Failed: []inventory.FailedUnit{}, Unread: u.Unread} + for _, f := range u.Failed { + out.Failed = append(out.Failed, inventory.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, Result: f.Result, + Resource: f.Resource, Since: f.Since}) + } + return out +} + +// judgeUnits raises the machine's own failed units as one finding, or clears it when the statement lists +// none. A statement that says nothing of units — an older engine — leaves it as it is. +func judgeUnits(ctx context.Context, k *conditions.Keeper, node string, u *link.UnitsHealth) error { + if k == nil || u == nil { + return nil + } + o, raise := machineUnitsObservation(node, u) + if raise { + _, err := k.Observe(ctx, o) + return err + } + open, err := k.Open(ctx) + if err != nil { + return err + } + for _, c := range open { + if c.Kind == kindMachineUnits && c.Key == o.Key() { + why := fmt.Sprintf("%s's service managers list no failed unit the mesh does not place", node) + if u.State == link.UnitsRunning { + why = fmt.Sprintf("%s's service managers are running, no unit failed", node) + } + _, err := k.Clear(ctx, c.Key, why) + return err + } + } + return nil +} + +// machineUnitsObservation is the machine's own failed units in one finding, and whether there are any. +// The summary names each unit and its manager, which is what a person looks for; how each failed and +// since when is evidence. Pure. +func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Observation, bool) { + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "units", Kind: kindMachineUnits, + Machine: node, Severity: conditions.Warning, Source: sourceUnits} + if len(u.Failed) == 0 { + return o, false + } + var names, said []string + for _, f := range u.Failed { + where := "" + if f.Scope == "user" { + where = " (the account's own manager)" + } + names = append(names, f.Unit+where) + line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, orSystem(f.Scope), orNotSaid(f.Result)) + if f.Load != "" && f.Load != "loaded" { + line += ", its unit " + f.Load + } + if f.Resource != "" { + line += ", named by the mesh's own " + f.Resource + } + line += ", since " + f.Since.UTC().Format("2006-01-02 15:04:05 MST") + said = append(said, line) + } + o.Summary = fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. "+ + "Each is the machine's own: mend or remove it there, or have a module place it", + node, len(u.Failed), strings.Join(names, ", ")) + o.Said = strings.Join(said, "; ") + return o, true +} + +func orSystem(scope string) string { + if scope == "" { + return "system" + } + return scope +} + +// unitsLines is what `node show` says of a machine's service managers. +func unitsLines(h inventory.NodeHealth, had bool, _ time.Time) []string { + if !had || h.Units == nil { + return []string{" its node-engine does not say which units failed — it is older than that reading (issue 315)"} + } + out := []string{" its service managers: " + h.Units.State} + for _, f := range h.Units.Failed { + line := fmt.Sprintf(" failed %s (%s) — no module places it", f.Unit, orSystem(f.Scope)) + if f.Result != "" { + line += ", " + f.Result + } + out = append(out, line) + } + for _, r := range h.Resources { + if r.Kind == link.KindUnit { + out = append(out, fmt.Sprintf(" failed %s — %s's, %s", r.Target, r.Module, r.Reason)) + } + } + for _, w := range h.Units.Unread { + out = append(out, " unread "+w) + } + return out +} diff --git a/cmd/mesh-controller/machine_units_test.go b/cmd/mesh-controller/machine_units_test.go new file mode 100644 index 00000000..162a7ffa --- /dev/null +++ b/cmd/mesh-controller/machine_units_test.go @@ -0,0 +1,128 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +// A failed unit is never silent (novox/hq issue 315, "how it is checked"): a module's failed unit raises +// the module's condition on the second statement, naming the unit; the machine's own failed units raise +// one finding for the machine, listing them, cleared when none is listed; an engine that says nothing of +// its units raises and clears nothing; and the gate never holds a send on the machine's own. + +// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged +// user unit, the module's; two mounts of the machine's own table and a unit a removed package left +// behind, the machine's. +func desktop(at time.Time) link.Health { + return link.Health{Contract: link.ReadinessContract, At: at, + Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit, + Target: "openrazer-daemon.service", State: link.StateUnhealthy, + Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}}, + Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{ + {Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0}, + {Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0}, + {Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0}, + }}} +} + +func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + openConditions := func() map[string]conditions.Condition { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range list { + out[c.Key] = c + } + return out + } + say := func(at time.Time, h link.Health) { + t.Helper() + if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil { + t.Fatal(err) + } + } + + say(h0, desktop(h0)) + got := openConditions() + machine, raised := got["machine.laptop.units"] + if !raised || len(got) != 1 { + t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got) + } + for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} { + if !strings.Contains(machine.Summary, unit) { + t.Errorf("the finding names %s: %s", unit, machine.Summary) + } + } + if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning { + t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine) + } + + say(h0.Add(time.Minute), desktop(h0.Add(time.Minute))) + got = openConditions() + module, raised := got["module.openrazer.laptop.unhealthy"] + if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") || + !strings.Contains(module.Summary, "account's own service manager") { + t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got) + } + + kept, had, err := inv.HealthOf(ctx, "laptop") + if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 { + t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err) + } + lines := strings.Join(unitsLines(kept, had, h0), "\n") + for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it", + "openrazer-daemon.service — openrazer's"} { + if !strings.Contains(lines, want) { + t.Errorf("node show does not say %q:\n%s", want, lines) + } + } + + // An older engine says nothing of its units: nothing cleared on its word. + older := desktop(h0.Add(2 * time.Minute)) + older.Units = nil + say(h0.Add(2*time.Minute), older) + if _, still := openConditions()["machine.laptop.units"]; !still { + t.Fatal("a statement saying nothing of units cleared the machine's finding") + } + + // The operator mends the mounts and removes the leftover link; the module's unit still fails. + mended := desktop(h0.Add(3 * time.Minute)) + mended.Units.Failed = []link.FailedUnit{} + say(h0.Add(3*time.Minute), mended) + got = openConditions() + if _, still := got["machine.laptop.units"]; still { + t.Fatalf("no failed unit of the machine's own, still open: %v", got) + } + if _, still := got["module.openrazer.laptop.unhealthy"]; !still { + t.Fatalf("the module's unit still fails, and its condition stands: %v", got) + } + + // And the module's unit mended: running, nothing open. + well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute), + Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}} + say(h0.Add(4*time.Minute), well) + if got = openConditions(); len(got) != 0 { + t.Fatalf("a running service manager leaves nothing open: %v", got) + } +} + +func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) { + since := h0 + units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, + Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits} + if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true, + open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 { + t.Fatalf("the machine's own failed units held a send: %+v", w) + } +} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 9aa8217f..25e75e05 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -92,7 +92,7 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke } } stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At, - HeardAt: now, Resources: resources, Streaks: streaks, Network: network}) + HeardAt: now, Resources: resources, Streaks: streaks, Network: network, Units: unitsKept(h.Units)}) if err != nil || !stored { if err == nil { healthRefused.Add(1) @@ -103,6 +103,14 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke return nil } err = judgeModuleHealth(ctx, inv, k, node, unhealthy, streaks, now) + // And the machine's own failed units (issue 315): what no module places, one finding for the machine. + if uerr := judgeUnits(ctx, k, node, h.Units); uerr != nil { + if err == nil { + err = uerr + } else { + err = fmt.Errorf("%w; %v", err, uerr) + } + } // And every machine's network, from every machine's newest statement (ADR 0241): a statement about one // machine can hold another's finding, or release it. if nerr := judgeNetworks(ctx, inv, k, now); nerr != nil { @@ -235,6 +243,14 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation { var words, said []string for _, r := range rs { + if r.Kind == link.KindUnit { + // A failed unit is named by the unit, which is what a person looks for (issue 315); the + // resource that places it — a package, a file — is evidence. + words = append(words, fmt.Sprintf("its unit %s %s", r.Target, r.Reason)) + said = append(said, fmt.Sprintf("%s (unit %s, placed by %s): %s, since %s", r.Target, r.Target, r.Resource, + orNotSaid(r.Reason), r.Since.UTC().Format("2006-01-02 15:04:05 MST"))) + continue + } words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r))) said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s", r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts, diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index d784e574..6a50adc4 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -634,6 +634,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error for _, line := range networkLines(h, had, time.Now()) { fmt.Println(line) } + for _, line := range unitsLines(h, had, time.Now()) { + fmt.Println(line) + } } held, err := inv.Profile(ctx, name) diff --git a/internal/inventory/health.go b/internal/inventory/health.go index fe31d5cf..8fca6b8a 100644 --- a/internal/inventory/health.go +++ b/internal/inventory/health.go @@ -43,6 +43,27 @@ type NodeHealth struct { // Network is the machine's own networking as its engine said it (novox/hq ADR 0241); nil from an // engine older than that judging. Network *NetworkHealth + // Units is the machine's service managers as its engine said them (novox/hq issue 315); nil from an + // engine older than that reading. + Units *UnitsHealth +} + +// UnitsHealth is a machine's service managers as its engine said them (issue 315): running, degraded or +// unknown, and each failed unit no module places. +type UnitsHealth struct { + State string `json:"state"` + Failed []FailedUnit `json:"failed"` + Unread []string `json:"unread,omitempty"` +} + +// FailedUnit is one failed unit no module places: the machine's own. +type FailedUnit struct { + Unit string `json:"unit"` + Scope string `json:"scope"` + Load string `json:"load,omitempty"` + Result string `json:"result,omitempty"` + Resource string `json:"resource,omitempty"` + Since time.Time `json:"since"` } // NetworkHealth is a machine's networking as its engine said it (ADR 0241). @@ -83,7 +104,7 @@ func (i *Inventory) Healths(ctx context.Context) (map[string]NodeHealth, error) func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHealth, error) { rows, err := i.store.Pool().Query(ctx, - `select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network + `select n.name, h.contract, h.said_at, h.heard_at, h.resources, h.streaks, h.network, h.units from node_health h join node n on n.id = h.node where $1 = '' or n.name = $1`, only) if err != nil { @@ -93,8 +114,8 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe out := map[string]NodeHealth{} for rows.Next() { var h NodeHealth - var resources, streaks, network []byte - if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network); err != nil { + var resources, streaks, network, units []byte + if err := rows.Scan(&h.Node, &h.Contract, &h.SaidAt, &h.HeardAt, &resources, &streaks, &network, &units); err != nil { return nil, err } if err := json.Unmarshal(resources, &h.Resources); err != nil { @@ -108,6 +129,11 @@ func (i *Inventory) healths(ctx context.Context, only string) (map[string]NodeHe return nil, fmt.Errorf("%s's network health cannot be read: %w", h.Node, err) } } + if len(units) > 0 { + if err := json.Unmarshal(units, &h.Units); err != nil { + return nil, fmt.Errorf("%s's units cannot be read: %w", h.Node, err) + } + } out[h.Node] = h } return out, rows.Err() @@ -136,19 +162,25 @@ func (i *Inventory) RecordHealth(ctx context.Context, h NodeHealth) (bool, error return false, err } } + var units []byte + if h.Units != nil { + if units, err = json.Marshal(h.Units); err != nil { + return false, err + } + } heard := h.HeardAt if heard.IsZero() { heard = time.Now() } var node string err = i.store.Pool().QueryRow(ctx, - `insert into node_health (node, contract, said_at, heard_at, resources, streaks, network) - select id, $2, $3, $4, $5, $6, $7 from node where name = $1 + `insert into node_health (node, contract, said_at, heard_at, resources, streaks, network, units) + select id, $2, $3, $4, $5, $6, $7, $8 from node where name = $1 on conflict (node) do update set contract = excluded.contract, said_at = excluded.said_at, heard_at = excluded.heard_at, resources = excluded.resources, streaks = excluded.streaks, - network = excluded.network + network = excluded.network, units = excluded.units where node_health.said_at <= excluded.said_at - returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network).Scan(&node) + returning node`, h.Node, h.Contract, h.SaidAt, heard, resources, streaks, network, units).Scan(&node) if errors.Is(err, pgx.ErrNoRows) { if _, nerr := i.NodeByName(ctx, h.Node); nerr != nil { return false, nerr diff --git a/internal/inventory/migrations/0080-a-failed-unit-is-never-silent.sql b/internal/inventory/migrations/0080-a-failed-unit-is-never-silent.sql new file mode 100644 index 00000000..58827fa8 --- /dev/null +++ b/internal/inventory/migrations/0080-a-failed-unit-is-never-silent.sql @@ -0,0 +1,9 @@ +-- A failed unit is never silent (novox/hq issue 315). +-- +-- Beside the state of every long-running resource and its networking, each machine's node-engine states +-- its service managers: running, degraded or unknown, and every failed unit no module places — a mount of +-- the machine's own table, a unit a removed package left behind. A module's failed unit is among the +-- resources, as that module's. Kept with the machine's newest statement, replaced with it, so `node show` +-- and a controller started again read the same word. Null for a machine whose node-engine is older than +-- this reading: its units are not known — never healthy, never a reason to raise anything. +alter table node_health add column units jsonb; diff --git a/internal/link/protocol.go b/internal/link/protocol.go index ad27bec8..5d883a92 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -431,6 +431,42 @@ type Health struct { // Network is the machine's own networking, judged by its engine (novox/hq ADR 0241); nil from an engine // older than that judging, which is "not known", never healthy. Network *NetworkHealth `json:"network,omitempty"` + // Units is the machine's service managers as its engine read them (novox/hq issue 315): whether any + // unit failed, and each failed unit no module places. A module's failed unit is among Resources, of + // kind KindUnit. Nil from an engine older than that reading: not known, never healthy. + Units *UnitsHealth `json:"units,omitempty"` +} + +// KindUnit is a module's unit its service manager says failed (issue 315): its package's unit, a unit +// file it writes, a service whose lifecycle is the machine's — said unhealthy while it stays failed. +const KindUnit = "unit" + +// The states of a machine's service managers (issue 315). +const ( + UnitsRunning = "running" + UnitsDegraded = "degraded" +) + +// UnitsHealth is the machine's service managers in one statement (issue 315). The node-engine's own +// (mesh-host internal/link UnitsHealth). +type UnitsHealth struct { + // State is running, degraded or unknown. + State string `json:"state"` + // Failed is every unit failed on two looks in a row that no module places: the machine's own. + Failed []FailedUnit `json:"failed"` + Unread []string `json:"unread,omitempty"` +} + +// FailedUnit is one failed unit no module places. +type FailedUnit struct { + Unit string `json:"unit"` + // Scope is system, or user: an account's own manager. + Scope string `json:"scope"` + Load string `json:"load,omitempty"` + Result string `json:"result,omitempty"` + // Resource is the mesh's own resource naming it, when the mesh placed it in its own right. + Resource string `json:"resource,omitempty"` + Since time.Time `json:"since"` } // NetworkHealth is a machine's networking in one statement (ADR 0241): the worst of its parts, since