package main import ( "context" "fmt" "strings" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A failed unit is never silent (novox/hq issue 315). // // **A module's failed unit was never raised, and neither was the machine's.** Liveness judges what a // module runs long-lived; a module whose daemon is a package's unit, started by D-Bus activation, states // no service, and its unit failed at every start while the machine read healthy. The profile's // `service-manager` said `degraded`, and nothing raised that. // // The node-engine now reads every failed unit in the managers the mesh places units in, and says whose // each is: // // - **a module's** — the declaration states it, writes its file, or installs the package its file // belongs to — is among the module's resources, unhealthy, of kind `unit`: the module's own // `module...unhealthy`, naming the unit, raised on the second statement and held by // the gate as any unhealthy resource is (ADR 0240 §4); // - **the machine's** — no module places it: a mount of the machine's own table, a unit a removed // package left behind — is one finding for the machine, `machine..units`, listing them, so a // degraded service manager is never silent. A warning; cleared on the first statement that lists none. // // The engine applies the two-look rule itself (ADR 0241 §2), so the machine's finding is raised on the // statement that first says it. It is nothing a send did — no module places what it lists — so the gate // never holds a send on it. An engine older than this reading says nothing of its units, and nothing is // raised or cleared for it. // The condition a machine's own failed units raise. const ( kindMachineUnits = "machine-units" sourceUnits = "units" ) // unitsKept is a statement's units as the inventory keeps them. func unitsKept(u *link.UnitsHealth) *inventory.UnitsHealth { if u == nil { return nil } out := &inventory.UnitsHealth{State: u.State, Failed: []inventory.FailedUnit{}, Unread: u.Unread} for _, f := range u.Failed { out.Failed = append(out.Failed, inventory.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, Result: f.Result, Resource: f.Resource, Since: f.Since}) } return out } // judgeUnits raises the machine's own failed units as one finding, or clears it when the statement lists // none. A statement that says nothing of units — an older engine — leaves it as it is. func judgeUnits(ctx context.Context, k *conditions.Keeper, node string, u *link.UnitsHealth) error { if k == nil || u == nil { return nil } o, raise := machineUnitsObservation(node, u) if raise { _, err := k.Observe(ctx, o) return err } open, err := k.Open(ctx) if err != nil { return err } for _, c := range open { if c.Kind == kindMachineUnits && c.Key == o.Key() { why := fmt.Sprintf("%s's service managers list no failed unit the mesh does not place", node) if u.State == link.UnitsRunning { why = fmt.Sprintf("%s's service managers are running, no unit failed", node) } _, err := k.Clear(ctx, c.Key, why) return err } } return nil } // machineUnitsObservation is the machine's own failed units in one finding, and whether there are any. // The summary names each unit and its manager, which is what a person looks for; how each failed and // since when is evidence. Pure. func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Observation, bool) { o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "units", Kind: kindMachineUnits, Machine: node, Severity: conditions.Warning, Source: sourceUnits} if len(u.Failed) == 0 { return o, false } var names, said, plain []string for _, f := range u.Failed { plain = append(plain, unitPlainWords(f.Unit)) where := "" if f.Scope == "user" { where = " (the account's own manager)" } names = append(names, f.Unit+where) line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, orSystem(f.Scope), orNotSaid(f.Result)) if f.Load != "" && f.Load != "loaded" { line += ", its unit " + f.Load } if f.Resource != "" { line += ", named by the mesh's own " + f.Resource } line += ", since " + f.Since.UTC().Format("2006-01-02 15:04:05 MST") said = append(said, line) } o.Summary = fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. "+ "Each is the machine's own: mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")) o.Said = strings.Join(said, "; ") failed := "1 failed service" if len(u.Failed) > 1 { failed = fmt.Sprintf("%d failed services", len(u.Failed)) } o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node) o.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node) o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them.", node, namesWords(plain, 3)) o.Actions = []conditions.Action{conditions.SilenceAction(o.Key())} o.Resolved = "No failed services on " + node + " any more" return o, true } func orSystem(scope string) string { if scope == "" { return "system" } return scope } // unitsLines is what `node show` says of a machine's service managers. func unitsLines(h inventory.NodeHealth, had bool, _ time.Time) []string { if !had || h.Units == nil { return []string{" its node-engine does not say which units failed — it is older than that reading (issue 315)"} } out := []string{" its service managers: " + h.Units.State} for _, f := range h.Units.Failed { line := fmt.Sprintf(" failed %s (%s) — no module places it", f.Unit, orSystem(f.Scope)) if f.Result != "" { line += ", " + f.Result } out = append(out, line) } for _, r := range h.Resources { if r.Kind == link.KindUnit { out = append(out, fmt.Sprintf(" failed %s — %s's, %s", r.Target, r.Module, r.Reason)) } } for _, w := range h.Units.Unread { out = append(out, " unread "+w) } return out }