mesh-controller/cmd/mesh-controller/machine_units.go

package main

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// A failed unit is never silent (novox/hq issue 315).
//
// **A module's failed unit was never raised, and neither was the machine's.** Liveness judges what a
// module runs long-lived; a module whose daemon is a package's unit, started by D-Bus activation, states
// no service, and its unit failed at every start while the machine read healthy. The profile's
// `service-manager` said `degraded`, and nothing raised that.
//
// The node-engine now reads every failed unit in the managers the mesh places units in, and says whose
// each is:
//
//   - **a module's** — the declaration states it, writes its file, or installs the package its file
//     belongs to — is among the module's resources, unhealthy, of kind `unit`: the module's own
//     `module.<module>.<machine>.unhealthy`, naming the unit, raised on the second statement and held by
//     the gate as any unhealthy resource is (ADR 0240 §4);
//   - **the machine's** — no module places it: a mount of the machine's own table, a unit a removed
//     package left behind — is one finding for the machine, `machine.<m>.units`, listing them, so a
//     degraded service manager is never silent. A warning; cleared on the first statement that lists none.
//
// The engine applies the two-look rule itself (ADR 0241 §2), so the machine's finding is raised on the
// statement that first says it. It is nothing a send did — no module places what it lists — so the gate
// never holds a send on it. An engine older than this reading says nothing of its units, and nothing is
// raised or cleared for it.

// The condition a machine's own failed units raise.
const (
	kindMachineUnits = "machine-units"
	sourceUnits      = "units"
)

// unitsKept is a statement's units as the inventory keeps them.
func unitsKept(u *link.UnitsHealth) *inventory.UnitsHealth {
	if u == nil {
		return nil
	}
	out := &inventory.UnitsHealth{State: u.State, Failed: []inventory.FailedUnit{}, Unread: u.Unread}
	for _, f := range u.Failed {
		out.Failed = append(out.Failed, inventory.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, Result: f.Result,
			Resource: f.Resource, Since: f.Since})
	}
	return out
}

// judgeUnits raises the machine's own failed units as one finding, or clears it when the statement lists
// none. A statement that says nothing of units — an older engine — leaves it as it is.
func judgeUnits(ctx context.Context, k *conditions.Keeper, node string, u *link.UnitsHealth) error {
	if k == nil || u == nil {
		return nil
	}
	o, raise := machineUnitsObservation(node, u)
	if raise {
		_, err := k.Observe(ctx, o)
		return err
	}
	open, err := k.Open(ctx)
	if err != nil {
		return err
	}
	for _, c := range open {
		if c.Kind == kindMachineUnits && c.Key == o.Key() {
			why := fmt.Sprintf("%s's service managers list no failed unit the mesh does not place", node)
			if u.State == link.UnitsRunning {
				why = fmt.Sprintf("%s's service managers are running, no unit failed", node)
			}
			_, err := k.Clear(ctx, c.Key, why)
			return err
		}
	}
	return nil
}

// machineUnitsObservation is the machine's own failed units in one finding, and whether there are any.
// The summary names each unit and its manager, which is what a person looks for; how each failed and
// since when is evidence. Pure.
func machineUnitsObservation(node string, u *link.UnitsHealth) (conditions.Observation, bool) {
	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: "units", Kind: kindMachineUnits,
		Machine: node, Severity: conditions.Warning, Source: sourceUnits}
	if len(u.Failed) == 0 {
		return o, false
	}
	var names, said, plain []string
	for _, f := range u.Failed {
		plain = append(plain, unitPlainWords(f.Unit))
		where := ""
		if f.Scope == "user" {
			where = " (the account's own manager)"
		}
		names = append(names, f.Unit+where)
		line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, orSystem(f.Scope), orNotSaid(f.Result))
		if f.Load != "" && f.Load != "loaded" {
			line += ", its unit " + f.Load
		}
		if f.Resource != "" {
			line += ", named by the mesh's own " + f.Resource
		}
		line += ", since " + f.Since.UTC().Format("2006-01-02 15:04:05 MST")
		said = append(said, line)
	}
	o.Summary = fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. "+
		"Each is the machine's own: mend or remove it there, or have a module place it",
		node, len(u.Failed), strings.Join(names, ", "))
	o.Said = strings.Join(said, "; ")
	failed := "1 failed service"
	if len(u.Failed) > 1 {
		failed = fmt.Sprintf("%d failed services", len(u.Failed))
	}
	o.Headline = fmt.Sprintf("%s on %s", conditions.Capital(failed), node)
	o.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)
	o.Explanation = fmt.Sprintf("On %s, %s failed. No module manages them, so the mesh does not repair them.",
		node, namesWords(plain, 3))
	o.Actions = []conditions.Action{conditions.SilenceAction(o.Key())}
	o.Resolved = "No failed services on " + node + " any more"
	return o, true
}

func orSystem(scope string) string {
	if scope == "" {
		return "system"
	}
	return scope
}

// unitsLines is what `node show` says of a machine's service managers.
func unitsLines(h inventory.NodeHealth, had bool, _ time.Time) []string {
	if !had || h.Units == nil {
		return []string{"  its node-engine does not say which units failed — it is older than that reading (issue 315)"}
	}
	out := []string{"  its service managers: " + h.Units.State}
	for _, f := range h.Units.Failed {
		line := fmt.Sprintf("    failed     %s (%s) — no module places it", f.Unit, orSystem(f.Scope))
		if f.Result != "" {
			line += ", " + f.Result
		}
		out = append(out, line)
	}
	for _, r := range h.Resources {
		if r.Kind == link.KindUnit {
			out = append(out, fmt.Sprintf("    failed     %s — %s's, %s", r.Target, r.Module, r.Reason))
		}
	}
	for _, w := range h.Units.Unread {
		out = append(out, "    unread     "+w)
	}
	return out
}