mesh-delivery/modules/mesh-delivery-watchman/internal/watch/nodehealth.go

package watch

import (
	"fmt"
	"maps"
	"slices"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// A node's own health report (novox/hq ADR 0241, issue 315, the operator's alert table of 2026-10-29): each node's engine
// writes `mesh-host_health.<node>` when what it says changes, and the watchman raises from it what the controller raised
// from the passing health message, under the same keys and words (mesh-control cmd/mesh-controller machine_network.go,
// machine_units.go, module_health.go at e04846e9):
//   - machine-network `machine.<node>.network`: the machine's own unhealthy parts; urgent on the control node or for the bus;
//   - network-rewritten `machine.<node>[.<owner>].rewritten`: the resolver file rewritten by another program;
//   - network-unreachable `machine.<node>.unreachable`: two or more nodes' tunnels no longer see it (their peers' handshake),
//     said once there, never while its own network or its silence says it;
//   - machine-units `machine.<node>.units`: failed units no module places;
//   - relogin-needed `module.<module>.<node>.relogin-needed` and directory-used-as-found
//     `module.<module>.<node>.directory-used-as-found`;
//   - node-account-fallback `machine.<node>.node-account-fallback`: the engine fell back to its node's old login during a
//     rotation (novox/hq ADR 0583 §6), raised while its report carries the fall back and cleared once it no longer does.
//
// A failing part that points at other mesh machines (NetworkPart.Targets: the hub's node, each resolver's node) is held
// there, as the controller holds it: when every machine it points at is down on the record (silent, its own network
// unhealthy, or two machines failing toward it), it is that machine's `unreachable`, said once there, and no finding of
// the reporter's. A tunnel peer two or more nodes no longer see points at its node the same way.

// The kinds a node's health report raises.
const (
	KindMachineNetwork     = "machine-network"
	KindNetworkRewritten   = "network-rewritten"
	KindNetworkUnreachable = "network-unreachable"
	KindMachineUnits       = "machine-units"
	KindReloginNeeded      = "relogin-needed"
	KindUsedAsFound        = "directory-used-as-found"
	KindAccountFallback    = delivery.NodeAccountFallbackFinding
)

// The parts and states a report names (mesh-control internal/link).
const (
	partResolvConf = "resolv-conf"
	partNames      = "names"
	partTunnel     = "tunnel"
	partBus        = "bus"
	stateUnhealthy = "unhealthy"
)

func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") }

// NodeHealth is a change of a node's report: written, or deleted (value nil).
func (w *Watchman) NodeHealth(node string, value []byte, op Op) {
	w.mu.Lock()
	defer w.mu.Unlock()
	if op == Put {
		h, err := delivery.DecodeNodeHealth(value)
		if err != nil {
			w.say("the health report of %s cannot be read, and is left: %v", node, err)
			return
		}
		w.health[node] = h
	} else {
		delete(w.health, node)
	}
	w.nodeFindings(node)
	w.networks()
}

// TunnelKey is a node's own tunnel key, from its record in its have: whose peers see it is read by it.
func (w *Watchman) tunnelKey(node string, record *delivery.NodeRecord) {
	key := ""
	if record != nil {
		key = record.TunnelKey
	}
	if w.tunnels[node] == key {
		return
	}
	if key == "" {
		delete(w.tunnels, node)
	} else {
		w.tunnels[node] = key
	}
	w.networks()
}

// nodeFindings writes and deletes one node's own findings from its report.
func (w *Watchman) nodeFindings(node string) {
	h, ok := w.health[node]
	want := map[string]bool{}
	if ok {
		if f, raise := unitsOf(node, h.Units); raise {
			want[f.Key()] = true
			w.find(f)
		}
		for _, f := range reloginOf(node, h.Relogin) {
			want[f.Key()] = true
			w.find(f)
		}
		for _, f := range foundOf(node, h.Found) {
			want[f.Key()] = true
			w.find(f)
		}
		if f, raise := fallbackOf(node, h.Fallback); raise {
			want[f.Key()] = true
			w.find(f)
		}
	}
	for key := range w.findings {
		if !want[key] && w.ownsHealthKey(key, node) {
			w.unfind(key)
		}
	}
}

// ownsHealthKey says a finding key is one of node's own report findings.
func (w *Watchman) ownsHealthKey(key, node string) bool {
	f, err := DecodeFinding(w.findings[key])
	if err != nil || f.Machine != node || f.Source != "health-report" {
		return false
	}
	return f.Kind != KindNetworkUnreachable && f.Kind != KindMachineNetwork && f.Kind != KindNetworkRewritten
}

func (w *Watchman) rewrittenOf(m string, p delivery.NetworkPart, all []delivery.NetworkPart) Finding {
	writer := ""
	if p.Writer != "" {
		writer = " (" + p.Writer + ")"
	}
	cost := "the names still resolve through what it wrote"
	said := []string{p.Part + ": " + p.Said}
	for _, q := range all {
		if q.Part == partNames {
			cost = strings.TrimSuffix(q.Reason, ".")
			said = append(said, q.Part+": "+q.Said)
		}
	}
	id := m
	summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+
		"back at its next reconcile, or that program gives it back", m, writer, cost)
	if p.Owner != "" {
		id = m + "." + p.Owner
		summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+
			"writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost)
	}
	severity := Warning
	if w.control[m] {
		severity = Urgent
	}
	return Finding{Scope: ScopeMachine, ID: id, Token: "rewritten", Kind: KindNetworkRewritten, Machine: m, Severity: severity,
		Source: "health-report", Since: p.Since, Summary: summary,
		Said: fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))}
}

// unitsOf is a machine's failed units no module places, one finding.
func unitsOf(node string, u *delivery.UnitsHealth) (Finding, bool) {
	if u == nil || len(u.Failed) == 0 {
		return Finding{}, false
	}
	var names, said []string
	since := u.Failed[0].Since
	for _, f := range u.Failed {
		where := ""
		if f.Scope == "user" {
			where = " (the account's own manager)"
		}
		names = append(names, f.Unit+where)
		scope, result := f.Scope, f.Result
		if scope == "" {
			scope = "system"
		}
		if result == "" {
			result = "no reason said"
		}
		line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result)
		if f.Load != "" && f.Load != "loaded" {
			line += ", its unit " + f.Load
		}
		if f.Resource != "" {
			line += ", named by the mesh's own " + f.Resource
		}
		said = append(said, line+", since "+stampSec(f.Since))
		if f.Since.Before(since) {
			since = f.Since
		}
	}
	failed := "1 failed service"
	if len(u.Failed) > 1 {
		failed = fmt.Sprintf("%d failed services", len(u.Failed))
	}
	return Finding{Scope: ScopeMachine, ID: node, Token: "units", Kind: KindMachineUnits, Machine: node, Severity: Warning,
		Source: "health-report", Since: since,
		Summary: fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+
			"mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")),
		Said:     strings.Join(said, "; "),
		Headline: strings.ToUpper(failed[:1]) + failed[1:] + " on " + node,
		Needs:    fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node),
		Resolved: "No failed services on " + node + " any more"}, true
}

// reloginOf is one finding per module whose accounts wait for a new login.
func reloginOf(node string, accounts []delivery.NodeAccount) []Finding {
	by := map[string][]delivery.NodeAccount{}
	for _, a := range accounts {
		by[a.Module] = append(by[a.Module], a)
	}
	var out []Finding
	for _, module := range slices.Sorted(maps.Keys(by)) {
		var names []string
		since := by[module][0].Since
		for _, a := range by[module] {
			names = append(names, a.Account)
			if a.Since.Before(since) {
				since = a.Since
			}
		}
		sort.Strings(names)
		said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+
			"before it was; log out of %[1]s completely and log in again, or restart it", node, orModule(module), strings.Join(names, ", "))
		out = append(out, Finding{Scope: ScopeModule, ID: orModule(module) + "." + node, Kind: KindReloginNeeded, Machine: node,
			Severity: Warning, Source: "health-report", Since: since, Summary: said, Said: said})
	}
	return out
}

// foundOf is one finding per module with a directory used as found.
func foundOf(node string, dirs []delivery.NodeDirectory) []Finding {
	by := map[string][]delivery.NodeDirectory{}
	for _, d := range dirs {
		by[d.Module] = append(by[d.Module], d)
	}
	var out []Finding
	for _, module := range slices.Sorted(maps.Keys(by)) {
		var ids, why []string
		since := by[module][0].Since
		for _, d := range by[module] {
			ids = append(ids, d.Resource)
			why = append(why, d.Reason)
			if d.Since.Before(since) {
				since = d.Since
			}
		}
		m := orModule(module)
		said := fmt.Sprintf("%s on %s uses %s as found: %s", m, node, strings.Join(ids, ", "), strings.Join(why, "; "))
		out = append(out, Finding{Scope: ScopeModule, ID: m + "." + node, Kind: KindUsedAsFound, Machine: node, Severity: Warning,
			Source: "health-report", Since: since, Summary: said, Said: said,
			Headline: fmt.Sprintf("%s waits for a directory on %s", m, node),
			Explanation: fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+
				"and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, node, m),
			Needs:    "hand the directory over from the control-node, as the operator; the details name it and the line to type.",
			Resolved: fmt.Sprintf("%s's directory on %s is the mesh's", m, node)})
	}
	return out
}

// fallbackOf is a node's fall back to its old login during a rotation of its account (ADR 0583 §6), one finding.
func fallbackOf(node string, f *delivery.LoginFallback) (Finding, bool) {
	if f == nil || f.Node != node {
		return Finding{}, false
	}
	said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why)
	return Finding{Scope: ScopeMachine, ID: node, Token: KindAccountFallback, Kind: KindAccountFallback, Machine: node, Severity: Warning,
		Source: "health-report", Since: f.At, Summary: said, Said: said,
		Headline: fmt.Sprintf("%s fell back to its old bus login", node),
		Needs:    fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node),
		Resolved: fmt.Sprintf("%s no longer falls back to its old bus login", node)}, true
}

func orModule(m string) string {
	if m == "" {
		return "_"
	}
	return m
}

// pointed is one machine's failing part that points at another machine.
type pointed struct {
	from string
	part delivery.NetworkPart
}

// networks is every machine's network findings from every report together (the controller's networkObservations): the
// resolver file rewritten, each machine's own unhealthy parts, and the machine others point at, held there.
func (w *Watchman) networks() {
	machines := slices.Sorted(maps.Keys(w.health))
	unhealthy := func(m string) []delivery.NetworkPart {
		h := w.health[m]
		if h.Network == nil {
			return nil
		}
		var out []delivery.NetworkPart
		for _, p := range h.Network.Parts {
			if p.State == stateUnhealthy {
				out = append(out, p)
			}
		}
		return out
	}
	targets := func(m string, p delivery.NetworkPart) ([]string, bool) {
		var out []string
		for _, x := range p.Targets {
			if x == "" || x == m {
				return nil, false
			}
			if !slices.Contains(out, x) {
				out = append(out, x)
			}
		}
		return out, len(out) > 0
	}
	pointing := map[string][]pointed{}
	for _, m := range machines {
		for _, p := range unhealthy(m) {
			if xs, ok := targets(m, p); ok {
				for _, x := range xs {
					pointing[x] = append(pointing[x], pointed{from: m, part: p})
				}
			}
		}
	}
	// A tunnel peer a node no longer sees points at that peer's node, as a tunnel part toward it would.
	byKey := map[string]string{}
	for node, key := range w.tunnels {
		byKey[key] = node
	}
	for _, m := range machines {
		for _, p := range w.health[m].Peers {
			if x, ok := byKey[p.Key]; ok && x != m && !p.Seen {
				pointing[x] = append(pointing[x], pointed{from: m, part: delivery.NetworkPart{Part: partTunnel, State: stateUnhealthy,
					Said: "no handshake with " + x + " within five minutes", Targets: []string{x}}})
			}
		}
	}
	from := func(x string) []string {
		var out []string
		for _, pt := range pointing[x] {
			if !slices.Contains(out, pt.from) {
				out = append(out, pt.from)
			}
		}
		sort.Strings(out)
		return out
	}
	silent := func(x string) bool {
		_, ok := w.findings[FindingKey(ScopeMachine, x, KindSilent)]
		return ok
	}
	down := func(x string) bool { return silent(x) || len(unhealthy(x)) > 0 || len(from(x)) >= 2 }
	allDown := func(xs []string) bool {
		for _, x := range xs {
			if !down(x) {
				return false
			}
		}
		return len(xs) > 0
	}
	want := map[string]bool{}
	saysOwn := map[string]bool{}
	for _, m := range machines {
		parts := unhealthy(m)
		if len(parts) == 0 {
			continue
		}
		var own []delivery.NetworkPart
		var rewritten *delivery.NetworkPart
		for i, p := range parts {
			if p.Part == partResolvConf {
				rewritten = &parts[i]
				continue
			}
			if xs, ok := targets(m, p); ok && allDown(xs) {
				continue // held at the machines it points at
			}
			own = append(own, p)
		}
		if rewritten != nil {
			f := w.rewrittenOf(m, *rewritten, parts)
			want[f.Key()] = true
			w.find(f)
			kept := own[:0]
			for _, p := range own {
				if p.Part != partNames {
					kept = append(kept, p)
				}
			}
			own = kept
		}
		if len(own) == 0 {
			continue
		}
		var words, said []string
		severity := Warning
		for _, p := range own {
			words = append(words, p.Reason)
			said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said))
			if p.Part == partBus {
				severity = Urgent
			}
		}
		if w.control[m] {
			severity = Urgent
		}
		summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; "))
		if waiting := from(m); len(waiting) > 0 {
			severity = Urgent
			summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", "))
		}
		f := Finding{Scope: ScopeMachine, ID: m, Token: "network", Kind: KindMachineNetwork, Machine: m, Severity: severity,
			Source: "health-report", Since: w.health[m].Network.Since, Summary: summary, Said: strings.Join(said, " | ")}
		want[f.Key()] = true
		saysOwn[m] = true
		w.find(f)
	}
	for _, x := range slices.Sorted(maps.Keys(pointing)) {
		if !down(x) || saysOwn[x] || silent(x) {
			continue
		}
		var what, said []string
		for _, pt := range pointing[x] {
			wd := map[string]string{partTunnel: "the tunnel to it", partBus: "the bus on it", partNames: "its resolver"}[pt.part.Part]
			if wd == "" {
				wd = pt.part.Part
			}
			if !slices.Contains(what, wd) {
				what = append(what, wd)
			}
			said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said))
		}
		severity := Warning
		if w.control[x] || slices.Contains(what, "the bus on it") {
			severity = Urgent
		}
		f := Finding{Scope: ScopeMachine, ID: x, Token: "unreachable", Kind: KindNetworkUnreachable, Machine: x, Also: from(x),
			Severity: severity, Source: "health-report",
			Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from(x), ", "), strings.Join(what, ", ")),
			Said:    strings.Join(said, " | ")}
		want[f.Key()] = true
		w.find(f)
	}
	for key, value := range w.findings {
		f, err := DecodeFinding(value)
		if err != nil || f.Source != "health-report" {
			continue
		}
		if (f.Kind == KindMachineNetwork || f.Kind == KindNetworkRewritten || f.Kind == KindNetworkUnreachable) && !want[key] {
			w.unfind(key)
		}
	}
}