Write each node's health report into its own key of mesh-host_health on a change, so the watchman raises the machine's alerts without the controller (issue 774, the operator's alert table)

Until now the six machine alerts came from a health message passing to the controller. The report is now kept in the
engine's declared state `health`, key `<node>`, created by the grants part as `acts` is. Only this node's engine
writes it, after comparing it on every look and only when what it says changed; it is read first after a restart.

It carries:
- the network parts (machine-network; network-rewritten through the resolver file part's writer and owner);
- the tunnel's peers, each seen within five minutes or not (network-unreachable);
- the failed units no module owns (machine-units);
- the accounts marked relogin needed (relogin-needed);
- the directories used as found (directory-used-as-found).

The streaks and the time of the look are left out, so a look alone writes nothing. A peer is reported seen or not,
never its handshake time, so a renewed handshake writes nothing.

The layout is internal/link's (NodeHealth) until it moves into mesh-sdk. Pins mesh-sdk 58a8324.

Tests:
- rows in TestOnTheHubAnyFreshPeerIsAHealthyTunnel: the peers, and a change only when one is seen or lost;
- a case in TestAHealthStatementIsSaidOnTheLinkOpenNow: what the report takes, and one write per change across a
  restart;
- the manifest's states in TestTheManifestConsumesTheBackupEvent.
This commit is contained in:
2026-10-29 22:56:14 +01:00
parent cbe8528f91
commit 65fbc1e6f8
10 changed files with 406 additions and 14 deletions
+46 -7
View File
@@ -32,6 +32,7 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
@@ -94,6 +95,18 @@ type Finding struct {
Writer string
// Toward is what the failure points at: TowardHub, or each resolver's address that failed.
Toward []string
// Peers is, for the tunnel, each peer of the mesh's interface and whether it was seen (Peer).
Peers []Peer
}
// Peer is one peer of the mesh's tunnel as this machine sees it: its public key, the addresses it is allowed, and
// whether it handshook within StaleHandshake. What the watchman reads to tell a machine nobody reaches from one whose
// tunnel alone is down (network-unreachable). Seen, not the handshake's time: the time moves every two minutes, the
// fact only when a peer comes or goes.
type Peer struct {
Key string `json:"key"`
Addresses []string `json:"addresses,omitempty"`
Seen bool `json:"seen"`
}
// Part is one part's state, as the statement says it.
@@ -115,6 +128,8 @@ type Statement struct {
Since time.Time `json:"since"`
At time.Time `json:"at"`
Parts []Part `json:"parts"`
// Peers are the tunnel's peers on the last look that read them, sorted by key; nil where the tunnel is not read.
Peers []Peer `json:"peers,omitempty"`
}
// Machine is what a look reads, replaced in tests.
@@ -266,6 +281,9 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
changed = true
}
st.State, st.Since = worst, j.overall.since
// The tunnel's peers as this look read them; a peer seen or lost is a change of its own (network-unreachable).
st.Peers = findings[PartTunnel].Peers
changed = changed || !samePeers(j.said.Peers, st.Peers)
j.said = st
return st, changed
}
@@ -435,14 +453,22 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
}
}
hub := ""
allowed := map[string][]string{}
for _, line := range strings.Split(ips, "\n") {
f := strings.Fields(line)
for _, prefix := range f[min(1, len(f)):] {
allowed[f[0]] = append(allowed[f[0]], prefix)
if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" {
hub = f[0]
}
}
}
peers := make([]Peer, 0, len(handshakes))
for key, at := range handshakes {
peers = append(peers, Peer{Key: key, Addresses: allowed[key], Seen: at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake})
}
sort.Slice(peers, func(i, k int) bool { return peers[i].Key < peers[k].Key })
found := func(f Finding) Finding { f.Peers = peers; return f }
age := func(at int64) string {
if at == 0 {
return "never"
@@ -452,23 +478,36 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
if hub != "" {
at := handshakes[hub]
if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake {
return Finding{OK: true}
return found(Finding{OK: true})
}
return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}
return found(Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}})
}
if len(handshakes) == 0 {
return Finding{OK: true}
return found(Finding{OK: true})
}
var newest int64
for _, at := range handshakes {
newest = max(newest, at)
}
if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake {
return Finding{OK: true}
return found(Finding{OK: true})
}
return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}
return found(Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))})
}
// samePeers says two looks saw the same peers, each seen or not alike.
func samePeers(a, b []Peer) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i].Key != b[i].Key || a[i].Seen != b[i].Seen || !slices.Equal(a[i].Addresses, b[i].Addresses) {
return false
}
}
return true
}
func (j *Judge) lookBus() Finding {