Write each node's health report into its own key of mesh-host_health on a change, so the watchman raises the machine's alerts without the controller (issue 774, the operator's alert table)
Until now the six machine alerts came from a health message passing to the controller. The report is now kept in the engine's declared state `health`, key `<node>`, created by the grants part as `acts` is. Only this node's engine writes it, after comparing it on every look and only when what it says changed; it is read first after a restart. It carries: - the network parts (machine-network; network-rewritten through the resolver file part's writer and owner); - the tunnel's peers, each seen within five minutes or not (network-unreachable); - the failed units no module owns (machine-units); - the accounts marked relogin needed (relogin-needed); - the directories used as found (directory-used-as-found). The streaks and the time of the look are left out, so a look alone writes nothing. A peer is reported seen or not, never its handshake time, so a renewed handshake writes nothing. The layout is internal/link's (NodeHealth) until it moves into mesh-sdk. Pins mesh-sdk 58a8324. Tests: - rows in TestOnTheHubAnyFreshPeerIsAHealthyTunnel: the peers, and a change only when one is seen or lost; - a case in TestAHealthStatementIsSaidOnTheLinkOpenNow: what the report takes, and one write per change across a restart; - the manifest's states in TestTheManifestConsumesTheBackupEvent.
This commit is contained in:
@@ -32,6 +32,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -94,6 +95,18 @@ type Finding struct {
|
||||
Writer string
|
||||
// Toward is what the failure points at: TowardHub, or each resolver's address that failed.
|
||||
Toward []string
|
||||
// Peers is, for the tunnel, each peer of the mesh's interface and whether it was seen (Peer).
|
||||
Peers []Peer
|
||||
}
|
||||
|
||||
// Peer is one peer of the mesh's tunnel as this machine sees it: its public key, the addresses it is allowed, and
|
||||
// whether it handshook within StaleHandshake. What the watchman reads to tell a machine nobody reaches from one whose
|
||||
// tunnel alone is down (network-unreachable). Seen, not the handshake's time: the time moves every two minutes, the
|
||||
// fact only when a peer comes or goes.
|
||||
type Peer struct {
|
||||
Key string `json:"key"`
|
||||
Addresses []string `json:"addresses,omitempty"`
|
||||
Seen bool `json:"seen"`
|
||||
}
|
||||
|
||||
// Part is one part's state, as the statement says it.
|
||||
@@ -115,6 +128,8 @@ type Statement struct {
|
||||
Since time.Time `json:"since"`
|
||||
At time.Time `json:"at"`
|
||||
Parts []Part `json:"parts"`
|
||||
// Peers are the tunnel's peers on the last look that read them, sorted by key; nil where the tunnel is not read.
|
||||
Peers []Peer `json:"peers,omitempty"`
|
||||
}
|
||||
|
||||
// Machine is what a look reads, replaced in tests.
|
||||
@@ -266,6 +281,9 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
|
||||
changed = true
|
||||
}
|
||||
st.State, st.Since = worst, j.overall.since
|
||||
// The tunnel's peers as this look read them; a peer seen or lost is a change of its own (network-unreachable).
|
||||
st.Peers = findings[PartTunnel].Peers
|
||||
changed = changed || !samePeers(j.said.Peers, st.Peers)
|
||||
j.said = st
|
||||
return st, changed
|
||||
}
|
||||
@@ -435,14 +453,22 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
|
||||
}
|
||||
}
|
||||
hub := ""
|
||||
allowed := map[string][]string{}
|
||||
for _, line := range strings.Split(ips, "\n") {
|
||||
f := strings.Fields(line)
|
||||
for _, prefix := range f[min(1, len(f)):] {
|
||||
allowed[f[0]] = append(allowed[f[0]], prefix)
|
||||
if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" {
|
||||
hub = f[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
peers := make([]Peer, 0, len(handshakes))
|
||||
for key, at := range handshakes {
|
||||
peers = append(peers, Peer{Key: key, Addresses: allowed[key], Seen: at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake})
|
||||
}
|
||||
sort.Slice(peers, func(i, k int) bool { return peers[i].Key < peers[k].Key })
|
||||
found := func(f Finding) Finding { f.Peers = peers; return f }
|
||||
age := func(at int64) string {
|
||||
if at == 0 {
|
||||
return "never"
|
||||
@@ -452,23 +478,36 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
|
||||
if hub != "" {
|
||||
at := handshakes[hub]
|
||||
if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake {
|
||||
return Finding{OK: true}
|
||||
return found(Finding{OK: true})
|
||||
}
|
||||
return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}
|
||||
return found(Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}})
|
||||
}
|
||||
if len(handshakes) == 0 {
|
||||
return Finding{OK: true}
|
||||
return found(Finding{OK: true})
|
||||
}
|
||||
var newest int64
|
||||
for _, at := range handshakes {
|
||||
newest = max(newest, at)
|
||||
}
|
||||
if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake {
|
||||
return Finding{OK: true}
|
||||
return found(Finding{OK: true})
|
||||
}
|
||||
return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}
|
||||
return found(Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))})
|
||||
}
|
||||
|
||||
// samePeers says two looks saw the same peers, each seen or not alike.
|
||||
func samePeers(a, b []Peer) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i].Key != b[i].Key || a[i].Seen != b[i].Seen || !slices.Equal(a[i].Addresses, b[i].Addresses) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (j *Judge) lookBus() Finding {
|
||||
|
||||
@@ -302,9 +302,27 @@ func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) {
|
||||
f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n"
|
||||
f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n"
|
||||
j := f.judge(t)
|
||||
if st := f.look(t, j); st.State != Healthy {
|
||||
st := f.look(t, j)
|
||||
if st.State != Healthy {
|
||||
t.Fatalf("the hub with one fresh peer is %+v", st.Parts)
|
||||
}
|
||||
// Each peer and whether it was seen within five minutes, for the watchman's network-unreachable (issue 774); a
|
||||
// peer seen again is a change, a handshake renewed alone is not.
|
||||
want := []Peer{{Key: "A=", Addresses: []string{"10.10.0.2/32"}}, {Key: "B=", Addresses: []string{"10.10.0.3/32"}, Seen: true},
|
||||
{Key: "C=", Addresses: []string{"10.10.0.4/32"}}}
|
||||
if !samePeers(st.Peers, want) {
|
||||
t.Fatalf("the peers are %+v", st.Peers)
|
||||
}
|
||||
f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n"
|
||||
f.now = f.now.Add(LookEvery)
|
||||
if st, changed := j.Look(t.Context()); !changed || !st.Peers[0].Seen {
|
||||
t.Fatalf("a peer seen again is no change: changed %v, %+v", changed, st.Peers)
|
||||
}
|
||||
f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n"
|
||||
f.now = f.now.Add(LookEvery)
|
||||
if _, changed := j.Look(t.Context()); changed {
|
||||
t.Fatal("handshakes renewed alone are a change")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user