From 65fbc1e6f876bafc53b71f4683c9058fd6ff8e1c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:56:14 +0100 Subject: [PATCH] Write each node's health report into its own key of mesh-host_health on a change, so the watchman raises the machine's alerts without the controller (issue 774, the operator's alert table) Until now the six machine alerts came from a health message passing to the controller. The report is now kept in the engine's declared state `health`, key ``, created by the grants part as `acts` is. Only this node's engine writes it, after comparing it on every look and only when what it says changed; it is read first after a restart. It carries: - the network parts (machine-network; network-rewritten through the resolver file part's writer and owner); - the tunnel's peers, each seen within five minutes or not (network-unreachable); - the failed units no module owns (machine-units); - the accounts marked relogin needed (relogin-needed); - the directories used as found (directory-used-as-found). The streaks and the time of the look are left out, so a look alone writes nothing. A peer is reported seen or not, never its handshake time, so a renewed handshake writes nothing. The layout is internal/link's (NodeHealth) until it moves into mesh-sdk. Pins mesh-sdk 58a8324. Tests: - rows in TestOnTheHubAnyFreshPeerIsAHealthyTunnel: the peers, and a change only when one is seen or lost; - a case in TestAHealthStatementIsSaidOnTheLinkOpenNow: what the report takes, and one write per change across a restart; - the manifest's states in TestTheManifestConsumesTheBackupEvent. --- cmd/mesh-host/main.go | 34 ++++- go.mod | 2 +- go.sum | 4 +- internal/link/health_test.go | 68 +++++++++ internal/link/nodehealth.go | 228 +++++++++++++++++++++++++++++++ internal/link/session.go | 3 + internal/network/network.go | 53 ++++++- internal/network/network_test.go | 20 ++- module.json | 3 + packaging/operator_keys_test.go | 5 +- 10 files changed, 406 insertions(+), 14 deletions(-) create mode 100644 internal/link/nodehealth.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 8e52318..0be295e 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1353,6 +1353,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, var lastSaid time.Time owed := true spaced := 1.0 + // healthUnwritten is why the health report was last not written, said once each time it changes. + var healthUnwritten string // A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing // long-running is then ok within a look of its apply, not at the five-minute statement. var nudged <-chan struct{} @@ -1436,6 +1438,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) && (unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 && accessesHealthy(accessSt) + h := *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), + unitSt), accountSt), found), accessSt) + // The node's own health report in its key on the bus, compared on every look and written only when what it says + // changed (issue 774): what the watchman raises the machine's alerts from, without the controller. + writeHealthReport(ctx, queue, h, netSt, &healthUnwritten, say) if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway { continue } @@ -1462,13 +1469,36 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, reconcile(inactive...) } } - if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), - unitSt), accountSt), found), accessSt)) { + if queue.SayHealth(ctx, h) { lastSaid, owed = time.Now(), false } } } +// writeHealthReport writes this node's health report to its key (link.WriteNodeHealth) when it changed. Why it could +// not is said once per reason, and said again when it is written after all. +func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, ns *network.Statement, unwritten *string, + say link.Announce) { + if queue == nil { + return + } + var peers []link.PeerSeen + if ns != nil { + for _, p := range ns.Peers { + peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen}) + } + } + _, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers)) + switch { + case err != nil && err.Error() != *unwritten: + *unwritten = err.Error() + say("this machine's health report is not written to the bus: " + err.Error()) + case err == nil && *unwritten != "": + *unwritten = "" + say("this machine's health report is written to the bus again") + } +} + // healthAsReported is a statement as the report and the event carry it. func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health { // LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a diff --git a/go.mod b/go.mod index 6a00b6c..e49e5bd 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 326dffd..e33c19d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b h1:1b3bqhFHWosEhxlsjMI6NEWFH+vdD3hqXVIysyLhPYU= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/link/health_test.go b/internal/link/health_test.go index db7935a..58282a2 100644 --- a/internal/link/health_test.go +++ b/internal/link/health_test.go @@ -50,6 +50,74 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { if HealthSubject("anchor") != "mesh.control.anchor.health" { t.Fatalf("the subject %s is not inside the host's own grant", HealthSubject("anchor")) } + + // Issue 774: the node's own health report in its key of mesh-host_health, for the watchman's six alerts: written on + // a change only, the streaks and the time of the look not being one, and read first after a restart. + t.Run("the node's health report is written on a change only", func(t *testing.T) { + since := time.Date(2026, 10, 29, 20, 0, 0, 0, time.UTC) + look := func(streak int, writer string) Health { + return Health{At: time.Now(), Network: &NetworkHealth{State: StateUnhealthy, Since: since, Parts: []NetworkPart{ + {Part: "resolv-conf", State: StateUnhealthy, Writer: writer, Owner: "dnsmasq", Since: since, Streak: streak}}}, + Units: &UnitsHealth{State: "degraded", Failed: []FailedUnit{{Unit: "cups.service", Scope: "system", Since: since}}}, + Resources: []ResourceHealth{ + {Module: "brother", Resource: "brother.scanner", Kind: KindAccount, Account: "jochen", State: StateUnhealthy, + Reason: "relogin needed: the session began before scanner", Since: since}, + {Module: "photos", Resource: "photos.library", Kind: KindDirectory, Target: "/srv/photos", State: StateUnhealthy, + Reason: ReasonUsedAsFound + " owned by root", Since: since}, + {Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since}}} + } + peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}} + r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers) + if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" || + len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" || + len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" { + t.Fatalf("the report is %+v", r) + } + store := &memHealth{} + var last *NodeHealth + clock := func() time.Time { return since } + for i, step := range []struct { + h NodeHealth + fresh bool // a restart: nothing written in this run yet + wrote bool + }{ + {h: r, wrote: true}, + {h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers)}, + {h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers), fresh: true}, + {h: NodeHealthOf("anchor", look(3, "openvpn"), peers), wrote: true}, + {h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}), wrote: true}, + } { + if step.fresh { + last = nil + } + wrote, err := writeNodeHealth(t.Context(), store, &last, step.h, clock) + if err != nil || wrote != step.wrote { + t.Fatalf("step %d: wrote %v (want %v), %v", i, wrote, step.wrote, err) + } + } + got, err := DecodeNodeHealth(store.values["anchor"]) + if err != nil || got.Node != "anchor" || got.Network.Parts[0].Writer != "openvpn" || !got.Peers[0].Seen || got.At.IsZero() { + t.Fatalf("the key holds %+v, %v", got, err) + } + if store.puts != 3 { + t.Fatalf("%d writes, want 3: one per change", store.puts) + } + }) +} + +// memHealth is a HealthStore in memory. +type memHealth struct { + values map[string][]byte + puts int +} + +func (m *memHealth) Get(_ context.Context, key string) ([]byte, error) { return m.values[key], nil } +func (m *memHealth) Put(_ context.Context, key string, value []byte) error { + if m.values == nil { + m.values = map[string][]byte{} + } + m.values[key], m.puts = value, m.puts+1 + return nil } // A declared tool check reads the node tools' reply envelope: healthy only when the tool says so (novox/hq diff --git a/internal/link/nodehealth.go b/internal/link/nodehealth.go new file mode 100644 index 0000000..4ec92c9 --- /dev/null +++ b/internal/link/nodehealth.go @@ -0,0 +1,228 @@ +package link + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/nats-io/nats.go" +) + +// The node-engine's own health report in the bus's own storage (issue 774, the operator's alert table, 2026-10-29): one +// key per node, ``, of the engine's declared state `health` (HealthBucket, created by the grants part from the +// engine's manifest as its state `acts` is), written by this node's engine alone and only when what it says changed. +// The watchman (seat mesh-watchdogs) reads it for six alerts the controller raised from the passing health message: +// +// - machine-network: Network, the machine's route, tunnel, name servers, resolver file and bus, each a part; +// - network-rewritten: the resolver file's part, unhealthy with Writer, the program that rewrote it, and Owner, the +// module whose file it is; +// - network-unreachable: Peers, each peer of the mesh's tunnel and whether this machine saw it within five minutes, +// read across every node's report; +// - machine-units: Units, the failed system units no module owns; +// - relogin-needed: Relogin, each account whose running session lacks a group it was given; +// - directory-used-as-found: Found, each directory a module declares that the engine uses as found. +// +// The layout moves into mesh-sdk before the push (the lead, 2026-10-29); until then it is this package's. + +// HealthBucket is the node-engine's declared state health: key ``, value NodeHealth. +const HealthBucket = "mesh-host_health" + +// NodeHealthVersion is the version of NodeHealth's layout. +const NodeHealthVersion = 1 + +// NodeHealth is one node's health report, as its engine last found it. +type NodeHealth struct { + V int `json:"v"` + Node string `json:"node"` + // Network is the machine's networking, its parts as Health.Network says them, without the streaks; nil where this + // engine does not judge it yet. + Network *NetworkHealth `json:"network,omitempty"` + // Peers are the mesh tunnel's peers on the last look that read them, sorted by key. + Peers []PeerSeen `json:"peers,omitempty"` + // Units is the machine's service managers: each failed unit no module places (Health.Units). + Units *UnitsHealth `json:"units,omitempty"` + // Relogin are the accounts whose running session needs a new login for a group it was given. + Relogin []NodeAccount `json:"relogin,omitempty"` + // Found are the directories used as found. + Found []NodeDirectory `json:"found,omitempty"` + // At is when the engine wrote this, on a change: never a reason to write. + At time.Time `json:"at"` +} + +// PeerSeen is one peer of the mesh's tunnel: its public key, its allowed addresses, and whether this machine saw a +// handshake with it within five minutes. +type PeerSeen struct { + Key string `json:"key"` + Addresses []string `json:"addresses,omitempty"` + Seen bool `json:"seen"` +} + +// NodeAccount is one account marked relogin needed, with the module whose resource it is and why. +type NodeAccount struct { + Account string `json:"account"` + Module string `json:"module,omitempty"` + Resource string `json:"resource,omitempty"` + Reason string `json:"reason"` + Since time.Time `json:"since"` +} + +// NodeDirectory is one directory used as found: the module declaring it, its resource and path, and why. +type NodeDirectory struct { + Module string `json:"module"` + Resource string `json:"resource"` + Path string `json:"path"` + Reason string `json:"reason"` + Since time.Time `json:"since"` +} + +// ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin). +const ReasonReloginNeeded = "relogin needed" + +// NodeHealthOf is the report a health statement and the tunnel's peers make: the statement's network without its +// streaks, its machine units, and each account and directory of its resources that the six alerts read. +func NodeHealthOf(node string, h Health, peers []PeerSeen) NodeHealth { + out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units} + if len(peers) > 0 { + out.Peers = append([]PeerSeen(nil), peers...) + sort.Slice(out.Peers, func(i, k int) bool { return out.Peers[i].Key < out.Peers[k].Key }) + } + if h.Network != nil { + n := NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: make([]NetworkPart, 0, len(h.Network.Parts))} + for _, p := range h.Network.Parts { + p.Streak = 0 // a look's count moves every look; the report says a part's state, not how often it was seen + n.Parts = append(n.Parts, p) + } + out.Network = &n + } + for _, r := range h.Resources { + switch { + case r.Kind == KindAccount && r.State == StateUnhealthy && strings.HasPrefix(r.Reason, ReasonReloginNeeded): + out.Relogin = append(out.Relogin, NodeAccount{Account: r.Account, Module: r.Module, Resource: r.Resource, + Reason: r.Reason, Since: r.Since}) + case r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound): + out.Found = append(out.Found, NodeDirectory{Module: r.Module, Resource: r.Resource, Path: r.Target, + Reason: r.Reason, Since: r.Since}) + } + } + sort.Slice(out.Relogin, func(i, k int) bool { + return out.Relogin[i].Account+out.Relogin[i].Module < out.Relogin[k].Account+out.Relogin[k].Module + }) + sort.Slice(out.Found, func(i, k int) bool { return out.Found[i].Path < out.Found[k].Path }) + return out +} + +// EncodeNodeHealth is h as JSON at its version. +func EncodeNodeHealth(h NodeHealth) ([]byte, error) { + h.V = NodeHealthVersion + if h.Node == "" { + return nil, errors.New("a node's health report names its node") + } + return json.Marshal(h) +} + +// DecodeNodeHealth reads a report. +func DecodeNodeHealth(raw []byte) (NodeHealth, error) { + var h NodeHealth + if err := json.Unmarshal(raw, &h); err != nil { + return NodeHealth{}, fmt.Errorf("a node's health report is not readable: %w", err) + } + if h.V != NodeHealthVersion { + return NodeHealth{}, fmt.Errorf("a node's health report of version %d, and this reads %d", h.V, NodeHealthVersion) + } + return h, nil +} + +// SameNodeHealth says two reports say the same, whenever each was written. +func SameNodeHealth(a, b NodeHealth) bool { + a.At, b.At = time.Time{}, time.Time{} + ra, errA := json.Marshal(a) + rb, errB := json.Marshal(b) + return errA == nil && errB == nil && bytes.Equal(ra, rb) +} + +// HealthStore is where a node's report is kept: its own key of HealthBucket. +type HealthStore interface { + // Get is the key's value, nil when it has none. + Get(ctx context.Context, key string) ([]byte, error) + Put(ctx context.Context, key string, value []byte) error +} + +// healthStore binds HealthBucket over the link open now: bound, never created. +func (l *natsLink) healthStore() (HealthStore, error) { + kv, err := l.js.KeyValue(HealthBucket) + if err != nil { + return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+ + "manifest; written again once it is): %w", HealthBucket, err) + } + return natsHealth{kv}, nil +} + +type natsHealth struct{ kv nats.KeyValue } + +func (n natsHealth) Get(_ context.Context, key string) ([]byte, error) { + e, err := n.kv.Get(key) + if errors.Is(err, nats.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return e.Value(), nil +} + +func (n natsHealth) Put(_ context.Context, key string, value []byte) error { + _, err := n.kv.Put(key, value) + return err +} + +// WriteNodeHealth writes this node's report to its key when it says something other than the key holds, and says +// whether it wrote. The key is read before the first write of a run, so a restart that finds the same report writes +// nothing; a store that cannot be bound or written is an error, and the next look tries again. +func (q *Session) WriteNodeHealth(ctx context.Context, h NodeHealth) (bool, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + l, ok := bus.(interface{ healthStore() (HealthStore, error) }) + if bus == nil || !ok { + return false, ErrNoLink + } + store, err := l.healthStore() + if err != nil { + return false, err + } + return writeNodeHealth(ctx, store, &q.healthWritten, h, time.Now) +} + +// writeNodeHealth is WriteNodeHealth over a store: last is the report last written or read, nil before the first. +func writeNodeHealth(ctx context.Context, store HealthStore, last **NodeHealth, h NodeHealth, now func() time.Time) (bool, error) { + if *last == nil { + raw, err := store.Get(ctx, h.Node) + if err != nil { + return false, fmt.Errorf("the health report of %s cannot be read: %w", h.Node, err) + } + if raw != nil { + if was, err := DecodeNodeHealth(raw); err == nil { + *last = &was + } + } + } + if *last != nil && SameNodeHealth(**last, h) { + return false, nil + } + h.At = now().UTC() + raw, err := EncodeNodeHealth(h) + if err != nil { + return false, err + } + if err := store.Put(ctx, h.Node, raw); err != nil { + return false, fmt.Errorf("the health report of %s cannot be written: %w", h.Node, err) + } + *last = &h + return true, nil +} diff --git a/internal/link/session.go b/internal/link/session.go index e33314d..5b2fb89 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -86,6 +86,9 @@ type Session struct { // haves is what this node-engine writes beside its facts (novox/hq ADR 0429 ยง4, issue 1132). haves haveState + // healthWritten is this node's health report as last written or read (WriteNodeHealth): the health loop alone + // writes it, one look at a time. + healthWritten *NodeHealth } func (q *Session) init() { diff --git a/internal/network/network.go b/internal/network/network.go index bd65d52..f1f511a 100644 --- a/internal/network/network.go +++ b/internal/network/network.go @@ -32,6 +32,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -94,6 +95,18 @@ type Finding struct { Writer string // Toward is what the failure points at: TowardHub, or each resolver's address that failed. Toward []string + // Peers is, for the tunnel, each peer of the mesh's interface and whether it was seen (Peer). + Peers []Peer +} + +// Peer is one peer of the mesh's tunnel as this machine sees it: its public key, the addresses it is allowed, and +// whether it handshook within StaleHandshake. What the watchman reads to tell a machine nobody reaches from one whose +// tunnel alone is down (network-unreachable). Seen, not the handshake's time: the time moves every two minutes, the +// fact only when a peer comes or goes. +type Peer struct { + Key string `json:"key"` + Addresses []string `json:"addresses,omitempty"` + Seen bool `json:"seen"` } // Part is one part's state, as the statement says it. @@ -115,6 +128,8 @@ type Statement struct { Since time.Time `json:"since"` At time.Time `json:"at"` Parts []Part `json:"parts"` + // Peers are the tunnel's peers on the last look that read them, sorted by key; nil where the tunnel is not read. + Peers []Peer `json:"peers,omitempty"` } // Machine is what a look reads, replaced in tests. @@ -266,6 +281,9 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) { changed = true } st.State, st.Since = worst, j.overall.since + // The tunnel's peers as this look read them; a peer seen or lost is a change of its own (network-unreachable). + st.Peers = findings[PartTunnel].Peers + changed = changed || !samePeers(j.said.Peers, st.Peers) j.said = st return st, changed } @@ -435,14 +453,22 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding { } } hub := "" + allowed := map[string][]string{} for _, line := range strings.Split(ips, "\n") { f := strings.Fields(line) for _, prefix := range f[min(1, len(f)):] { + allowed[f[0]] = append(allowed[f[0]], prefix) if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" { hub = f[0] } } } + peers := make([]Peer, 0, len(handshakes)) + for key, at := range handshakes { + peers = append(peers, Peer{Key: key, Addresses: allowed[key], Seen: at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake}) + } + sort.Slice(peers, func(i, k int) bool { return peers[i].Key < peers[k].Key }) + found := func(f Finding) Finding { f.Peers = peers; return f } age := func(at int64) string { if at == 0 { return "never" @@ -452,23 +478,36 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding { if hub != "" { at := handshakes[hub] if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake { - return Finding{OK: true} + return found(Finding{OK: true}) } - return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes", - Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}} + return found(Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes", + Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}) } if len(handshakes) == 0 { - return Finding{OK: true} + return found(Finding{OK: true}) } var newest int64 for _, at := range handshakes { newest = max(newest, at) } if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake { - return Finding{OK: true} + return found(Finding{OK: true}) } - return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes", - Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))} + return found(Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes", + Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}) +} + +// samePeers says two looks saw the same peers, each seen or not alike. +func samePeers(a, b []Peer) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i].Key != b[i].Key || a[i].Seen != b[i].Seen || !slices.Equal(a[i].Addresses, b[i].Addresses) { + return false + } + } + return true } func (j *Judge) lookBus() Finding { diff --git a/internal/network/network_test.go b/internal/network/network_test.go index f27caa0..67155e7 100644 --- a/internal/network/network_test.go +++ b/internal/network/network_test.go @@ -302,9 +302,27 @@ func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) { f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n" f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n" j := f.judge(t) - if st := f.look(t, j); st.State != Healthy { + st := f.look(t, j) + if st.State != Healthy { t.Fatalf("the hub with one fresh peer is %+v", st.Parts) } + // Each peer and whether it was seen within five minutes, for the watchman's network-unreachable (issue 774); a + // peer seen again is a change, a handshake renewed alone is not. + want := []Peer{{Key: "A=", Addresses: []string{"10.10.0.2/32"}}, {Key: "B=", Addresses: []string{"10.10.0.3/32"}, Seen: true}, + {Key: "C=", Addresses: []string{"10.10.0.4/32"}}} + if !samePeers(st.Peers, want) { + t.Fatalf("the peers are %+v", st.Peers) + } + f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n" + f.now = f.now.Add(LookEvery) + if st, changed := j.Look(t.Context()); !changed || !st.Peers[0].Seen { + t.Fatalf("a peer seen again is no change: changed %v, %+v", changed, st.Peers) + } + f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n" + f.now = f.now.Add(LookEvery) + if _, changed := j.Look(t.Context()); changed { + t.Fatal("handshakes renewed alone are a change") + } } func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) { diff --git a/module.json b/module.json index d10f063..146d6a5 100644 --- a/module.json +++ b/module.json @@ -21,6 +21,9 @@ { "name": "acts", "ttl-seconds": 7776000 + }, + { + "name": "health" } ], "config": { diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 6794b9e..ecf8ed1 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -74,7 +74,10 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { if err != nil { t.Fatal(err) } - if len(parsed.State) != 1 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 { + // 774 (the alert table): beside it the state health, one key per node, the node's own health report, kept until + // written again (no TTL: a node that stops writing keeps its last word, which the watchman reads with its time). + if len(parsed.State) != 2 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 || + parsed.State[1].Name != "health" || parsed.State[1].TTLSeconds != 0 { t.Fatalf("the manifest declares the state %+v", parsed.State) } }