Write each node's health report into its own key of mesh-host_health on a change, so the watchman raises the machine's alerts without the controller (issue 774, the operator's alert table)
Until now the six machine alerts came from a health message passing to the controller. The report is now kept in the engine's declared state `health`, key `<node>`, created by the grants part as `acts` is. Only this node's engine writes it, after comparing it on every look and only when what it says changed; it is read first after a restart. It carries: - the network parts (machine-network; network-rewritten through the resolver file part's writer and owner); - the tunnel's peers, each seen within five minutes or not (network-unreachable); - the failed units no module owns (machine-units); - the accounts marked relogin needed (relogin-needed); - the directories used as found (directory-used-as-found). The streaks and the time of the look are left out, so a look alone writes nothing. A peer is reported seen or not, never its handshake time, so a renewed handshake writes nothing. The layout is internal/link's (NodeHealth) until it moves into mesh-sdk. Pins mesh-sdk 58a8324. Tests: - rows in TestOnTheHubAnyFreshPeerIsAHealthyTunnel: the peers, and a change only when one is seen or lost; - a case in TestAHealthStatementIsSaidOnTheLinkOpenNow: what the report takes, and one write per change across a restart; - the manifest's states in TestTheManifestConsumesTheBackupEvent.
This commit is contained in:
+32
-2
@@ -1353,6 +1353,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
var lastSaid time.Time
|
||||
owed := true
|
||||
spaced := 1.0
|
||||
// healthUnwritten is why the health report was last not written, said once each time it changes.
|
||||
var healthUnwritten string
|
||||
// A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing
|
||||
// long-running is then ok within a look of its apply, not at the five-minute statement.
|
||||
var nudged <-chan struct{}
|
||||
@@ -1436,6 +1438,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
|
||||
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 &&
|
||||
accessesHealthy(accessSt)
|
||||
h := *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
||||
unitSt), accountSt), found), accessSt)
|
||||
// The node's own health report in its key on the bus, compared on every look and written only when what it says
|
||||
// changed (issue 774): what the watchman raises the machine's alerts from, without the controller.
|
||||
writeHealthReport(ctx, queue, h, netSt, &healthUnwritten, say)
|
||||
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
||||
continue
|
||||
}
|
||||
@@ -1462,13 +1469,36 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
reconcile(inactive...)
|
||||
}
|
||||
}
|
||||
if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
||||
unitSt), accountSt), found), accessSt)) {
|
||||
if queue.SayHealth(ctx, h) {
|
||||
lastSaid, owed = time.Now(), false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeHealthReport writes this node's health report to its key (link.WriteNodeHealth) when it changed. Why it could
|
||||
// not is said once per reason, and said again when it is written after all.
|
||||
func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, ns *network.Statement, unwritten *string,
|
||||
say link.Announce) {
|
||||
if queue == nil {
|
||||
return
|
||||
}
|
||||
var peers []link.PeerSeen
|
||||
if ns != nil {
|
||||
for _, p := range ns.Peers {
|
||||
peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen})
|
||||
}
|
||||
}
|
||||
_, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers))
|
||||
switch {
|
||||
case err != nil && err.Error() != *unwritten:
|
||||
*unwritten = err.Error()
|
||||
say("this machine's health report is not written to the bus: " + err.Error())
|
||||
case err == nil && *unwritten != "":
|
||||
*unwritten = ""
|
||||
say("this machine's health report is written to the bus again")
|
||||
}
|
||||
}
|
||||
|
||||
// healthAsReported is a statement as the report and the event carry it.
|
||||
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
||||
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b h1:1b3bqhFHWosEhxlsjMI6NEWFH+vdD3hqXVIysyLhPYU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -50,6 +50,74 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) {
|
||||
if HealthSubject("anchor") != "mesh.control.anchor.health" {
|
||||
t.Fatalf("the subject %s is not inside the host's own grant", HealthSubject("anchor"))
|
||||
}
|
||||
|
||||
// Issue 774: the node's own health report in its key of mesh-host_health, for the watchman's six alerts: written on
|
||||
// a change only, the streaks and the time of the look not being one, and read first after a restart.
|
||||
t.Run("the node's health report is written on a change only", func(t *testing.T) {
|
||||
since := time.Date(2026, 10, 29, 20, 0, 0, 0, time.UTC)
|
||||
look := func(streak int, writer string) Health {
|
||||
return Health{At: time.Now(), Network: &NetworkHealth{State: StateUnhealthy, Since: since, Parts: []NetworkPart{
|
||||
{Part: "resolv-conf", State: StateUnhealthy, Writer: writer, Owner: "dnsmasq", Since: since, Streak: streak}}},
|
||||
Units: &UnitsHealth{State: "degraded", Failed: []FailedUnit{{Unit: "cups.service", Scope: "system", Since: since}}},
|
||||
Resources: []ResourceHealth{
|
||||
{Module: "brother", Resource: "brother.scanner", Kind: KindAccount, Account: "jochen", State: StateUnhealthy,
|
||||
Reason: "relogin needed: the session began before scanner", Since: since},
|
||||
{Module: "photos", Resource: "photos.library", Kind: KindDirectory, Target: "/srv/photos", State: StateUnhealthy,
|
||||
Reason: ReasonUsedAsFound + " owned by root", Since: since},
|
||||
{Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since}}}
|
||||
}
|
||||
peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}}
|
||||
r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers)
|
||||
if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" ||
|
||||
len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" ||
|
||||
len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" {
|
||||
t.Fatalf("the report is %+v", r)
|
||||
}
|
||||
store := &memHealth{}
|
||||
var last *NodeHealth
|
||||
clock := func() time.Time { return since }
|
||||
for i, step := range []struct {
|
||||
h NodeHealth
|
||||
fresh bool // a restart: nothing written in this run yet
|
||||
wrote bool
|
||||
}{
|
||||
{h: r, wrote: true},
|
||||
{h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers)},
|
||||
{h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers), fresh: true},
|
||||
{h: NodeHealthOf("anchor", look(3, "openvpn"), peers), wrote: true},
|
||||
{h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}), wrote: true},
|
||||
} {
|
||||
if step.fresh {
|
||||
last = nil
|
||||
}
|
||||
wrote, err := writeNodeHealth(t.Context(), store, &last, step.h, clock)
|
||||
if err != nil || wrote != step.wrote {
|
||||
t.Fatalf("step %d: wrote %v (want %v), %v", i, wrote, step.wrote, err)
|
||||
}
|
||||
}
|
||||
got, err := DecodeNodeHealth(store.values["anchor"])
|
||||
if err != nil || got.Node != "anchor" || got.Network.Parts[0].Writer != "openvpn" || !got.Peers[0].Seen || got.At.IsZero() {
|
||||
t.Fatalf("the key holds %+v, %v", got, err)
|
||||
}
|
||||
if store.puts != 3 {
|
||||
t.Fatalf("%d writes, want 3: one per change", store.puts)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// memHealth is a HealthStore in memory.
|
||||
type memHealth struct {
|
||||
values map[string][]byte
|
||||
puts int
|
||||
}
|
||||
|
||||
func (m *memHealth) Get(_ context.Context, key string) ([]byte, error) { return m.values[key], nil }
|
||||
func (m *memHealth) Put(_ context.Context, key string, value []byte) error {
|
||||
if m.values == nil {
|
||||
m.values = map[string][]byte{}
|
||||
}
|
||||
m.values[key], m.puts = value, m.puts+1
|
||||
return nil
|
||||
}
|
||||
|
||||
// A declared tool check reads the node tools' reply envelope: healthy only when the tool says so (novox/hq
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// The node-engine's own health report in the bus's own storage (issue 774, the operator's alert table, 2026-10-29): one
|
||||
// key per node, `<node>`, of the engine's declared state `health` (HealthBucket, created by the grants part from the
|
||||
// engine's manifest as its state `acts` is), written by this node's engine alone and only when what it says changed.
|
||||
// The watchman (seat mesh-watchdogs) reads it for six alerts the controller raised from the passing health message:
|
||||
//
|
||||
// - machine-network: Network, the machine's route, tunnel, name servers, resolver file and bus, each a part;
|
||||
// - network-rewritten: the resolver file's part, unhealthy with Writer, the program that rewrote it, and Owner, the
|
||||
// module whose file it is;
|
||||
// - network-unreachable: Peers, each peer of the mesh's tunnel and whether this machine saw it within five minutes,
|
||||
// read across every node's report;
|
||||
// - machine-units: Units, the failed system units no module owns;
|
||||
// - relogin-needed: Relogin, each account whose running session lacks a group it was given;
|
||||
// - directory-used-as-found: Found, each directory a module declares that the engine uses as found.
|
||||
//
|
||||
// The layout moves into mesh-sdk before the push (the lead, 2026-10-29); until then it is this package's.
|
||||
|
||||
// HealthBucket is the node-engine's declared state health: key `<node>`, value NodeHealth.
|
||||
const HealthBucket = "mesh-host_health"
|
||||
|
||||
// NodeHealthVersion is the version of NodeHealth's layout.
|
||||
const NodeHealthVersion = 1
|
||||
|
||||
// NodeHealth is one node's health report, as its engine last found it.
|
||||
type NodeHealth struct {
|
||||
V int `json:"v"`
|
||||
Node string `json:"node"`
|
||||
// Network is the machine's networking, its parts as Health.Network says them, without the streaks; nil where this
|
||||
// engine does not judge it yet.
|
||||
Network *NetworkHealth `json:"network,omitempty"`
|
||||
// Peers are the mesh tunnel's peers on the last look that read them, sorted by key.
|
||||
Peers []PeerSeen `json:"peers,omitempty"`
|
||||
// Units is the machine's service managers: each failed unit no module places (Health.Units).
|
||||
Units *UnitsHealth `json:"units,omitempty"`
|
||||
// Relogin are the accounts whose running session needs a new login for a group it was given.
|
||||
Relogin []NodeAccount `json:"relogin,omitempty"`
|
||||
// Found are the directories used as found.
|
||||
Found []NodeDirectory `json:"found,omitempty"`
|
||||
// At is when the engine wrote this, on a change: never a reason to write.
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// PeerSeen is one peer of the mesh's tunnel: its public key, its allowed addresses, and whether this machine saw a
|
||||
// handshake with it within five minutes.
|
||||
type PeerSeen struct {
|
||||
Key string `json:"key"`
|
||||
Addresses []string `json:"addresses,omitempty"`
|
||||
Seen bool `json:"seen"`
|
||||
}
|
||||
|
||||
// NodeAccount is one account marked relogin needed, with the module whose resource it is and why.
|
||||
type NodeAccount struct {
|
||||
Account string `json:"account"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Resource string `json:"resource,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// NodeDirectory is one directory used as found: the module declaring it, its resource and path, and why.
|
||||
type NodeDirectory struct {
|
||||
Module string `json:"module"`
|
||||
Resource string `json:"resource"`
|
||||
Path string `json:"path"`
|
||||
Reason string `json:"reason"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin).
|
||||
const ReasonReloginNeeded = "relogin needed"
|
||||
|
||||
// NodeHealthOf is the report a health statement and the tunnel's peers make: the statement's network without its
|
||||
// streaks, its machine units, and each account and directory of its resources that the six alerts read.
|
||||
func NodeHealthOf(node string, h Health, peers []PeerSeen) NodeHealth {
|
||||
out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units}
|
||||
if len(peers) > 0 {
|
||||
out.Peers = append([]PeerSeen(nil), peers...)
|
||||
sort.Slice(out.Peers, func(i, k int) bool { return out.Peers[i].Key < out.Peers[k].Key })
|
||||
}
|
||||
if h.Network != nil {
|
||||
n := NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: make([]NetworkPart, 0, len(h.Network.Parts))}
|
||||
for _, p := range h.Network.Parts {
|
||||
p.Streak = 0 // a look's count moves every look; the report says a part's state, not how often it was seen
|
||||
n.Parts = append(n.Parts, p)
|
||||
}
|
||||
out.Network = &n
|
||||
}
|
||||
for _, r := range h.Resources {
|
||||
switch {
|
||||
case r.Kind == KindAccount && r.State == StateUnhealthy && strings.HasPrefix(r.Reason, ReasonReloginNeeded):
|
||||
out.Relogin = append(out.Relogin, NodeAccount{Account: r.Account, Module: r.Module, Resource: r.Resource,
|
||||
Reason: r.Reason, Since: r.Since})
|
||||
case r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound):
|
||||
out.Found = append(out.Found, NodeDirectory{Module: r.Module, Resource: r.Resource, Path: r.Target,
|
||||
Reason: r.Reason, Since: r.Since})
|
||||
}
|
||||
}
|
||||
sort.Slice(out.Relogin, func(i, k int) bool {
|
||||
return out.Relogin[i].Account+out.Relogin[i].Module < out.Relogin[k].Account+out.Relogin[k].Module
|
||||
})
|
||||
sort.Slice(out.Found, func(i, k int) bool { return out.Found[i].Path < out.Found[k].Path })
|
||||
return out
|
||||
}
|
||||
|
||||
// EncodeNodeHealth is h as JSON at its version.
|
||||
func EncodeNodeHealth(h NodeHealth) ([]byte, error) {
|
||||
h.V = NodeHealthVersion
|
||||
if h.Node == "" {
|
||||
return nil, errors.New("a node's health report names its node")
|
||||
}
|
||||
return json.Marshal(h)
|
||||
}
|
||||
|
||||
// DecodeNodeHealth reads a report.
|
||||
func DecodeNodeHealth(raw []byte) (NodeHealth, error) {
|
||||
var h NodeHealth
|
||||
if err := json.Unmarshal(raw, &h); err != nil {
|
||||
return NodeHealth{}, fmt.Errorf("a node's health report is not readable: %w", err)
|
||||
}
|
||||
if h.V != NodeHealthVersion {
|
||||
return NodeHealth{}, fmt.Errorf("a node's health report of version %d, and this reads %d", h.V, NodeHealthVersion)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// SameNodeHealth says two reports say the same, whenever each was written.
|
||||
func SameNodeHealth(a, b NodeHealth) bool {
|
||||
a.At, b.At = time.Time{}, time.Time{}
|
||||
ra, errA := json.Marshal(a)
|
||||
rb, errB := json.Marshal(b)
|
||||
return errA == nil && errB == nil && bytes.Equal(ra, rb)
|
||||
}
|
||||
|
||||
// HealthStore is where a node's report is kept: its own key of HealthBucket.
|
||||
type HealthStore interface {
|
||||
// Get is the key's value, nil when it has none.
|
||||
Get(ctx context.Context, key string) ([]byte, error)
|
||||
Put(ctx context.Context, key string, value []byte) error
|
||||
}
|
||||
|
||||
// healthStore binds HealthBucket over the link open now: bound, never created.
|
||||
func (l *natsLink) healthStore() (HealthStore, error) {
|
||||
kv, err := l.js.KeyValue(HealthBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+
|
||||
"manifest; written again once it is): %w", HealthBucket, err)
|
||||
}
|
||||
return natsHealth{kv}, nil
|
||||
}
|
||||
|
||||
type natsHealth struct{ kv nats.KeyValue }
|
||||
|
||||
func (n natsHealth) Get(_ context.Context, key string) ([]byte, error) {
|
||||
e, err := n.kv.Get(key)
|
||||
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.Value(), nil
|
||||
}
|
||||
|
||||
func (n natsHealth) Put(_ context.Context, key string, value []byte) error {
|
||||
_, err := n.kv.Put(key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
// WriteNodeHealth writes this node's report to its key when it says something other than the key holds, and says
|
||||
// whether it wrote. The key is read before the first write of a run, so a restart that finds the same report writes
|
||||
// nothing; a store that cannot be bound or written is an error, and the next look tries again.
|
||||
func (q *Session) WriteNodeHealth(ctx context.Context, h NodeHealth) (bool, error) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
l, ok := bus.(interface{ healthStore() (HealthStore, error) })
|
||||
if bus == nil || !ok {
|
||||
return false, ErrNoLink
|
||||
}
|
||||
store, err := l.healthStore()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return writeNodeHealth(ctx, store, &q.healthWritten, h, time.Now)
|
||||
}
|
||||
|
||||
// writeNodeHealth is WriteNodeHealth over a store: last is the report last written or read, nil before the first.
|
||||
func writeNodeHealth(ctx context.Context, store HealthStore, last **NodeHealth, h NodeHealth, now func() time.Time) (bool, error) {
|
||||
if *last == nil {
|
||||
raw, err := store.Get(ctx, h.Node)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the health report of %s cannot be read: %w", h.Node, err)
|
||||
}
|
||||
if raw != nil {
|
||||
if was, err := DecodeNodeHealth(raw); err == nil {
|
||||
*last = &was
|
||||
}
|
||||
}
|
||||
}
|
||||
if *last != nil && SameNodeHealth(**last, h) {
|
||||
return false, nil
|
||||
}
|
||||
h.At = now().UTC()
|
||||
raw, err := EncodeNodeHealth(h)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := store.Put(ctx, h.Node, raw); err != nil {
|
||||
return false, fmt.Errorf("the health report of %s cannot be written: %w", h.Node, err)
|
||||
}
|
||||
*last = &h
|
||||
return true, nil
|
||||
}
|
||||
@@ -86,6 +86,9 @@ type Session struct {
|
||||
|
||||
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
|
||||
haves haveState
|
||||
// healthWritten is this node's health report as last written or read (WriteNodeHealth): the health loop alone
|
||||
// writes it, one look at a time.
|
||||
healthWritten *NodeHealth
|
||||
}
|
||||
|
||||
func (q *Session) init() {
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -94,6 +95,18 @@ type Finding struct {
|
||||
Writer string
|
||||
// Toward is what the failure points at: TowardHub, or each resolver's address that failed.
|
||||
Toward []string
|
||||
// Peers is, for the tunnel, each peer of the mesh's interface and whether it was seen (Peer).
|
||||
Peers []Peer
|
||||
}
|
||||
|
||||
// Peer is one peer of the mesh's tunnel as this machine sees it: its public key, the addresses it is allowed, and
|
||||
// whether it handshook within StaleHandshake. What the watchman reads to tell a machine nobody reaches from one whose
|
||||
// tunnel alone is down (network-unreachable). Seen, not the handshake's time: the time moves every two minutes, the
|
||||
// fact only when a peer comes or goes.
|
||||
type Peer struct {
|
||||
Key string `json:"key"`
|
||||
Addresses []string `json:"addresses,omitempty"`
|
||||
Seen bool `json:"seen"`
|
||||
}
|
||||
|
||||
// Part is one part's state, as the statement says it.
|
||||
@@ -115,6 +128,8 @@ type Statement struct {
|
||||
Since time.Time `json:"since"`
|
||||
At time.Time `json:"at"`
|
||||
Parts []Part `json:"parts"`
|
||||
// Peers are the tunnel's peers on the last look that read them, sorted by key; nil where the tunnel is not read.
|
||||
Peers []Peer `json:"peers,omitempty"`
|
||||
}
|
||||
|
||||
// Machine is what a look reads, replaced in tests.
|
||||
@@ -266,6 +281,9 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
|
||||
changed = true
|
||||
}
|
||||
st.State, st.Since = worst, j.overall.since
|
||||
// The tunnel's peers as this look read them; a peer seen or lost is a change of its own (network-unreachable).
|
||||
st.Peers = findings[PartTunnel].Peers
|
||||
changed = changed || !samePeers(j.said.Peers, st.Peers)
|
||||
j.said = st
|
||||
return st, changed
|
||||
}
|
||||
@@ -435,14 +453,22 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
|
||||
}
|
||||
}
|
||||
hub := ""
|
||||
allowed := map[string][]string{}
|
||||
for _, line := range strings.Split(ips, "\n") {
|
||||
f := strings.Fields(line)
|
||||
for _, prefix := range f[min(1, len(f)):] {
|
||||
allowed[f[0]] = append(allowed[f[0]], prefix)
|
||||
if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" {
|
||||
hub = f[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
peers := make([]Peer, 0, len(handshakes))
|
||||
for key, at := range handshakes {
|
||||
peers = append(peers, Peer{Key: key, Addresses: allowed[key], Seen: at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake})
|
||||
}
|
||||
sort.Slice(peers, func(i, k int) bool { return peers[i].Key < peers[k].Key })
|
||||
found := func(f Finding) Finding { f.Peers = peers; return f }
|
||||
age := func(at int64) string {
|
||||
if at == 0 {
|
||||
return "never"
|
||||
@@ -452,23 +478,36 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
|
||||
if hub != "" {
|
||||
at := handshakes[hub]
|
||||
if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake {
|
||||
return Finding{OK: true}
|
||||
return found(Finding{OK: true})
|
||||
}
|
||||
return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}
|
||||
return found(Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}})
|
||||
}
|
||||
if len(handshakes) == 0 {
|
||||
return Finding{OK: true}
|
||||
return found(Finding{OK: true})
|
||||
}
|
||||
var newest int64
|
||||
for _, at := range handshakes {
|
||||
newest = max(newest, at)
|
||||
}
|
||||
if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake {
|
||||
return Finding{OK: true}
|
||||
return found(Finding{OK: true})
|
||||
}
|
||||
return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}
|
||||
return found(Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))})
|
||||
}
|
||||
|
||||
// samePeers says two looks saw the same peers, each seen or not alike.
|
||||
func samePeers(a, b []Peer) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i].Key != b[i].Key || a[i].Seen != b[i].Seen || !slices.Equal(a[i].Addresses, b[i].Addresses) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (j *Judge) lookBus() Finding {
|
||||
|
||||
@@ -302,9 +302,27 @@ func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) {
|
||||
f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n"
|
||||
f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n"
|
||||
j := f.judge(t)
|
||||
if st := f.look(t, j); st.State != Healthy {
|
||||
st := f.look(t, j)
|
||||
if st.State != Healthy {
|
||||
t.Fatalf("the hub with one fresh peer is %+v", st.Parts)
|
||||
}
|
||||
// Each peer and whether it was seen within five minutes, for the watchman's network-unreachable (issue 774); a
|
||||
// peer seen again is a change, a handshake renewed alone is not.
|
||||
want := []Peer{{Key: "A=", Addresses: []string{"10.10.0.2/32"}}, {Key: "B=", Addresses: []string{"10.10.0.3/32"}, Seen: true},
|
||||
{Key: "C=", Addresses: []string{"10.10.0.4/32"}}}
|
||||
if !samePeers(st.Peers, want) {
|
||||
t.Fatalf("the peers are %+v", st.Peers)
|
||||
}
|
||||
f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n"
|
||||
f.now = f.now.Add(LookEvery)
|
||||
if st, changed := j.Look(t.Context()); !changed || !st.Peers[0].Seen {
|
||||
t.Fatalf("a peer seen again is no change: changed %v, %+v", changed, st.Peers)
|
||||
}
|
||||
f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n"
|
||||
f.now = f.now.Add(LookEvery)
|
||||
if _, changed := j.Look(t.Context()); changed {
|
||||
t.Fatal("handshakes renewed alone are a change")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) {
|
||||
|
||||
@@ -21,6 +21,9 @@
|
||||
{
|
||||
"name": "acts",
|
||||
"ttl-seconds": 7776000
|
||||
},
|
||||
{
|
||||
"name": "health"
|
||||
}
|
||||
],
|
||||
"config": {
|
||||
|
||||
@@ -74,7 +74,10 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(parsed.State) != 1 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 {
|
||||
// 774 (the alert table): beside it the state health, one key per node, the node's own health report, kept until
|
||||
// written again (no TTL: a node that stops writing keeps its last word, which the watchman reads with its time).
|
||||
if len(parsed.State) != 2 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 ||
|
||||
parsed.State[1].Name != "health" || parsed.State[1].TTLSeconds != 0 {
|
||||
t.Fatalf("the manifest declares the state %+v", parsed.State)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user