Write each node's health report into its own key of mesh-host_health on a change, so the watchman raises the machine's alerts without the controller (issue 774, the operator's alert table)
Until now the six machine alerts came from a health message passing to the controller. The report is now kept in the engine's declared state `health`, key `<node>`, created by the grants part as `acts` is. Only this node's engine writes it, after comparing it on every look and only when what it says changed; it is read first after a restart. It carries: - the network parts (machine-network; network-rewritten through the resolver file part's writer and owner); - the tunnel's peers, each seen within five minutes or not (network-unreachable); - the failed units no module owns (machine-units); - the accounts marked relogin needed (relogin-needed); - the directories used as found (directory-used-as-found). The streaks and the time of the look are left out, so a look alone writes nothing. A peer is reported seen or not, never its handshake time, so a renewed handshake writes nothing. The layout is internal/link's (NodeHealth) until it moves into mesh-sdk. Pins mesh-sdk 58a8324. Tests: - rows in TestOnTheHubAnyFreshPeerIsAHealthyTunnel: the peers, and a change only when one is seen or lost; - a case in TestAHealthStatementIsSaidOnTheLinkOpenNow: what the report takes, and one write per change across a restart; - the manifest's states in TestTheManifestConsumesTheBackupEvent.
This commit is contained in:
+32
-2
@@ -1353,6 +1353,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
var lastSaid time.Time
|
||||
owed := true
|
||||
spaced := 1.0
|
||||
// healthUnwritten is why the health report was last not written, said once each time it changes.
|
||||
var healthUnwritten string
|
||||
// A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing
|
||||
// long-running is then ok within a look of its apply, not at the five-minute statement.
|
||||
var nudged <-chan struct{}
|
||||
@@ -1436,6 +1438,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
|
||||
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 &&
|
||||
accessesHealthy(accessSt)
|
||||
h := *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
||||
unitSt), accountSt), found), accessSt)
|
||||
// The node's own health report in its key on the bus, compared on every look and written only when what it says
|
||||
// changed (issue 774): what the watchman raises the machine's alerts from, without the controller.
|
||||
writeHealthReport(ctx, queue, h, netSt, &healthUnwritten, say)
|
||||
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
||||
continue
|
||||
}
|
||||
@@ -1462,13 +1469,36 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
reconcile(inactive...)
|
||||
}
|
||||
}
|
||||
if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
||||
unitSt), accountSt), found), accessSt)) {
|
||||
if queue.SayHealth(ctx, h) {
|
||||
lastSaid, owed = time.Now(), false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// writeHealthReport writes this node's health report to its key (link.WriteNodeHealth) when it changed. Why it could
|
||||
// not is said once per reason, and said again when it is written after all.
|
||||
func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, ns *network.Statement, unwritten *string,
|
||||
say link.Announce) {
|
||||
if queue == nil {
|
||||
return
|
||||
}
|
||||
var peers []link.PeerSeen
|
||||
if ns != nil {
|
||||
for _, p := range ns.Peers {
|
||||
peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen})
|
||||
}
|
||||
}
|
||||
_, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers))
|
||||
switch {
|
||||
case err != nil && err.Error() != *unwritten:
|
||||
*unwritten = err.Error()
|
||||
say("this machine's health report is not written to the bus: " + err.Error())
|
||||
case err == nil && *unwritten != "":
|
||||
*unwritten = ""
|
||||
say("this machine's health report is written to the bus again")
|
||||
}
|
||||
}
|
||||
|
||||
// healthAsReported is a statement as the report and the event carry it.
|
||||
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
||||
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
||||
|
||||
Reference in New Issue
Block a user