121 lines
4.1 KiB
Go
121 lines
4.1 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// Last heard (novox/hq ADR 0433 §3, issue 1183): one answer to when the controller last heard a machine, read from the
|
|
// watchman's presence. The alive message that used to keep last_seen fresh is gone; a report, which is rare, is the
|
|
// answer only where presence has no key for the machine.
|
|
|
|
// presenceReader reads presence.<node>'s entry for each node; a node without an entry is left out.
|
|
type presenceReader func(ctx context.Context, nodes []string) (map[string]presenceEntry, error)
|
|
|
|
// lastHeard is when a machine was last heard: now when presence says it is online, the entry's time when it says offline
|
|
// (the time it went), and the machine's last report only when presence has no entry for it. Zero is never heard.
|
|
func lastHeard(report time.Time, entry *presenceEntry, now time.Time) time.Time {
|
|
switch {
|
|
case entry == nil:
|
|
return report
|
|
case entry.Online:
|
|
return now
|
|
default:
|
|
return entry.At
|
|
}
|
|
}
|
|
|
|
// nodesHeard returns the nodes with their last-heard time (LastSeen) as lastHeard says it. Unreadable presence is
|
|
// returned as an error with the nodes unchanged, so a caller says it and shows the reports.
|
|
func nodesHeard(ctx context.Context, nodes []inventory.Node, read presenceReader, now time.Time) ([]inventory.Node, error) {
|
|
names := make([]string, len(nodes))
|
|
for i, n := range nodes {
|
|
names[i] = n.Name
|
|
}
|
|
entries, err := read(ctx, names)
|
|
if err != nil {
|
|
return nodes, fmt.Errorf("what the watchman says of the machines' presence cannot be read: %w", err)
|
|
}
|
|
out := make([]inventory.Node, len(nodes))
|
|
for i, n := range nodes {
|
|
var entry *presenceEntry
|
|
if e, ok := entries[n.Name]; ok {
|
|
entry = &e
|
|
}
|
|
n.LastSeen = lastHeard(n.LastSeen, entry, now)
|
|
out[i] = n
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// heardNodes is every node, read with its last-heard time from the watchman's presence over a connection of its own. A bus
|
|
// that cannot be reached is said on standard error and the reports stand, which reads a machine as quiet rather than well.
|
|
func (h *stores) heardNodes(ctx context.Context) ([]inventory.Node, error) {
|
|
nodes, err := h.inventory.Nodes(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return h.withPresence(ctx, nodes), nil
|
|
}
|
|
|
|
// heardNode is one node, read the same way.
|
|
func (h *stores) heardNode(ctx context.Context, name string) (inventory.Node, error) {
|
|
n, err := h.inventory.NodeByName(ctx, name)
|
|
if err != nil {
|
|
return n, err
|
|
}
|
|
return h.withPresence(ctx, []inventory.Node{n})[0], nil
|
|
}
|
|
|
|
func (h *stores) withPresence(ctx context.Context, nodes []inventory.Node) []inventory.Node {
|
|
address, err := broker.BusAddress()
|
|
var js *broker.JetStream
|
|
if err == nil {
|
|
js, err = broker.Dial(address)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "what the watchman says of the machines' presence cannot be read: %v\n", err)
|
|
return nodes
|
|
}
|
|
defer js.Close()
|
|
read := func(ctx context.Context, names []string) (map[string]presenceEntry, error) {
|
|
return readPresence(ctx, js.Conn(), names)
|
|
}
|
|
heard, err := nodesHeard(ctx, nodes, read, time.Now())
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
}
|
|
return heard
|
|
}
|
|
|
|
// heardFromAt says when a node was last heard from, in a form somebody can act on.
|
|
//
|
|
// "never" and "an hour ago" are different answers and are kept different. A node that has never spoken did not finish
|
|
// joining; a node last heard from an hour ago is running an hour-old picture of the mesh.
|
|
func heardFromAt(n inventory.Node, now time.Time) string {
|
|
if n.LastSeen.IsZero() {
|
|
return "never spoken"
|
|
}
|
|
if silent := now.Sub(n.LastSeen); silent > SilentFor {
|
|
return "out of touch " + roughly(silent)
|
|
}
|
|
return "here"
|
|
}
|
|
|
|
// notHeardLately is the nodes never heard from or not within the hour: different from failing, for a machine that says
|
|
// nothing may be new, switched off or unreachable, and none of those is a machine that tried and could not.
|
|
func notHeardLately(nodes []inventory.Node, now time.Time) []inventory.Node {
|
|
var out []inventory.Node
|
|
for _, n := range nodes {
|
|
if n.LastSeen.IsZero() || now.Sub(n.LastSeen) > time.Hour {
|
|
out = append(out, n)
|
|
}
|
|
}
|
|
return out
|
|
}
|