jochen
0ae7933d54
The mesh noticed 48 core failures in six days and told nobody (ADR 0227). messenger holds the operator-channel seat: it consumes the controller's condition events and sends them to Telegram and the desktop notifier, deduplicated by key, reminded once, edited on clear, capped at 20 an hour with the rest folded, and refusing anything carrying an address, a path or a secret. mesh-watcher, on a machine other than the control node, sends to Telegram directly when the self-check heartbeat or the bus goes silent.
115 lines
3.8 KiB
Go
115 lines
3.8 KiB
Go
package main
|
|
|
|
// The self-check's heartbeat, read in this one place (novox/hq to-be 45 §4, S10).
|
|
//
|
|
// **The contract this reads, and every assumption it makes**, because the controller's side was built
|
|
// at the same time from the same design, which says only "every run ends with a heartbeat event
|
|
// carrying the run's id and counts":
|
|
//
|
|
// - The event is the mesh-controller seat's own, `doctor-heartbeat`, consumed as
|
|
// `mesh-controller.doctor-heartbeat` (subject `mesh.seat.mesh-controller.event.doctor-heartbeat`).
|
|
// - The body is one JSON object: `run` (the run's id), `at` (when the run ended, RFC 3339),
|
|
// `interval-seconds` (how often doctor runs), and `counts` ({pass, fail, failed-to-run}).
|
|
// `finished`/`time` are read for `at`; `interval_seconds`, `interval` (seconds, or a duration such
|
|
// as "5m") for the interval.
|
|
// - Without a time, the heartbeat is taken as made when it arrived (said in the status as such).
|
|
// Without an interval, the design's five minutes stand.
|
|
// - The counts are kept for the status only. A run that found failures is still a heartbeat: the
|
|
// watcher watches that the checker runs; what it finds is the controller's to raise.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// HeartbeatEvent is the event's local name under the controller's seat.
|
|
const (
|
|
HeartbeatEvent = "doctor-heartbeat"
|
|
ControllerSeat = "mesh-controller"
|
|
)
|
|
|
|
// Beat is one heartbeat.
|
|
type Beat struct {
|
|
Run string
|
|
At time.Time
|
|
Interval time.Duration
|
|
Counts map[string]int
|
|
}
|
|
|
|
// isHeartbeat says whether an envelope's key is the controller's heartbeat.
|
|
func isHeartbeat(key string) bool { return key == ControllerSeat+"."+HeartbeatEvent }
|
|
|
|
// DecodeBeat reads one heartbeat's body.
|
|
func DecodeBeat(body []byte) (Beat, error) {
|
|
var raw map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &raw); err != nil || raw == nil {
|
|
return Beat{}, fmt.Errorf("%s: the body is not a JSON object", HeartbeatEvent)
|
|
}
|
|
var b Beat
|
|
if v, ok := raw["run"]; ok {
|
|
var s string
|
|
var n float64
|
|
switch {
|
|
case json.Unmarshal(v, &s) == nil:
|
|
b.Run = s
|
|
case json.Unmarshal(v, &n) == nil:
|
|
b.Run = fmt.Sprintf("%.0f", n)
|
|
default:
|
|
return Beat{}, fmt.Errorf("%s: run is neither a string nor a number", HeartbeatEvent)
|
|
}
|
|
}
|
|
for _, name := range []string{"at", "finished", "time"} {
|
|
v, ok := raw[name]
|
|
if !ok || string(v) == "null" {
|
|
continue
|
|
}
|
|
var s string
|
|
if json.Unmarshal(v, &s) != nil {
|
|
return Beat{}, fmt.Errorf("%s: %s is not a string", HeartbeatEvent, name)
|
|
}
|
|
t, err := time.Parse(time.RFC3339Nano, s)
|
|
if err != nil {
|
|
return Beat{}, fmt.Errorf("%s: %s is not an RFC 3339 time: %q", HeartbeatEvent, name, s)
|
|
}
|
|
b.At = t
|
|
break
|
|
}
|
|
for _, name := range []string{"interval-seconds", "interval_seconds", "interval"} {
|
|
v, ok := raw[name]
|
|
if !ok || string(v) == "null" {
|
|
continue
|
|
}
|
|
var n float64
|
|
var s string
|
|
switch {
|
|
case json.Unmarshal(v, &n) == nil:
|
|
b.Interval = time.Duration(n * float64(time.Second))
|
|
case json.Unmarshal(v, &s) == nil:
|
|
d, err := time.ParseDuration(strings.TrimSpace(s))
|
|
if err != nil {
|
|
return Beat{}, fmt.Errorf("%s: %s is not a duration: %q", HeartbeatEvent, name, s)
|
|
}
|
|
b.Interval = d
|
|
default:
|
|
return Beat{}, fmt.Errorf("%s: %s is neither seconds nor a duration", HeartbeatEvent, name)
|
|
}
|
|
if b.Interval < 0 || b.Interval > 24*time.Hour {
|
|
return Beat{}, fmt.Errorf("%s: an interval of %s is not one doctor runs at", HeartbeatEvent, b.Interval)
|
|
}
|
|
break
|
|
}
|
|
if v, ok := raw["counts"]; ok && string(v) != "null" {
|
|
var counts map[string]float64
|
|
if json.Unmarshal(v, &counts) != nil {
|
|
return Beat{}, fmt.Errorf("%s: counts is not an object of numbers", HeartbeatEvent)
|
|
}
|
|
b.Counts = map[string]int{}
|
|
for k, n := range counts {
|
|
b.Counts[k] = int(n)
|
|
}
|
|
}
|
|
return b, nil
|
|
}
|