mesh-catalog/modules/mesh-watcher/cmd/mesh-watcher/main.go

// mesh-watcher: the watcher's watcher (novox/hq to-be 45 §5, S10, ADR 0227 rule 6). A Go bundle the
// node's runtime launches on one machine that is not the control node. It hears the self-check's
// heartbeat and makes a round trip through the bus every minute; when either goes silent
// past its bound, it tells the operator on Telegram directly over HTTPS — the one sender that does not
// pass through the control node — and tells them again when it returns. stdout is the MCP channel;
// what this module says, it says on stderr.
package main

import (
	"context"
	"encoding/json"
	"fmt"
	assignerclient "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
	mesh "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
	"os"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
	meshselfcheck "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshselfcheck"
	stdio "git.novox.be/novox/mesh-sdk/go"

	"git.novox.be/novox/mesh-sdk/go/views"
)

func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }

func readChatID(path string) (string, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return "", err
	}
	var s map[string]any
	if err := json.Unmarshal(raw, &s); err != nil {
		return "", fmt.Errorf("the settings file is not JSON: %v", err)
	}
	switch v := s["telegram-chat-id"].(type) {
	case string:
		return strings.TrimSpace(v), nil
	case float64:
		return fmt.Sprintf("%.0f", v), nil
	}
	return "", nil
}

type listening struct {
	mu  sync.Mutex
	now string
}

func (l *listening) set(s string) { l.mu.Lock(); l.now = s; l.mu.Unlock() }
func (l *listening) get() string  { l.mu.Lock(); defer l.mu.Unlock(); return l.now }

func main() {
	settings := os.Getenv("MESH_WATCHER_SETTINGS")
	var said sync.Mutex
	lastSaid := ""
	tg := NewTelegram(TelegramConfig{
		TokenFile: os.Getenv("MESH_WATCHER_TELEGRAM_TOKEN_FILE"),
		ChatID: func() string {
			id, err := readChatID(settings)
			said.Lock()
			defer said.Unlock()
			if err != nil && err.Error() != lastSaid {
				logf("[mesh-watcher] settings cannot be read: %v", err)
			}
			lastSaid = ""
			if err != nil {
				lastSaid = err.Error()
			}
			return id
		},
	})
	node := os.Getenv("MESH_NODE")
	w := NewWatcher(tg, time.Now, logf, node)
	l := &listening{now: "not yet: starting"}
	assignments := &views.Reader{Module: "mesh-watcher", Logf: logf, Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}}
	assignments.Follow(assignerModule, mesh.StateSeatHolders, SeatHoldersTool)
	go run(w, l, node, assignments)
	if err := stdio.Serve("", append(tools(w, l), assignments.Tools("watcher")...)); err != nil {
		logf("%v", err)
		os.Exit(1)
	}
}

// run keeps time, makes the round trips, reads where the controller is, and listens for heartbeats.
// The clock runs whatever the bus does: it is what notices the bus is gone.
func run(w *Watcher, l *listening, node string, assignments *views.Reader) {
	if err := w.Telegram.Ready(); err != nil {
		logf("[mesh-watcher] BLIND until given: %v", err)
	}
	go func() {
		for range time.Tick(time.Minute) {
			go func() { w.BusAnswered(roundTrip(node)) }()
			w.Tick()
		}
	}()
	// Where the bus runs: the assigner's seat-holders, asked once and then followed by its events (ADR 0615), never read
	// from its state and never on a timer.
	placed := func() {
		if nodes, err := controlNodes(assignments); err == nil {
			w.Placed(nodes)
		} else {
			logf("[mesh-watcher] cannot read where the bus runs (%v)", err)
		}
	}
	assignments.Changed = func(string, string, json.RawMessage, bool) { placed() }
	assignments.Start(context.Background())
	handle := func(e stdio.Envelope) error {
		if !isHeartbeat(e.Key) {
			return nil
		}
		b, err := DecodeBeat(e.Body)
		if err != nil {
			w.BadHeartbeat(err)
			return nil
		}
		w.Heartbeat(b)
		return nil
	}
	<-stdio.Served() // subscribed as soon as the runtime serves, never after a guessed wait: a heartbeat sent meanwhile is heard
	for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
		err := stdio.Subscribe(meshselfcheck.Seat+".*", handle)
		if err == nil {
			l.set("listening")
			logf("[mesh-watcher] listening for the self-check's heartbeat; watching from %s", node)
			return
		}
		l.set("not yet: " + err.Error())
		logf("[mesh-watcher] not hearing heartbeats yet (%v); asking again in %s", err, wait)
		time.Sleep(wait)
	}
}

// roundTrip asks this module's own ping on this machine, through the bus server: an answer is the bus
// carrying a request and its reply.
func roundTrip(node string) error {
	key := "mesh-watcher.watcher_ping"
	if node != "" {
		key += "@" + node
	}
	done := make(chan error, 1)
	go func() {
		_, err := stdio.Ask(key, map[string]any{})
		done <- err
	}()
	select {
	case err := <-done:
		return err
	case <-time.After(30 * time.Second):
		return fmt.Errorf("no answer in 30 s")
	}
}

// SeatHoldersTool is the assigner's read tool of the state of who holds which seat, which the manifest invokes; the
// state is followed as a view (ADR 0615), named as the manifest named it before: SeatHoldersRead.
// assignerModule is the holder of the seat mesh-module-assigner whose seat-holders view this module follows: the one name
// of it here, the same as its manifest's `consumes` and `invokes` carry (the manifest test holds them equal).
const assignerModule = "mesh-delivery-assigner"

const SeatHoldersTool = assignerModule + ".assigner_read_seat_holders"

// controlNodes reads which machines hold the bus, from the assignments seat's state of who holds which seat (novox/hq
// ADR 0571 clause 4). The controller is not placed (ADR 0590), so the bus's machines are the control node.
func controlNodes(seats *views.Reader) ([]string, error) {
	keys, err := seats.Values(assignerModule + "." + mesh.StateSeatHolders)
	if err != nil {
		return nil, err
	}
	return holderNodes(keys, "mesh-broker")
}

// holderNodes is the machines holding any of the seats, in the order the seats are named, each once.
func holderNodes(keys map[string][]byte, seats ...string) ([]string, error) {
	var out []string
	seen := map[string]bool{}
	for _, seat := range seats {
		holders, err := assignerclient.HoldersOf(keys, seat, "")
		if err != nil {
			return nil, err
		}
		for _, h := range holders {
			if !seen[h.Node] {
				seen[h.Node] = true
				out = append(out, h.Node)
			}
		}
	}
	if len(out) == 0 {
		return nil, fmt.Errorf("the assignments seat's state names no holder of %s", strings.Join(seats, " or "))
	}
	return out, nil
}

func tools(w *Watcher, l *listening) []stdio.Tool {
	return []stdio.Tool{
		{Name: "watcher_status",
			Description: "The watcher's own health first — whether it can tell the operator anything (the Telegram token and " +
				"chat id), whether it runs on the machine it watches, whether heartbeats reach it — then each watched " +
				"signal (the self-check's heartbeat, a round trip through the bus): last heard, how long ago, " +
				"its bound, whether it is said silent, and any message not sent yet.",
			Run: func(map[string]any) (any, error) { return w.Status(l.get()), nil }},
		{Name: "watcher_last_heard",
			Description: "When each watched signal was last heard, and what the watcher sent to Telegram lately, newest first.",
			Run:         func(map[string]any) (any, error) { return w.LastHeard(), nil }},
		{Name: "watcher_test",
			Description: "Send a test message to Telegram now, directly: proves the watcher can reach the operator when the " +
				"mesh cannot. Answers sent, or why not.",
			Run: func(map[string]any) (any, error) {
				err := w.send("test", Message{Title: "TEST: mesh-watcher on " + w.Node + " reaches you directly",
					Body: "nothing is wrong; this was asked for"})
				if err != nil {
					return map[string]string{"telegram": "not sent: " + err.Error()}, nil
				}
				return map[string]string{"telegram": "sent"}, nil
			}},
		{Name: "watcher_ping",
			Description: "Answers at once: the round trip the watcher makes through the bus every minute to know the bus carries a request and its reply.",
			Run: func(map[string]any) (any, error) {
				return map[string]string{"pong": time.Now().UTC().Format(time.RFC3339)}, nil
			}},
	}
}