mesh-controller/internal/link/busfaults.go

package link

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"time"

	"github.com/nats-io/nats.go"

	"github.com/novox/mesh-controller/internal/broker"
)

// What the bus did to other users' connections, as the bus module says it (novox/hq ADR 0357, issue 270).
//
// **The server says another user's slow consumer only in its record of closed connections**, which the
// bus module reads on its own machine and says as events: `bus.slow-consumer` for a connection the server
// closed as a slow consumer, `bus.connection-closed` for one it closed for breaking the bus's rules. They
// are read back from EVENTS on every watchdog tick for the hour S9 says an advisory, rather than taken by
// the controller's durable consumer: that one takes a message at a time while a merge builds for
// minutes, and read back, a controller that restarts says the last hour again without keeping anything.
//
// **Said at least once, counted once.** The module says the last hour again whenever it starts; a closure
// is its user, its connection id and when it opened, and is counted once however often it is said.

// BusModule is the module whose events say what the bus did to its users' connections.
const BusModule = "nats"

// The bus module's events, by their local names (its manifest's `emits`).
const (
	BusSlowConsumerEvent     = "bus.slow-consumer"
	BusConnectionClosedEvent = "bus.connection-closed"
	// BusWatchEvent is the bus module's watch saying it is there after every look, and whether it is healthy
	// (novox/hq ADR 0357, review 274): a watch that stops saying it is a watch nobody would otherwise hear.
	BusWatchEvent = "bus.watch"
)

// AdvisoryConnectionClosed is a connection the bus closed for breaking its rules, as conditions name it.
const AdvisoryConnectionClosed = "connection-closed"

// BusClosure is one closed connection as the bus module says it: the user, never a credential.
type BusClosure struct {
	User    string    `json:"user"`
	Name    string    `json:"name,omitempty"`
	CID     uint64    `json:"cid"`
	Reason  string    `json:"reason"`
	Opened  time.Time `json:"opened"`
	Closed  time.Time `json:"closed"`
	Pending int64     `json:"pending_bytes,omitempty"`
}

func (c BusClosure) identity() string {
	return fmt.Sprintf("%s/%d/%s", c.User, c.CID, c.Opened.UTC().Format(time.RFC3339Nano))
}

// BusClosures are the advisories the bus module's closures make, closed at or after since: one per user
// and kind, each closure counted once, the newest said.
func BusClosures(closures []BusClosure, kinds []string, since time.Time) []Advisory {
	type entry struct {
		a      *Advisory
		seen   map[string]bool
		newest BusClosure
	}
	by := map[string]*entry{}
	var order []string
	for i, c := range closures {
		if c.User == "" || c.Closed.Before(since) {
			continue
		}
		kind := kinds[i]
		key := kind + "/" + c.User
		e := by[key]
		if e == nil {
			e = &entry{a: &Advisory{Kind: kind, ID: c.User, User: c.User}, seen: map[string]bool{}}
			by[key] = e
			order = append(order, key)
		}
		if e.seen[c.identity()] {
			continue
		}
		e.seen[c.identity()] = true
		e.a.Count++
		if e.a.First.IsZero() || c.Closed.Before(e.a.First) {
			e.a.First = c.Closed
		}
		if !c.Closed.Before(e.a.Last) {
			e.a.Last, e.newest = c.Closed, c
		}
	}
	sort.Strings(order)
	out := make([]Advisory, 0, len(order))
	for _, key := range order {
		e := by[key]
		c := e.newest
		name := ""
		if c.Name != "" && c.Name != c.User {
			name = fmt.Sprintf(" (connection %q)", c.Name)
		}
		pending := ""
		if c.Pending > 0 {
			pending = fmt.Sprintf(", %d bytes still to be written to it", c.Pending)
		}
		switch e.a.Kind {
		case AdvisorySlowConsumer:
			e.a.Said = fmt.Sprintf("the bus closed %s's connection %d%s as a slow consumer: %s%s", c.User, c.CID,
				name, c.Reason, pending)
		default:
			e.a.Said = fmt.Sprintf("the bus closed %s's connection %d%s: %s", c.User, c.CID, name, c.Reason)
		}
		e.a.Reason = c.Reason
		out = append(out, *e.a)
	}
	return out
}

// busEventKind is the advisory kind of one of the bus module's events, by its subject.
func busEventKind(subject string) (string, bool) {
	switch subject {
	case EventSubject(BusModule, BusSlowConsumerEvent):
		return AdvisorySlowConsumer, true
	case EventSubject(BusModule, BusConnectionClosedEvent):
		return AdvisoryConnectionClosed, true
	}
	return "", false
}

// BusFaults reads back what the bus module said of its users' connections since a moment, from the events
// stream, on a consumer of its own that acknowledges nothing (as PowerStates reads), and says it as
// advisories. Nothing to read is no advisory, and costs two direct reads: the newest of each event.
func (s *Server) BusFaults(ctx context.Context, since time.Time) ([]Advisory, error) {
	if s.js == nil {
		return nil, errors.New("this control plane is not on the bus, so it cannot read what the bus module said")
	}
	js := s.js.Context()
	recent := false
	for _, event := range []string{BusSlowConsumerEvent, BusConnectionClosedEvent} {
		last, err := js.GetLastMsg(broker.EventsStream, EventSubject(BusModule, event), nats.Context(ctx))
		if errors.Is(err, nats.ErrMsgNotFound) {
			continue
		}
		if err != nil {
			return nil, fmt.Errorf("reading what the bus module said of its users' connections: %w", err)
		}
		if !last.Time.Before(since) {
			recent = true
		}
	}
	if !recent {
		return nil, nil
	}
	sub, err := js.SubscribeSync(EventSubject(BusModule, "bus.>"), nats.OrderedConsumer(), nats.StartTime(since))
	if err != nil {
		return nil, fmt.Errorf("reading what the bus module said of its users' connections: %w", err)
	}
	defer func() { _ = sub.Unsubscribe() }()
	var closures []BusClosure
	var kinds []string
	read, cut := 0, false
	for {
		if read >= MaxBusEventsRead {
			cut = true
			break
		}
		read++
		wait, cancel := context.WithTimeout(ctx, readQuiet)
		msg, err := sub.NextMsgWithContext(wait)
		cancel()
		if err != nil {
			if ctx.Err() != nil {
				return nil, ctx.Err()
			}
			break
		}
		if kind, ok := busEventKind(msg.Subject); ok {
			var c BusClosure
			if json.Unmarshal(msg.Data, &c) == nil && c.User != "" && !c.Closed.IsZero() {
				closures, kinds = append(closures, c), append(kinds, kind)
			}
		}
		if meta, err := msg.Metadata(); err != nil || meta.NumPending == 0 {
			break
		}
	}
	out := BusClosures(closures, kinds, since)
	if cut {
		// Said, never silent: a busy hour is read up to the bound, and each advisory says it is a floor.
		for i := range out {
			out[i].Said += fmt.Sprintf(" (at least: the read stopped at %d of the bus module's events this hour)",
				MaxBusEventsRead)
		}
	}
	return out, nil
}

// MaxBusEventsRead bounds one tick's read of the bus module's events: a watchdog tick must not spend itself
// reading a storm of closures back. A read cut there says so in every advisory it makes.
var MaxBusEventsRead = 2000

// MergeAdvisories is two lists of advisories as one: the same kind about the same thing is one advisory,
// counted once (the larger count of the two), said by the newer. The controller's own slow consumer as its client library sees it and
// as the server closed it are the same user's (`controller`).
func MergeAdvisories(a, b []Advisory) []Advisory {
	out := append([]Advisory(nil), a...)
	at := map[string]int{}
	for i, x := range out {
		at[x.Kind+"/"+x.ID+"/"+x.Token] = i
	}
	for _, y := range b {
		key := y.Kind + "/" + y.ID + "/" + y.Token
		i, had := at[key]
		if !had {
			at[key] = len(out)
			out = append(out, y)
			continue
		}
		x := &out[i]
		// The same fault seen twice, not two faults: the controller's own slow consumer is told to its client
		// library and closed by the server, and is counted once — by whichever saw more of it.
		if y.Count > x.Count {
			x.Count = y.Count
		}
		if y.First.Before(x.First) {
			x.First = y.First
		}
		if y.Last.After(x.Last) {
			x.Last, x.Said = y.Last, y.Said
			if y.Reason != "" {
				x.Reason = y.Reason
			}
		}
		if x.User == "" {
			x.User = y.User
		}
	}
	return out
}

// BusWatchBeat is the bus module's watch's newest heartbeat.
type BusWatchBeat struct {
	At      time.Time `json:"at"`
	Healthy bool      `json:"healthy"`
	Why     string    `json:"why,omitempty"`
	// Machine is the machine the runtime stamped on it (`x-node`).
	Machine string `json:"-"`
}

// BusWatchLast is the newest heartbeat of the bus module's watch, read directly from the events stream; false
// when the stream holds none — a bus module older than its watch, which says nothing either way.
func (s *Server) BusWatchLast(ctx context.Context) (BusWatchBeat, bool, error) {
	if s.js == nil {
		return BusWatchBeat{}, false, errors.New("this control plane is not on the bus, so it cannot read the bus module's watch")
	}
	msg, err := s.js.Context().GetLastMsg(broker.EventsStream, EventSubject(BusModule, BusWatchEvent), nats.Context(ctx))
	if errors.Is(err, nats.ErrMsgNotFound) {
		return BusWatchBeat{}, false, nil
	}
	if err != nil {
		return BusWatchBeat{}, false, fmt.Errorf("reading the bus module's watch's heartbeat: %w", err)
	}
	var b BusWatchBeat
	if err := json.Unmarshal(msg.Data, &b); err != nil || b.At.IsZero() {
		b = BusWatchBeat{At: msg.Time, Why: "its heartbeat could not be read"}
	}
	b.Machine = msg.Header.Get("x-node")
	return b, true, nil
}