mesh-delivery/modules/mesh-delivery-watchman/internal/watch/busstate.go

package watch

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)

// **A refused or overdue creation of bus state is loud** (novox/hq ADR 0595 §4; design 25 § 12.2; DLV-W1). The watchman
// hears, through its `consumes`, the indexer's `manifest-accepted` (which carries the module's bus-state entries whole)
// and the bus module's own `created` and `refused`. An entry an accepted manifest asks for that has no `created` or
// `refused` within OverdueBound is the finding bus-state-overdue, naming the module and the entry; a `refused` is the
// finding bus-state-refused, with why. A later `created` clears both, and so does a later accepted manifest of that
// module that no longer asks for the entry. Never the bus module's to raise: it cannot while it is down.
//
// The bound is a row of the signals table: a pulse key per entry with OverdueBound to live, whose expiry is the finding,
// so no loop looks and a pulse that stood across a restart still acts. Both findings outlive a restart (signalKind),
// since what ends them is heard, not read.
//
// The bus module says `created` once per entry, so an entry it said created is not waited on again when a later manifest
// of its module is accepted with the same entry (review round 1): its mark, a pulse key with no time to live under the
// row bus-state-created, is read back at a restart (SeedSignal), and goes when a refused is heard or a manifest no
// longer asks for the entry, so a later ask of it waits again.

// OverdueBound is how long an accepted entry may wait for the bus module's created or refused.
const OverdueBound = 5 * time.Minute

// The kinds, and the signals table's row.
const (
	KindBusStateOverdue = "bus-state-overdue"
	KindBusStateRefused = "bus-state-refused"
	rowBusState         = "bus-state"
	rowBusCreated       = "bus-state-created"
)

// BrokerSaid is the part of the bus module's created or refused the watchman reads: the module, the entry, and why a
// create was refused. It stands until the seat mesh-broker's definition is generated in mesh-contracts.
type BrokerSaid struct {
	Module string `json:"module"`
	// Local is the entry: the contract's `local-name` (mesh-contracts meshbroker.Said), or `local` where a body names it so.
	Local     string `json:"local,omitempty"`
	LocalName string `json:"local-name,omitempty"`
	Why       string `json:"why,omitempty"`
}

// busStateID is the signal's and the findings' id of one entry: `<module>.<local>`.
func busStateID(module, local string) string { return module + "." + local }

func overdueKey(module, local string) string {
	return FindingKey(ScopeModule, module, KindBusStateOverdue+"-"+local)
}

func refusedKey(module, local string) string {
	return FindingKey(ScopeModule, module, KindBusStateRefused+"-"+local)
}

// Accepted is the indexer's manifest-accepted heard: each entry it asks for waits OverdueBound for the bus module's word,
// and every entry of the module this watchman waits on or holds a finding of that it no longer asks for is ended.
func (w *Watchman) Accepted(a meshcatalog.ManifestAcceptedEvent) {
	w.mu.Lock()
	defer w.mu.Unlock()
	asked := map[string]bool{}
	for _, e := range a.BusState {
		if e.Name == "" {
			continue
		}
		asked[e.Name] = true
		if w.busCreated[signalKey(rowBusCreated, busStateID(a.Module, e.Name))] {
			continue // said created already, and created is said once per entry: nothing to wait for
		}
		// One refused since an earlier verdict waits again: a new verdict asks the bus module again.
		w.signal(rowBusState, busStateID(a.Module, e.Name), OverdueBound, signalDue{Do: dueFind, Finding: Finding{
			Scope: ScopeModule, ID: a.Module, Token: KindBusStateOverdue + "-" + e.Name, Kind: KindBusStateOverdue,
			Severity: Warning, Source: rowBusState, Since: w.now().Add(OverdueBound).UTC().Truncate(time.Second),
			Summary: fmt.Sprintf("the bus module said nothing of %s's %s (%s) within %s of its accepted manifest %s", a.Module,
				e.Name, e.Storage, OverdueBound, short(a.Digest)),
			Headline: "The bus state " + e.Name + " of " + a.Module + " was not created",
			Explanation: "The indexer accepted a manifest of " + a.Module + " asking the bus module for " + e.Name + ", and the bus " +
				"module said neither created nor refused within " + OverdueBound.String() + ": it is down, or behind. " + a.Module +
				" fails at its start while the entry's binding is missing (novox/hq ADR 0595 §1).",
			Needs:    "a look at the bus module's health and its process's log; nats_first_view_again asks it to read the verdicts again",
			Resolved: "the bus module said " + e.Name + " of " + a.Module + " created, or a later manifest no longer asks for it"}})
	}
	for _, local := range w.busStateEntriesOf(a.Module) {
		if asked[local] {
			continue
		}
		w.unsignal(rowBusState, busStateID(a.Module, local))
		w.unfind(overdueKey(a.Module, local))
		w.unfind(refusedKey(a.Module, local))
		w.uncreated(a.Module, local)
	}
}

// markCreated keeps that the bus module said module's local created, on the pulse bucket with no time to live.
func (w *Watchman) markCreated(module, local string) {
	pk := signalKey(rowBusCreated, busStateID(module, local))
	owes, _ := json.Marshal(signalDue{Do: dueCreated})
	value, err := meshwatchdogsclient.EncodePulse(meshwatchdogsclient.Pulse{Bucket: signalBucket, Key: strings.TrimPrefix(pk, signalBucket+"."), Owes: string(owes)})
	if err == nil {
		err = w.out.PutPulse(pk, value, 0)
	}
	if err != nil {
		w.say("cannot keep that %s of %s was created (%v): its next accepted manifest waits on it again", local, module, err)
	}
	w.busCreated[pk] = true
}

// uncreated drops the mark of module's local said created, so its next ask waits again.
func (w *Watchman) uncreated(module, local string) {
	pk := signalKey(rowBusCreated, busStateID(module, local))
	if !w.busCreated[pk] {
		return
	}
	delete(w.busCreated, pk)
	if err := w.out.DeletePulse(pk); err != nil {
		w.say("cannot delete the mark %s: %v", pk, err)
	}
}

// BrokerCreated is the bus module's created heard: the entry's wait ends, and its findings with it.
func (w *Watchman) BrokerCreated(s BrokerSaid) {
	w.mu.Lock()
	defer w.mu.Unlock()
	w.unsignal(rowBusState, busStateID(s.Module, s.Local))
	w.unfind(overdueKey(s.Module, s.Local))
	w.unfind(refusedKey(s.Module, s.Local))
	w.markCreated(s.Module, s.Local)
}

// BrokerRefused is the bus module's refused heard: the entry's wait ends in the finding bus-state-refused, until a later
// created.
func (w *Watchman) BrokerRefused(s BrokerSaid) {
	w.mu.Lock()
	defer w.mu.Unlock()
	w.unsignal(rowBusState, busStateID(s.Module, s.Local))
	w.unfind(overdueKey(s.Module, s.Local))
	w.uncreated(s.Module, s.Local)
	why := strings.TrimSpace(s.Why)
	if why == "" {
		why = "it gave no reason"
	}
	w.find(Finding{Scope: ScopeModule, ID: s.Module, Token: KindBusStateRefused + "-" + s.Local, Kind: KindBusStateRefused,
		Severity: Urgent, Source: rowBusState, Since: w.now().UTC().Truncate(time.Second), Said: why,
		Summary:  fmt.Sprintf("the bus module refused to create %s's %s: %s", s.Module, s.Local, why),
		Headline: "The bus state " + s.Local + " of " + s.Module + " was refused",
		Explanation: "The bus module could not create " + s.Local + " for " + s.Module + ", so no binding is handed and " + s.Module +
			" fails at its start, naming the entry (novox/hq ADR 0595 §1): " + why,
		Needs:    "a pull request that mends the entry in the manifest of " + s.Module + ", or the bus module's cause fixed",
		Resolved: "the bus module said " + s.Local + " of " + s.Module + " created"})
}

// busStateEntriesOf are the entries of module this watchman waits on or holds a finding of, by local name, sorted.
func (w *Watchman) busStateEntriesOf(module string) []string {
	seen := map[string]bool{}
	prefix := signalKey(rowBusState, module) + "."
	for pk := range w.signals {
		if local, ok := strings.CutPrefix(pk, prefix); ok && !strings.Contains(local, ".") {
			seen[local] = true
		}
	}
	for _, kind := range []string{KindBusStateOverdue, KindBusStateRefused} {
		p := FindingKey(ScopeModule, module, kind+"-")
		for key := range w.findings {
			if local, ok := strings.CutPrefix(key, p); ok {
				seen[local] = true
			}
		}
	}
	out := make([]string, 0, len(seen))
	for l := range seen {
		out = append(out, l)
	}
	sort.Strings(out)
	return out
}

// ReadBrokerSaid reads the bus module's created or refused.
func ReadBrokerSaid(body []byte) (BrokerSaid, error) {
	var s BrokerSaid
	if err := json.Unmarshal(body, &s); err != nil {
		return BrokerSaid{}, fmt.Errorf("the bus module's word cannot be read: %w", err)
	}
	if s.Local == "" {
		s.Local = s.LocalName
	}
	if s.Module == "" || s.Local == "" {
		return BrokerSaid{}, fmt.Errorf("the bus module's word names no module or no entry")
	}
	return s, nil
}

func short(digest string) string {
	if len(digest) > 12 {
		return digest[:12]
	}
	return digest
}