mesh-tools/node-tools/internal/console/runtimes.go

package console

// What discovery heard from each runtime (2026-10-05): the console said a module ran nowhere when the
// largest runtime's answer arrived after the window closed, and nothing showed that it had. Every
// round of discovery is remembered per runtime, so a runtime heard before and not now is named, and
// `mesh_runtimes` says who answered, how fast and how much.

import (
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/novox/mesh-tools/node-tools/internal/announce"
)

// RuntimeForgotten is how long a runtime heard once is expected again: within it, its silence is said
// wherever an address it might answer is asked for; after it, it is listed only by mesh_runtimes.
var RuntimeForgotten = 15 * time.Minute

// runtimeSeen is one runtime instance as discovery last heard it.
type runtimeSeen struct {
	Name      string
	ID        string
	Machine   string
	LastHeard time.Time
	Took      time.Duration
	Bytes     int
	Modules   int
	Tools     int
	SeatVerbs int
	Shortened string
	Answered  bool   // in the latest round
	Why       string // why not, when it did not
	// Reused is true when its PING said its announcement was unchanged, so it was not fetched again.
	Reused bool
	// Last is its last whole answer: what a round that does not hear it uses instead, and what an
	// unchanged digest stands for (novox/hq issue 554).
	Last announce.Heard
}

func runtimeKey(name, id string) string { return name + "/" + id }

// seenOf is what one answer announced, counted.
func seenOf(h announce.Heard, at time.Time) *runtimeSeen {
	modules := map[string]bool{}
	tools := map[string]bool{}
	verbs := map[string]bool{}
	for _, e := range announce.Endpoints(h.Info) {
		switch e.Kind {
		case announce.KindTool:
			modules[e.Module] = true
			tools[e.Module+"."+e.Tool] = true
		case announce.KindSeat:
			verbs[e.Seat+"."+e.Tool] = true
		}
	}
	return &runtimeSeen{Name: h.Info.Name, ID: h.Info.ID, Machine: h.Machine(), LastHeard: at, Took: h.Took,
		Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true,
		Reused: h.Reused, Last: h}
}

// held is the answer this console holds for an instance at this digest (announce.Known): an instance
// whose announcement is unchanged is not asked for it again (novox/hq issue 554).
func (s *Surface) held(name, id, digest string) (announce.Heard, bool) {
	s.mu.Lock()
	defer s.mu.Unlock()
	r := s.runtimes[runtimeKey(name, id)]
	if r == nil || r.Last.Digest == "" || r.Last.Digest != digest || r.Last.Info.Name == "" {
		return announce.Heard{}, false
	}
	h := r.Last
	h.Took, h.Reused = 0, false
	return h, true
}

// before is the last answer of every runtime this round did not hear, heard within RuntimeForgotten
// and not since restarted under another instance (novox/hq issue 554).
func (s *Surface) before(d announce.Discovery) []earlier {
	heard := map[string]bool{}
	running := map[string]bool{}
	for _, h := range d.Heard {
		heard[runtimeKey(h.Info.Name, h.Info.ID)] = true
		if m := h.Machine(); m != "" {
			running[h.Info.Name+"@"+m] = true
		}
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	var out []earlier
	for k, r := range s.runtimes {
		if heard[k] || r.Last.Info.Name == "" || r.LastHeard.IsZero() || time.Since(r.LastHeard) > RuntimeForgotten {
			continue
		}
		if r.Machine != "" && running[r.Name+"@"+r.Machine] {
			continue // restarted: the new instance answered
		}
		out = append(out, earlier{Heard: r.Last, At: r.LastHeard})
	}
	sort.Slice(out, func(i, j int) bool {
		return runtimeKey(out[i].Heard.Info.Name, out[i].Heard.Info.ID) < runtimeKey(out[j].Heard.Info.Name, out[j].Heard.Info.ID)
	})
	return out
}

// sayRound logs a round of discovery that did not hear every runtime — once, when what it missed
// changes, and once when every runtime is heard again — so the next case leaves evidence in the
// journal (novox/hq issue 554). Called with s.mu held, after remember.
func (s *Surface) sayRound(x *index) {
	d := x.Discovery
	var missed []string
	for _, u := range x.Unheard {
		who := u.Runtime
		if u.Machine != "" {
			who += " on " + u.Machine
		}
		missed = append(missed, who)
	}
	sort.Strings(missed)
	shape := strings.Join(missed, ", ")
	if d.Empty() {
		shape = "empty|" + shape
	}
	if shape == s.roundSaid {
		return
	}
	s.roundSaid = shape
	reused := 0
	for _, h := range d.Heard {
		if h.Reused {
			reused++
		}
	}
	logf := log.Printf
	if s.conn != nil && s.conn.Logf != nil {
		logf = s.conn.Logf
	}
	window := fmt.Sprintf("waited %s (window %s, patience %s)", d.Waited.Round(time.Millisecond), announce.Window, announce.Patience)
	held := ""
	if len(x.Earlier) > 0 {
		held = fmt.Sprintf("; what %d of them announced earlier is used, with its age", len(x.Earlier))
	}
	switch {
	case d.Empty():
		logf("[mesh-tools] discovery heard nothing at all: no PING arrived; %s; not heard: %s%s; asking again at the next question",
			window, orNone(missed), held)
	case len(missed) > 0:
		logf("[mesh-tools] discovery did not hear every runtime: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s; not heard: %s%s",
			d.Pinged, len(d.Heard), reused, window, shape, held)
	default:
		logf("[mesh-tools] discovery hears every runtime again: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s",
			d.Pinged, len(d.Heard), reused, window)
	}
}

func orNone(xs []string) string {
	if len(xs) == 0 {
		return "none heard before either"
	}
	return strings.Join(xs, ", ")
}

// remember records a round of discovery and adds to the index every runtime heard lately that did not
// answer this time. Called with s.mu held.
func (s *Surface) remember(x *index) {
	if s.runtimes == nil {
		s.runtimes = map[string]*runtimeSeen{}
	}
	at := x.Discovery.At
	if at.IsZero() {
		at = time.Now().UTC()
	}
	heard := map[string]bool{}
	for _, h := range x.Discovery.Heard {
		k := runtimeKey(h.Info.Name, h.Info.ID)
		seen := seenOf(h, at)
		if heard[k] {
			// Two instances under one name and id: both counted, the slower answer's time kept.
			prev := s.runtimes[k]
			seen.Modules += prev.Modules
			seen.Tools += prev.Tools
			seen.SeatVerbs += prev.SeatVerbs
			seen.Bytes += prev.Bytes
			if prev.Took > seen.Took {
				seen.Took = prev.Took
			}
		}
		heard[k] = true
		s.runtimes[k] = seen
	}
	// A runtime restarted answers under a new instance id: the old one is gone, not silent, and saying
	// it was missed would hide nothing but cry wolf after every restart.
	now := map[string]bool{}
	for k := range heard {
		now[s.runtimes[k].Name+"@"+s.runtimes[k].Machine] = true
	}
	for k, r := range s.runtimes {
		if !heard[k] && r.Machine != "" && now[r.Name+"@"+r.Machine] {
			delete(s.runtimes, k)
		}
	}
	silent := map[string]string{}
	for _, u := range x.Discovery.Silent {
		silent[runtimeKey(u.Name, u.ID)] = fmt.Sprintf("it answered PING; what it serves did not arrive within %s", announce.Patience)
	}
	for k, r := range s.runtimes {
		if heard[k] {
			continue
		}
		r.Answered = false
		used := ""
		if at, ok := x.Earlier[k]; ok {
			used = fmt.Sprintf("; what it announced %s ago is used", time.Since(at).Round(time.Second))
		}
		if why, ok := silent[k]; ok {
			r.Why = why
			if used != "" {
				for i := range x.Unheard {
					if x.Unheard[i].Runtime == r.Name && x.Unheard[i].Machine == r.Machine && x.Unheard[i].Why == why {
						x.Unheard[i].Why += used
					}
				}
			}
			continue // already in the index's unheard
		}
		r.Why = "it did not answer this round of discovery"
		if time.Since(r.LastHeard) <= RuntimeForgotten {
			x.Unheard = append(x.Unheard, unheard{Runtime: r.Name, Machine: r.Machine,
				Why: fmt.Sprintf("it answered discovery %s ago, not now%s", time.Since(r.LastHeard).Round(time.Second), used)})
		}
	}
	for _, u := range x.Discovery.Silent {
		k := runtimeKey(u.Name, u.ID)
		if _, known := s.runtimes[k]; !known {
			s.runtimes[k] = &runtimeSeen{Name: u.Name, ID: u.ID, Machine: u.Machine, Why: silent[k]}
		}
	}
}

// runtimesAnswer is mesh_runtimes: discovery asked now, and every runtime as it was heard.
func (s *Surface) runtimesAnswer() map[string]any {
	x, err := s.indexAsked(true)
	if err != nil {
		return failure("the mesh's discovery failed: " + err.Error())
	}
	s.mu.Lock()
	var all []runtimeSeen
	for _, r := range s.runtimes {
		all = append(all, *r)
	}
	s.mu.Unlock()
	sort.Slice(all, func(i, j int) bool {
		if all[i].Machine != all[j].Machine {
			return all[i].Machine < all[j].Machine
		}
		return runtimeKey(all[i].Name, all[i].ID) < runtimeKey(all[j].Name, all[j].ID)
	})
	now := time.Now()
	var answered, notHeard []map[string]any
	onMachine := map[string]bool{}
	for _, r := range all {
		entry := map[string]any{"runtime": r.Name, "instance": r.ID}
		if r.Machine != "" {
			entry["machine"] = r.Machine
		}
		if !r.LastHeard.IsZero() {
			entry["last heard"] = r.LastHeard.Format(time.RFC3339)
			entry["last heard, ago"] = now.Sub(r.LastHeard).Round(time.Second).String()
			entry["took"] = r.Took.Round(time.Millisecond).String()
			entry["bytes"] = r.Bytes
			entry["modules"] = r.Modules
			entry["tools"] = r.Tools
			entry["seat verbs"] = r.SeatVerbs
			entry["shortened"] = r.Shortened != ""
			if r.Shortened != "" {
				entry["shortened, how"] = r.Shortened
			}
			if r.Reused && r.Answered {
				entry["announcement"] = "unchanged (its PING's digest), so not fetched again; took is its PING's"
			}
		}
		if r.Answered {
			onMachine[r.Machine] = true
			answered = append(answered, entry)
		} else {
			entry["why"] = r.Why
			if r.LastHeard.IsZero() {
				entry["last heard"] = "never in full, since this console started"
			}
			notHeard = append(notHeard, entry)
		}
	}
	for _, m := range x.Machines {
		if !onMachine[m] && !machineListed(notHeard, m) {
			notHeard = append(notHeard, map[string]any{"machine": m,
				"why": "the mesh knows this machine and no runtime on it has answered discovery since this console started"})
		}
	}
	if answered == nil {
		answered = []map[string]any{}
	}
	if notHeard == nil {
		notHeard = []map[string]any{}
	}
	return answerText(map[string]any{
		"asked":     x.Discovery.At.Format(time.RFC3339),
		"waited":    x.Discovery.Waited.Round(time.Millisecond).String(),
		"window":    fmt.Sprintf("at least %s; up to %s for a runtime that answered PING, or until one does", announce.Window, announce.Patience),
		"answered":  answered,
		"not heard": notHeard,
	})
}

func machineListed(entries []map[string]any, machine string) bool {
	for _, e := range entries {
		if e["machine"] == machine {
			return true
		}
	}
	return false
}