mesh-tools/node-tools/internal/console/runtimes.go

package console

// What discovery heard from each runtime (2026-10-05): the console said a module ran nowhere when the
// largest runtime's answer arrived after the window closed, and nothing showed that it had. Every
// round of discovery is remembered per runtime, so a runtime heard before and not now is named, and
// `mesh_runtimes` says who answered, how fast and how much.

import (
	"fmt"
	"sort"
	"time"

	"github.com/novox/mesh-tools/node-tools/internal/announce"
)

// RuntimeForgotten is how long a runtime heard once is expected again: within it, its silence is said
// wherever an address it might answer is asked for; after it, it is listed only by mesh_runtimes.
var RuntimeForgotten = 15 * time.Minute

// runtimeSeen is one runtime instance as discovery last heard it.
type runtimeSeen struct {
	Name      string
	ID        string
	Machine   string
	LastHeard time.Time
	Took      time.Duration
	Bytes     int
	Modules   int
	Tools     int
	SeatVerbs int
	Shortened string
	Answered  bool   // in the latest round
	Why       string // why not, when it did not
}

func runtimeKey(name, id string) string { return name + "/" + id }

// seenOf is what one answer announced, counted.
func seenOf(h announce.Heard, at time.Time) *runtimeSeen {
	modules := map[string]bool{}
	tools := map[string]bool{}
	verbs := map[string]bool{}
	for _, e := range announce.Endpoints(h.Info) {
		switch e.Kind {
		case announce.KindTool:
			modules[e.Module] = true
			tools[e.Module+"."+e.Tool] = true
		case announce.KindSeat:
			verbs[e.Seat+"."+e.Tool] = true
		}
	}
	return &runtimeSeen{Name: h.Info.Name, ID: h.Info.ID, Machine: h.Machine(), LastHeard: at, Took: h.Took,
		Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true}
}

// remember records a round of discovery and adds to the index every runtime heard lately that did not
// answer this time. Called with s.mu held.
func (s *Surface) remember(x *index) {
	if s.runtimes == nil {
		s.runtimes = map[string]*runtimeSeen{}
	}
	at := x.Discovery.At
	if at.IsZero() {
		at = time.Now().UTC()
	}
	heard := map[string]bool{}
	for _, h := range x.Discovery.Heard {
		k := runtimeKey(h.Info.Name, h.Info.ID)
		seen := seenOf(h, at)
		if heard[k] {
			// Two instances under one name and id: both counted, the slower answer's time kept.
			prev := s.runtimes[k]
			seen.Modules += prev.Modules
			seen.Tools += prev.Tools
			seen.SeatVerbs += prev.SeatVerbs
			seen.Bytes += prev.Bytes
			if prev.Took > seen.Took {
				seen.Took = prev.Took
			}
		}
		heard[k] = true
		s.runtimes[k] = seen
	}
	// A runtime restarted answers under a new instance id: the old one is gone, not silent, and saying
	// it was missed would hide nothing but cry wolf after every restart.
	now := map[string]bool{}
	for k := range heard {
		now[s.runtimes[k].Name+"@"+s.runtimes[k].Machine] = true
	}
	for k, r := range s.runtimes {
		if !heard[k] && r.Machine != "" && now[r.Name+"@"+r.Machine] {
			delete(s.runtimes, k)
		}
	}
	silent := map[string]string{}
	for _, u := range x.Discovery.Silent {
		silent[runtimeKey(u.Name, u.ID)] = fmt.Sprintf("it answered PING; what it serves did not arrive within %s", announce.Patience)
	}
	for k, r := range s.runtimes {
		if heard[k] {
			continue
		}
		r.Answered = false
		if why, ok := silent[k]; ok {
			r.Why = why
			continue // already in the index's unheard
		}
		r.Why = "it did not answer this round of discovery"
		if time.Since(r.LastHeard) <= RuntimeForgotten {
			x.Unheard = append(x.Unheard, unheard{Runtime: r.Name, Machine: r.Machine,
				Why: fmt.Sprintf("it answered discovery %s ago, not now", time.Since(r.LastHeard).Round(time.Second))})
		}
	}
	for _, u := range x.Discovery.Silent {
		k := runtimeKey(u.Name, u.ID)
		if _, known := s.runtimes[k]; !known {
			s.runtimes[k] = &runtimeSeen{Name: u.Name, ID: u.ID, Machine: u.Machine, Why: silent[k]}
		}
	}
}

// runtimesAnswer is mesh_runtimes: discovery asked now, and every runtime as it was heard.
func (s *Surface) runtimesAnswer() map[string]any {
	x, err := s.indexAsked(true)
	if err != nil {
		return failure("the mesh's discovery failed: " + err.Error())
	}
	s.mu.Lock()
	var all []runtimeSeen
	for _, r := range s.runtimes {
		all = append(all, *r)
	}
	s.mu.Unlock()
	sort.Slice(all, func(i, j int) bool {
		if all[i].Machine != all[j].Machine {
			return all[i].Machine < all[j].Machine
		}
		return runtimeKey(all[i].Name, all[i].ID) < runtimeKey(all[j].Name, all[j].ID)
	})
	now := time.Now()
	var answered, notHeard []map[string]any
	onMachine := map[string]bool{}
	for _, r := range all {
		entry := map[string]any{"runtime": r.Name, "instance": r.ID}
		if r.Machine != "" {
			entry["machine"] = r.Machine
		}
		if !r.LastHeard.IsZero() {
			entry["last heard"] = r.LastHeard.Format(time.RFC3339)
			entry["last heard, ago"] = now.Sub(r.LastHeard).Round(time.Second).String()
			entry["took"] = r.Took.Round(time.Millisecond).String()
			entry["bytes"] = r.Bytes
			entry["modules"] = r.Modules
			entry["tools"] = r.Tools
			entry["seat verbs"] = r.SeatVerbs
			entry["shortened"] = r.Shortened != ""
			if r.Shortened != "" {
				entry["shortened, how"] = r.Shortened
			}
		}
		if r.Answered {
			onMachine[r.Machine] = true
			answered = append(answered, entry)
		} else {
			entry["why"] = r.Why
			if r.LastHeard.IsZero() {
				entry["last heard"] = "never in full, since this console started"
			}
			notHeard = append(notHeard, entry)
		}
	}
	for _, m := range x.Machines {
		if !onMachine[m] && !machineListed(notHeard, m) {
			notHeard = append(notHeard, map[string]any{"machine": m,
				"why": "the mesh knows this machine and no runtime on it has answered discovery since this console started"})
		}
	}
	if answered == nil {
		answered = []map[string]any{}
	}
	if notHeard == nil {
		notHeard = []map[string]any{}
	}
	return answerText(map[string]any{
		"asked":     x.Discovery.At.Format(time.RFC3339),
		"waited":    x.Discovery.Waited.Round(time.Millisecond).String(),
		"window":    fmt.Sprintf("at least %s; up to %s for a runtime that answered PING", announce.Window, announce.Patience),
		"answered":  answered,
		"not heard": notHeard,
	})
}

func machineListed(entries []map[string]any, machine string) bool {
	for _, e := range entries {
		if e["machine"] == machine {
			return true
		}
	}
	return false
}