package console // What discovery heard from each runtime (2026-10-05): the console said a module ran nowhere when the // largest runtime's answer arrived after the window closed, and nothing showed that it had. Every // round of discovery is remembered per runtime, so a runtime heard before and not now is named, and // `mesh_runtimes` says who answered, how fast and how much. import ( "fmt" "log" "sort" "strings" "time" "github.com/novox/mesh-tools/node-tools/internal/announce" ) // RuntimeForgotten is how long a runtime heard once is expected again: within it, its silence is said // wherever an address it might answer is asked for; after it, it is listed only by mesh_runtimes. var RuntimeForgotten = 15 * time.Minute // runtimeSeen is one runtime instance as discovery last heard it. type runtimeSeen struct { Name string ID string Machine string LastHeard time.Time Took time.Duration Bytes int Modules int Tools int SeatVerbs int Shortened string Answered bool // in the latest round Why string // why not, when it did not // Reused is true when its PING said its announcement was unchanged, so it was not fetched again. Reused bool // Last is its last whole answer: what a round that does not hear it uses instead, and what an // unchanged digest stands for (novox/hq issue 554). Last announce.Heard } func runtimeKey(name, id string) string { return name + "/" + id } // seenOf is what one answer announced, counted. func seenOf(h announce.Heard, at time.Time) *runtimeSeen { modules := map[string]bool{} tools := map[string]bool{} verbs := map[string]bool{} for _, e := range announce.Endpoints(h.Info) { switch e.Kind { case announce.KindTool: modules[e.Module] = true tools[e.Module+"."+e.Tool] = true case announce.KindSeat: verbs[e.Seat+"."+e.Tool] = true } } return &runtimeSeen{Name: h.Info.Name, ID: h.Info.ID, Machine: h.Machine(), LastHeard: at, Took: h.Took, Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true, Reused: h.Reused, Last: h} } // held is the answer this console holds for an instance at this digest (announce.Known): an instance // whose announcement is unchanged is not asked for it again (novox/hq issue 554). func (s *Surface) held(name, id, digest string) (announce.Heard, bool) { s.mu.Lock() defer s.mu.Unlock() r := s.runtimes[runtimeKey(name, id)] if r == nil || r.Last.Digest == "" || r.Last.Digest != digest || r.Last.Info.Name == "" { return announce.Heard{}, false } h := r.Last h.Took, h.Reused = 0, false return h, true } // before is the last answer of every runtime this round did not hear, heard within RuntimeForgotten // and not since restarted under another instance (novox/hq issue 554). func (s *Surface) before(d announce.Discovery) []earlier { heard := map[string]bool{} running := map[string]bool{} for _, h := range d.Heard { heard[runtimeKey(h.Info.Name, h.Info.ID)] = true if m := h.Machine(); m != "" { running[h.Info.Name+"@"+m] = true } } s.mu.Lock() defer s.mu.Unlock() var out []earlier for k, r := range s.runtimes { if heard[k] || r.Last.Info.Name == "" || r.LastHeard.IsZero() || time.Since(r.LastHeard) > RuntimeForgotten { continue } if r.Machine != "" && running[r.Name+"@"+r.Machine] { continue // restarted: the new instance answered } out = append(out, earlier{Heard: r.Last, At: r.LastHeard}) } sort.Slice(out, func(i, j int) bool { return runtimeKey(out[i].Heard.Info.Name, out[i].Heard.Info.ID) < runtimeKey(out[j].Heard.Info.Name, out[j].Heard.Info.ID) }) return out } // sayRound logs a round of discovery that did not hear every runtime — once, when what it missed // changes, and once when every runtime is heard again — so the next case leaves evidence in the // journal (novox/hq issue 554). Called with s.mu held, after remember. func (s *Surface) sayRound(x *index) { d := x.Discovery var missed []string for _, u := range x.Unheard { who := u.Runtime if u.Machine != "" { who += " on " + u.Machine } missed = append(missed, who) } sort.Strings(missed) shape := strings.Join(missed, ", ") if d.Empty() { shape = "empty|" + shape } if shape == s.roundSaid { return } s.roundSaid = shape reused := 0 for _, h := range d.Heard { if h.Reused { reused++ } } logf := log.Printf if s.conn != nil && s.conn.Logf != nil { logf = s.conn.Logf } window := fmt.Sprintf("waited %s (window %s, patience %s)", d.Waited.Round(time.Millisecond), announce.Window, announce.Patience) held := "" if len(x.Earlier) > 0 { held = fmt.Sprintf("; what %d of them announced earlier is used, with its age", len(x.Earlier)) } switch { case d.Empty(): logf("[mesh-tools] discovery heard nothing at all: no PING arrived; %s; not heard: %s%s; asking again at the next question", window, orNone(missed), held) case len(missed) > 0: logf("[mesh-tools] discovery did not hear every runtime: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s; not heard: %s%s", d.Pinged, len(d.Heard), reused, window, shape, held) default: logf("[mesh-tools] discovery hears every runtime again: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s", d.Pinged, len(d.Heard), reused, window) } } func orNone(xs []string) string { if len(xs) == 0 { return "none heard before either" } return strings.Join(xs, ", ") } // remember records a round of discovery and adds to the index every runtime heard lately that did not // answer this time. Called with s.mu held. func (s *Surface) remember(x *index) { if s.runtimes == nil { s.runtimes = map[string]*runtimeSeen{} } at := x.Discovery.At if at.IsZero() { at = time.Now().UTC() } heard := map[string]bool{} for _, h := range x.Discovery.Heard { k := runtimeKey(h.Info.Name, h.Info.ID) seen := seenOf(h, at) if heard[k] { // Two instances under one name and id: both counted, the slower answer's time kept. prev := s.runtimes[k] seen.Modules += prev.Modules seen.Tools += prev.Tools seen.SeatVerbs += prev.SeatVerbs seen.Bytes += prev.Bytes if prev.Took > seen.Took { seen.Took = prev.Took } } heard[k] = true s.runtimes[k] = seen } // A runtime restarted answers under a new instance id: the old one is gone, not silent, and saying // it was missed would hide nothing but cry wolf after every restart. now := map[string]bool{} for k := range heard { now[s.runtimes[k].Name+"@"+s.runtimes[k].Machine] = true } for k, r := range s.runtimes { if !heard[k] && r.Machine != "" && now[r.Name+"@"+r.Machine] { delete(s.runtimes, k) } } silent := map[string]string{} for _, u := range x.Discovery.Silent { silent[runtimeKey(u.Name, u.ID)] = fmt.Sprintf("it answered PING; what it serves did not arrive within %s", announce.Patience) } for k, r := range s.runtimes { if heard[k] { continue } r.Answered = false used := "" if at, ok := x.Earlier[k]; ok { used = fmt.Sprintf("; what it announced %s ago is used", time.Since(at).Round(time.Second)) } if why, ok := silent[k]; ok { r.Why = why if used != "" { for i := range x.Unheard { if x.Unheard[i].Runtime == r.Name && x.Unheard[i].Machine == r.Machine && x.Unheard[i].Why == why { x.Unheard[i].Why += used } } } continue // already in the index's unheard } r.Why = "it did not answer this round of discovery" if time.Since(r.LastHeard) <= RuntimeForgotten { x.Unheard = append(x.Unheard, unheard{Runtime: r.Name, Machine: r.Machine, Why: fmt.Sprintf("it answered discovery %s ago, not now%s", time.Since(r.LastHeard).Round(time.Second), used)}) } } for _, u := range x.Discovery.Silent { k := runtimeKey(u.Name, u.ID) if _, known := s.runtimes[k]; !known { s.runtimes[k] = &runtimeSeen{Name: u.Name, ID: u.ID, Machine: u.Machine, Why: silent[k]} } } } // runtimesAnswer is mesh_runtimes: discovery asked now, and every runtime as it was heard. func (s *Surface) runtimesAnswer() map[string]any { x, err := s.indexAsked(true) if err != nil { return failure("the mesh's discovery failed: " + err.Error()) } s.mu.Lock() var all []runtimeSeen for _, r := range s.runtimes { all = append(all, *r) } s.mu.Unlock() sort.Slice(all, func(i, j int) bool { if all[i].Machine != all[j].Machine { return all[i].Machine < all[j].Machine } return runtimeKey(all[i].Name, all[i].ID) < runtimeKey(all[j].Name, all[j].ID) }) now := time.Now() var answered, notHeard []map[string]any onMachine := map[string]bool{} for _, r := range all { entry := map[string]any{"runtime": r.Name, "instance": r.ID} if r.Machine != "" { entry["machine"] = r.Machine } if !r.LastHeard.IsZero() { entry["last heard"] = r.LastHeard.Format(time.RFC3339) entry["last heard, ago"] = now.Sub(r.LastHeard).Round(time.Second).String() entry["took"] = r.Took.Round(time.Millisecond).String() entry["bytes"] = r.Bytes entry["modules"] = r.Modules entry["tools"] = r.Tools entry["seat verbs"] = r.SeatVerbs entry["shortened"] = r.Shortened != "" if r.Shortened != "" { entry["shortened, how"] = r.Shortened } if r.Reused && r.Answered { entry["announcement"] = "unchanged (its PING's digest), so not fetched again; took is its PING's" } } if r.Answered { onMachine[r.Machine] = true answered = append(answered, entry) } else { entry["why"] = r.Why if r.LastHeard.IsZero() { entry["last heard"] = "never in full, since this console started" } notHeard = append(notHeard, entry) } } for _, m := range x.Machines { if !onMachine[m] && !machineListed(notHeard, m) { notHeard = append(notHeard, map[string]any{"machine": m, "why": "the mesh knows this machine and no runtime on it has answered discovery since this console started"}) } } if answered == nil { answered = []map[string]any{} } if notHeard == nil { notHeard = []map[string]any{} } return answerText(map[string]any{ "asked": x.Discovery.At.Format(time.RFC3339), "waited": x.Discovery.Waited.Round(time.Millisecond).String(), "window": fmt.Sprintf("at least %s; up to %s for a runtime that answered PING, or until one does", announce.Window, announce.Patience), "answered": answered, "not heard": notHeard, }) } func machineListed(entries []map[string]any, machine string) bool { for _, e := range entries { if e["machine"] == machine { return true } } return false }