Keep what discovery heard when a round hears nothing, and fetch only changed announcements (issue 554)

A round in which no PING reached the laptop's console within the 750 ms window
was taken as the mesh's answer that nothing is there, and replaced the index:
every seat and module call was refused for five seconds at a time. Discovery
now waits up to its patience for a first PING, a round that heard nothing is
not kept, a runtime not heard this round is resolved from what it announced
last (said with its age), and a round that misses runtimes is logged once.
A PING carries a digest of what the instance serves, so an unchanged
announcement is not pulled again every round (some 880 kB to the laptop).

node-tools/internal/console/runtimes.go

@@ -7,7 +7,9 @@ package console
 
 import (
 	"fmt"
+	"log"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/novox/mesh-tools/node-tools/internal/announce"
@@ -31,6 +33,11 @@ type runtimeSeen struct {
 	Shortened string
 	Answered  bool   // in the latest round
 	Why       string // why not, when it did not
+	// Reused is true when its PING said its announcement was unchanged, so it was not fetched again.
+	Reused bool
+	// Last is its last whole answer: what a round that does not hear it uses instead, and what an
+	// unchanged digest stands for (novox/hq issue 554).
+	Last announce.Heard
 }
 
 func runtimeKey(name, id string) string { return name + "/" + id }
@@ -50,7 +57,108 @@ func seenOf(h announce.Heard, at time.Time) *runtimeSeen {
 		}
 	}
 	return &runtimeSeen{Name: h.Info.Name, ID: h.Info.ID, Machine: h.Machine(), LastHeard: at, Took: h.Took,
-		Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true}
+		Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true,
+		Reused: h.Reused, Last: h}
+}
+
+// held is the answer this console holds for an instance at this digest (announce.Known): an instance
+// whose announcement is unchanged is not asked for it again (novox/hq issue 554).
+func (s *Surface) held(name, id, digest string) (announce.Heard, bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	r := s.runtimes[runtimeKey(name, id)]
+	if r == nil || r.Last.Digest == "" || r.Last.Digest != digest || r.Last.Info.Name == "" {
+		return announce.Heard{}, false
+	}
+	h := r.Last
+	h.Took, h.Reused = 0, false
+	return h, true
+}
+
+// before is the last answer of every runtime this round did not hear, heard within RuntimeForgotten
+// and not since restarted under another instance (novox/hq issue 554).
+func (s *Surface) before(d announce.Discovery) []earlier {
+	heard := map[string]bool{}
+	running := map[string]bool{}
+	for _, h := range d.Heard {
+		heard[runtimeKey(h.Info.Name, h.Info.ID)] = true
+		if m := h.Machine(); m != "" {
+			running[h.Info.Name+"@"+m] = true
+		}
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var out []earlier
+	for k, r := range s.runtimes {
+		if heard[k] || r.Last.Info.Name == "" || r.LastHeard.IsZero() || time.Since(r.LastHeard) > RuntimeForgotten {
+			continue
+		}
+		if r.Machine != "" && running[r.Name+"@"+r.Machine] {
+			continue // restarted: the new instance answered
+		}
+		out = append(out, earlier{Heard: r.Last, At: r.LastHeard})
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return runtimeKey(out[i].Heard.Info.Name, out[i].Heard.Info.ID) < runtimeKey(out[j].Heard.Info.Name, out[j].Heard.Info.ID)
+	})
+	return out
+}
+
+// sayRound logs a round of discovery that did not hear every runtime — once, when what it missed
+// changes, and once when every runtime is heard again — so the next case leaves evidence in the
+// journal (novox/hq issue 554). Called with s.mu held, after remember.
+func (s *Surface) sayRound(x *index) {
+	d := x.Discovery
+	var missed []string
+	for _, u := range x.Unheard {
+		who := u.Runtime
+		if u.Machine != "" {
+			who += " on " + u.Machine
+		}
+		missed = append(missed, who)
+	}
+	sort.Strings(missed)
+	shape := strings.Join(missed, ", ")
+	if d.Empty() {
+		shape = "empty|" + shape
+	}
+	if shape == s.roundSaid {
+		return
+	}
+	s.roundSaid = shape
+	reused := 0
+	for _, h := range d.Heard {
+		if h.Reused {
+			reused++
+		}
+	}
+	logf := log.Printf
+	if s.conn != nil && s.conn.Logf != nil {
+		logf = s.conn.Logf
+	}
+	window := fmt.Sprintf("waited %s (window %s, patience %s)", d.Waited.Round(time.Millisecond), announce.Window, announce.Patience)
+	held := ""
+	if len(x.Earlier) > 0 {
+		held = fmt.Sprintf("; what %d of them announced earlier is used, with its age", len(x.Earlier))
+	}
+	switch {
+	case d.Empty():
+		logf("[mesh-tools] discovery heard nothing at all: no PING arrived; %s; not heard: %s%s; asking again at the next question",
+			window, orNone(missed), held)
+	case len(missed) > 0:
+		logf("[mesh-tools] discovery did not hear every runtime: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s; not heard: %s%s",
+			d.Pinged, len(d.Heard), reused, window, shape, held)
+	default:
+		logf("[mesh-tools] discovery hears every runtime again: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s",
+			d.Pinged, len(d.Heard), reused, window)
+	}
+}
+
+func orNone(xs []string) string {
+	if len(xs) == 0 {
+		return "none heard before either"
+	}
+	return strings.Join(xs, ", ")
 }
 
 // remember records a round of discovery and adds to the index every runtime heard lately that did not
@@ -101,14 +209,25 @@ func (s *Surface) remember(x *index) {
 			continue
 		}
 		r.Answered = false
+		used := ""
+		if at, ok := x.Earlier[k]; ok {
+			used = fmt.Sprintf("; what it announced %s ago is used", time.Since(at).Round(time.Second))
+		}
 		if why, ok := silent[k]; ok {
 			r.Why = why
+			if used != "" {
+				for i := range x.Unheard {
+					if x.Unheard[i].Runtime == r.Name && x.Unheard[i].Machine == r.Machine && x.Unheard[i].Why == why {
+						x.Unheard[i].Why += used
+					}
+				}
+			}
 			continue // already in the index's unheard
 		}
 		r.Why = "it did not answer this round of discovery"
 		if time.Since(r.LastHeard) <= RuntimeForgotten {
 			x.Unheard = append(x.Unheard, unheard{Runtime: r.Name, Machine: r.Machine,
-				Why: fmt.Sprintf("it answered discovery %s ago, not now", time.Since(r.LastHeard).Round(time.Second))})
+				Why: fmt.Sprintf("it answered discovery %s ago, not now%s", time.Since(r.LastHeard).Round(time.Second), used)})
 		}
 	}
 	for _, u := range x.Discovery.Silent {
@@ -157,6 +276,9 @@ func (s *Surface) runtimesAnswer() map[string]any {
 			if r.Shortened != "" {
 				entry["shortened, how"] = r.Shortened
 			}
+			if r.Reused && r.Answered {
+				entry["announcement"] = "unchanged (its PING's digest), so not fetched again; took is its PING's"
+			}
 		}
 		if r.Answered {
 			onMachine[r.Machine] = true
@@ -184,7 +306,7 @@ func (s *Surface) runtimesAnswer() map[string]any {
 	return answerText(map[string]any{
 		"asked":     x.Discovery.At.Format(time.RFC3339),
 		"waited":    x.Discovery.Waited.Round(time.Millisecond).String(),
-		"window":    fmt.Sprintf("at least %s; up to %s for a runtime that answered PING", announce.Window, announce.Patience),
+		"window":    fmt.Sprintf("at least %s; up to %s for a runtime that answered PING, or until one does", announce.Window, announce.Patience),
 		"answered":  answered,
 		"not heard": notHeard,
 	})