Keep what discovery heard when a round hears nothing, and fetch only changed announcements (issue 554)

A round in which no PING reached the laptop's console within the 750 ms window
was taken as the mesh's answer that nothing is there, and replaced the index:
every seat and module call was refused for five seconds at a time. Discovery
now waits up to its patience for a first PING, a round that heard nothing is
not kept, a runtime not heard this round is resolved from what it announced
last (said with its age), and a round that misses runtimes is logged once.
A PING carries a digest of what the instance serves, so an unchanged
announcement is not pulled again every round (some 880 kB to the laptop).

node-tools/internal/console/address.go

@@ -134,6 +134,42 @@ type index struct {
 	// Replaces is what each verb and tool replaces, from the controller's records (novox/hq ADR 0245):
 	// `seat:<seat>.<verb>` and `<module>.<tool>`.
 	Replaces map[string][]string
+	// Earlier is every runtime not heard this round whose last answer, within RuntimeForgotten, this
+	// index holds instead (novox/hq issue 554), by runtimeKey: when it was heard.
+	Earlier map[string]time.Time
+	// Stale is every module on a machine (`mod:<module>@<node>`) and every seat's holder
+	// (`seat:<seat>@<node>`) this index knows only from an earlier answer: from whom, and when.
+	Stale map[string]staleFrom
+}
+
+// staleFrom is where an address known only from an earlier round of discovery was heard.
+type staleFrom struct {
+	Runtime string
+	Machine string
+	At      time.Time
+}
+
+// say is the sentence a call or a description carries for it.
+func (f staleFrom) say() string {
+	who := f.Runtime
+	if f.Machine != "" {
+		who += " on " + f.Machine
+	}
+	return fmt.Sprintf("found from what %s announced %s ago: it did not answer the latest round of discovery",
+		who, time.Since(f.At).Round(time.Second))
+}
+
+// earlier is a runtime's last answer, used for a round that did not hear it.
+type earlier struct {
+	Heard announce.Heard
+	At    time.Time
+}
+
+func endpointKey(e announce.Endpoint) string {
+	if e.Kind == announce.KindSeat {
+		return "seat:" + e.Seat + "@" + e.Node
+	}
+	return "mod:" + e.Module + "@" + e.Node
 }
 
 // unheard is one runtime discovery did not hear in full, and why.
@@ -237,7 +273,11 @@ type recordedMachine struct {
 // indexOn asks the mesh what it holds (novox/hq ADR 0197): what answers, from every runtime's own
 // announcement on the bus — one `$SRV.INFO` request — and what should, from the controller's records
 // read as JSON. Nothing is inferred from a roster and nothing is parsed from print.
-func indexOn(conn *bus.Conn) (*index, error) {
+//
+// `held` is what the console holds already, so an instance whose announcement is unchanged is not
+// asked for it again; `before` is the last answer of every runtime this round did not hear, which the
+// index holds instead, marked with its age (novox/hq issue 554).
+func indexOn(conn *bus.Conn, held announce.Known, before func(announce.Discovery) []earlier) (*index, error) {
 	var wg sync.WaitGroup
 	var nodesOut, modulesOut string
 	var toolsAnswer json.RawMessage
@@ -252,19 +292,44 @@ func indexOn(conn *bus.Conn) (*index, error) {
 			toolsAnswer = got.Result
 		}
 	}()
-	d, err := announce.Gather(conn)
+	d, err := announce.Gather(conn, held)
 	wg.Wait()
 	if err != nil {
 		return nil, err
 	}
 
 	var endpoints []announce.Endpoint
+	fresh := map[string]bool{}
 	for _, h := range d.Heard {
-		endpoints = append(endpoints, announce.Endpoints(h.Info)...)
+		for _, e := range announce.Endpoints(h.Info) {
+			fresh[endpointKey(e)] = true
+			endpoints = append(endpoints, e)
+		}
+	}
+	// **A runtime not heard this round is not gone** (novox/hq issue 554): a round in which no answer
+	// crossed the laptop's link in time replaced a full index with an empty one, and every seat and
+	// module call was refused. What it announced last, within RuntimeForgotten, stands — said with its age.
+	stale := map[string]staleFrom{}
+	earlierAt := map[string]time.Time{}
+	if before != nil {
+		for _, b := range before(d) {
+			earlierAt[runtimeKey(b.Heard.Info.Name, b.Heard.Info.ID)] = b.At
+			for _, e := range announce.Endpoints(b.Heard.Info) {
+				k := endpointKey(e)
+				if fresh[k] {
+					continue
+				}
+				if _, have := stale[k]; !have {
+					stale[k] = staleFrom{Runtime: b.Heard.Info.Name, Machine: b.Heard.Machine(), At: b.At}
+				}
+				endpoints = append(endpoints, e)
+			}
+		}
 	}
 	x, announced, machines := indexOfEndpoints(endpoints)
 	l := x.Listing
 	x.Discovery = d
+	x.Stale, x.Earlier = stale, earlierAt
 	for _, s := range d.Silent {
 		x.Unheard = append(x.Unheard, unheard{Runtime: s.Name, Machine: s.Machine,
 			Why: fmt.Sprintf("it answered PING; what it serves did not arrive within %s", announce.Patience)})
@@ -325,6 +390,10 @@ func containsHolder(hs []holder, h holder) bool {
 func (s *Surface) index() (*index, error) { return s.indexAsked(false) }
 
 // indexAsked is what the mesh answered, asked again when `fresh` or when the kept answer is old.
+//
+// **A round that heard nothing is not kept** (novox/hq issue 554): the index it gives holds what every
+// runtime announced last, marked with its age, and the next question asks again rather than living
+// with it for IndexKept.
 func (s *Surface) indexAsked(fresh bool) (*index, error) {
 	s.mu.Lock()
 	if !fresh && s.idx != nil && time.Since(s.idxAt) <= IndexKept {
@@ -333,13 +402,17 @@ func (s *Surface) indexAsked(fresh bool) (*index, error) {
 		return x, nil
 	}
 	s.mu.Unlock()
-	x, err := indexOn(s.conn)
+	x, err := indexOn(s.conn, s.held, s.before)
 	if err != nil {
 		return nil, err
 	}
 	s.mu.Lock()
 	s.remember(x)
-	s.idx, s.idxAt = x, time.Now()
+	s.sayRound(x)
+	s.idx = x
+	if !x.Discovery.Empty() {
+		s.idxAt = time.Now()
+	}
 	s.mu.Unlock()
 	return x, nil
 }
@@ -450,6 +523,26 @@ type target struct {
 	Node    string
 	Tool    Tool
 	Seat    bool
+	// Stale is set when the target is known only from an earlier round of discovery (novox/hq issue 554).
+	Stale *staleFrom
+}
+
+// staleOf is where a target was heard, when every place it answers is known only from an earlier round.
+func (x *index) staleOf(keys []string) *staleFrom {
+	if len(keys) == 0 || len(x.Stale) == 0 {
+		return nil
+	}
+	var first *staleFrom
+	for _, k := range keys {
+		f, ok := x.Stale[k]
+		if !ok {
+			return nil
+		}
+		if first == nil {
+			first = &f
+		}
+	}
+	return first
 }
 
 // resolve turns an address into exactly one target, or says why it cannot.
@@ -487,12 +580,17 @@ func resolve(x *index, address string) (target, error) {
 				return target{}, fmt.Errorf("%s is held once per machine: write <node>/%s — it is held on %s",
 					prefix, rest, orNobody(nodesOf(s.Holders)))
 			}
-			return target{Address: node + "/" + rest, Key: "seat:" + rest + "@" + node, Name: rest, Node: node, Tool: *verb, Seat: true}, nil
+			return target{Address: node + "/" + rest, Key: "seat:" + rest + "@" + node, Name: rest, Node: node, Tool: *verb, Seat: true,
+				Stale: x.staleOf([]string{"seat:" + prefix + "@" + node})}, nil
 		}
 		if node != "" {
 			return target{}, fmt.Errorf("%s is held once for the whole mesh: write %s, without a machine", prefix, rest)
 		}
-		return target{Address: rest, Key: "seat:" + rest, Name: rest, Tool: *verb, Seat: true}, nil
+		var holders []string
+		for _, h := range s.Holders {
+			holders = append(holders, "seat:"+prefix+"@"+h.Node)
+		}
+		return target{Address: rest, Key: "seat:" + rest, Name: rest, Tool: *verb, Seat: true, Stale: x.staleOf(holders)}, nil
 	}
 
 	m := x.Modules[prefix]
@@ -509,6 +607,11 @@ func resolve(x *index, address string) (target, error) {
 			return target{}, fmt.Errorf("nothing that answered discovery is called %s, but not every runtime answered: %s — "+
 				"%s may be theirs. Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, sayUnheard(x.Unheard), prefix)
 		}
+		if x.Discovery.Empty() {
+			return target{}, fmt.Errorf("nothing answered discovery at all within %s, not even PING, and this console holds no "+
+				"earlier answer: whether anything is called %s is not known. Ask again in a moment; mesh_runtimes shows which "+
+				"runtimes answered", x.Discovery.Waited.Round(time.Millisecond), prefix)
+		}
 		return target{}, fmt.Errorf("nothing in the mesh is called %s: no seat, and no module that answers tools. "+
 			"mesh_search finds a tool by words", prefix)
 	}
@@ -521,7 +624,11 @@ func resolve(x *index, address string) (target, error) {
 			return target{}, fmt.Errorf("%s keeps state on each machine it runs on, so a call names the machine: "+
 				"write <node>/%s — it runs on %s", prefix, rest, orNobody(m.On))
 		}
-		return target{Address: rest, Key: rest, Name: rest, Tool: *tool}, nil
+		var on []string
+		for _, n := range m.On {
+			on = append(on, "mod:"+prefix+"@"+n)
+		}
+		return target{Address: rest, Key: rest, Name: rest, Tool: *tool, Stale: x.staleOf(on)}, nil
 	}
 	if len(m.On) > 0 && !contains(m.On, node) {
 		if us := x.unheardOn(node); len(us) > 0 || contains(x.Recorded[prefix], node) {
@@ -534,7 +641,8 @@ func resolve(x *index, address string) (target, error) {
 		}
 		return target{}, fmt.Errorf("%s does not run on %s; it runs on %s", prefix, node, orNobody(m.On))
 	}
-	return target{Address: node + "/" + rest, Key: rest + "@" + node, Name: rest, Node: node, Tool: *tool}, nil
+	return target{Address: node + "/" + rest, Key: rest + "@" + node, Name: rest, Node: node, Tool: *tool,
+		Stale: x.staleOf([]string{"mod:" + prefix + "@" + node})}, nil
 }
 
 // unheardOnAny is what is unheard on any of these machines.
@@ -861,6 +969,9 @@ func (s *Surface) discover(name string, args map[string]any) map[string]any {
 		if r := x.Replaces[key]; len(r) > 0 {
 			described["instead_of"] = r
 		}
+		if t.Stale != nil {
+			described["found"] = t.Stale.say()
+		}
 		return answerText(described)
 
 	case verbCall:
@@ -880,12 +991,19 @@ func (s *Surface) discover(name string, args map[string]any) map[string]any {
 			got, err = callTool(s.conn, t.Key, callArgs, nil, x.Listing)
 		}
 		if err != nil {
-			return failure(whyItFailed(t.Address, err))
+			why := whyItFailed(t.Address, err)
+			if t.Stale != nil {
+				why += " (" + t.Stale.say() + ")"
+			}
+			return failure(why)
 		}
 		content := []map[string]any{{"type": "text", "text": pretty(got.Result)}}
 		if got.Node != "" {
 			content = append(content, map[string]any{"type": "text", "text": "answered by " + got.Node})
 		}
+		if t.Stale != nil {
+			content = append(content, map[string]any{"type": "text", "text": t.Stale.say()})
+		}
 		return map[string]any{"content": content}
 	}
 	return failure("no discovery verb " + name)