Keep what discovery heard when a round hears nothing, and fetch only changed announcements (issue 554)

A round in which no PING reached the laptop's console within the 750 ms window
was taken as the mesh's answer that nothing is there, and replaced the index:
every seat and module call was refused for five seconds at a time. Discovery
now waits up to its patience for a first PING, a round that heard nothing is
not kept, a runtime not heard this round is resolved from what it announced
last (said with its age), and a round that misses runtimes is logged once.
A PING carries a digest of what the instance serves, so an unchanged
announcement is not pulled again every round (some 880 kB to the laptop).

node-tools/internal/announce/empty_round_test.go

@@ -0,0 +1,126 @@
+package announce
+
+import (
+	"encoding/json"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go/micro"
+)
+
+// novox/hq issue 554: the laptop's console took a round in which not even a PING arrived within the
+// 750 ms window as the mesh's answer that nothing is there, and refused every seat and module call. A
+// PING that arrives after the window is waited for, up to the patience.
+func TestARoundWaitsForAPingThatArrivesAfterTheWindow(t *testing.T) {
+	was := Patience
+	Patience = 2500 * time.Millisecond
+	t.Cleanup(func() { Patience = was })
+
+	late := connect(t, "node-tools", "laptop")
+	s := Service{Name: "node-tools", ID: "laptop", Metadata: map[string]string{"node": "laptop"}}
+	ping, _ := json.Marshal(micro.Ping{ServiceIdentity: s.identity(), Type: micro.PingResponseType})
+	stop, err := late.Raw("$SRV.PING", func(string, []byte) []byte { time.Sleep(Window + 300*time.Millisecond); return ping })
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(stop)
+	body, _ := json.Marshal(Info(s, []Endpoint{{Kind: KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Subject: "mesh.mod.slack.tool.slack_check.laptop"}}))
+	for _, subject := range []string{"$SRV.INFO", "$SRV.INFO.node-tools.laptop"} {
+		stop, err := late.Raw(subject, func(string, []byte) []byte { return body })
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(stop)
+	}
+	late.Flush()
+
+	d, err := Gather(connect(t, "console", "laptop"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if d.Empty() || len(d.Heard) != 1 || d.Heard[0].Machine() != "laptop" {
+		t.Fatalf("a PING after the window was not waited for: waited %s, pinged %d, heard %+v", d.Waited, d.Pinged, d.Heard)
+	}
+	if d.Waited < Window || d.Waited >= Patience {
+		t.Errorf("waited %s; the PING came after %s and patience is %s", d.Waited, Window+300*time.Millisecond, Patience)
+	}
+}
+
+// A round in which nothing at all arrives is said to be empty, after the patience — not answered as an
+// empty mesh at the window.
+func TestARoundThatHearsNothingIsEmptyAfterThePatience(t *testing.T) {
+	was := Patience
+	Patience = 1500 * time.Millisecond
+	t.Cleanup(func() { Patience = was })
+	d, err := Gather(connect(t, "console", "laptop"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !d.Empty() || d.Waited < Patience {
+		t.Errorf("empty %v after %s (patience %s)", d.Empty(), d.Waited, Patience)
+	}
+}
+
+// What a runtime announces is fetched again only when its PING's digest changed: every round pulled
+// some 880 kB of announcements to the laptop, and the PING answers queued behind them (issue 554).
+func TestAnUnchangedAnnouncementIsNotFetchedAgain(t *testing.T) {
+	runtime := connect(t, "node-tools", "desk")
+	var endpoints atomic.Value
+	endpoints.Store([]Endpoint{{Kind: KindTool, Module: "alpha", Tool: "one", Node: "desk", Subject: "mesh.mod.alpha.tool.one.desk"}})
+	stop, err := Serve(runtime, Service{Name: "node-tools", ID: "desk", Metadata: map[string]string{"node": "desk"}},
+		func() []Endpoint { return endpoints.Load().([]Endpoint) })
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(stop)
+	runtime.Flush()
+	// Every request for what desk serves, counted where it is answered.
+	var asked atomic.Int32
+	spy := connect(t, "node-tools", "spy")
+	for _, subject := range []string{"$SRV.INFO", "$SRV.INFO.node-tools", "$SRV.INFO.node-tools.desk"} {
+		stop, err := spy.Raw(subject, func(string, []byte) []byte { asked.Add(1); return nil })
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(stop)
+	}
+	spy.Flush()
+
+	console := connect(t, "console", "laptop")
+	first, err := Gather(console, nil)
+	if err != nil || len(first.Heard) != 1 || first.Heard[0].Digest == "" || first.Heard[0].Reused {
+		t.Fatalf("first round: %v %+v", err, first.Heard)
+	}
+	if asked.Load() != 1 {
+		t.Errorf("the first round asked what desk serves %d time(s)", asked.Load())
+	}
+	held := first.Heard[0]
+	known := func(name, id, digest string) (Heard, bool) {
+		if name == held.Info.Name && id == held.Info.ID && digest == held.Digest {
+			return held, true
+		}
+		return Heard{}, false
+	}
+	asked.Store(0)
+	second, err := Gather(console, known)
+	if err != nil || len(second.Heard) != 1 || !second.Heard[0].Reused || len(Endpoints(second.Heard[0].Info)) != 1 {
+		t.Fatalf("an unchanged announcement: %v %+v", err, second.Heard)
+	}
+	if asked.Load() != 0 {
+		t.Errorf("an unchanged announcement was asked for %d time(s)", asked.Load())
+	}
+
+	endpoints.Store([]Endpoint{
+		{Kind: KindTool, Module: "alpha", Tool: "one", Node: "desk", Subject: "mesh.mod.alpha.tool.one.desk"},
+		{Kind: KindTool, Module: "alpha", Tool: "two", Node: "desk", Subject: "mesh.mod.alpha.tool.two.desk"},
+	})
+	third, err := Gather(console, known)
+	if err != nil || len(third.Heard) != 1 || third.Heard[0].Reused || len(Endpoints(third.Heard[0].Info)) != 2 ||
+		third.Heard[0].Digest == held.Digest {
+		t.Fatalf("a changed announcement was not fetched: %v %+v", err, third.Heard)
+	}
+	if asked.Load() != 1 {
+		t.Errorf("a changed announcement was asked for %d time(s)", asked.Load())
+	}
+}