A watch given up or stopped hands over nothing more; arrival of current values restarts its bound (issue 774)

Quick: check the watch context before each deliver in StateWatch. Thorough (chosen, same
file kind): also LogWatch drops a message fetched as it stopped, and the bound restarts
on arrival, not only after the bundle took a value.

node-tools/internal/bus/state_watch_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -192,3 +193,72 @@ func owedWatcher(t *testing.T) (*bus.Conn, interface {
 	})
 	return conn, kv
 }
+
+// **A watch given up, or stopped, hands over nothing more** (issue 774). The runtime's watch stopped its watcher
+// when the current values did not arrive within the bound, but the watcher still held values, and its goroutine
+// went on handing them over: to a watch id the bundle no longer held (on the provisioner, 51 failed watches were
+// followed by 1257 orphan changes). Each row blocks the bundle on its first value, gives the watch up, lets the
+// bundle go and counts what it is still handed.
+func TestAWatchGivenUpHandsOverNothingMore(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		refused bool // the bound gives it up, or the caller stops it
+	}{
+		{"refused at the bound", true},
+		{"stopped by its caller", false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			was := bus.StateTimeout
+			bus.StateTimeout = 300 * time.Millisecond
+			t.Cleanup(func() { bus.StateTimeout = was })
+			conn, kv := owedWatcher(t)
+			put := func(i int) {
+				if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if tc.refused {
+				for i := 0; i < 4; i++ {
+					put(i)
+				}
+			}
+			var calls atomic.Int32
+			first := make(chan struct{}, 1)
+			release := make(chan struct{})
+			deliver := func(ch bus.StateChange) error {
+				if calls.Add(1) == 1 {
+					first <- struct{}{}
+					<-release
+				}
+				return nil
+			}
+			var stop func()
+			if tc.refused {
+				if _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err == nil {
+					t.Fatal("the watch whose bundle took nothing was not refused")
+				}
+			} else {
+				var err error
+				if stop, err = conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err != nil {
+					t.Fatal(err)
+				}
+				for i := 0; i < 4; i++ {
+					put(i)
+				}
+				select {
+				case <-first:
+				case <-time.After(5 * time.Second):
+					t.Fatal("the bundle was handed no change")
+				}
+				stop()
+			}
+			// Let the watcher's server side deliver what it held, then free the bundle.
+			time.Sleep(300 * time.Millisecond)
+			close(release)
+			time.Sleep(500 * time.Millisecond)
+			if n := calls.Load(); n != 1 {
+				t.Fatalf("the bundle was handed %d changes of a watch given up, want 1 (the one in hand)", n)
+			}
+		})
+	}
+}