mesh-tools/node-tools/internal/bus/state_watch_test.go

//go:build integration

package bus_test

import (
	"errors"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/novox/mesh-tools/node-tools/internal/bus"
	mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
)

// **A watch whose current values take longer than the bound to hand over still holds** (novox/hq issue
// 1726). The gitea module watches the checker's owed: one key per pull request head, each a round trip
// to the bundle. With a few hundred keys the hand-over outlasted StateTimeout at every attempt, the
// watch was refused, and the newest keys, handed over last, never reached mesh/check: a group's passing
// verdict was never written on its members' heads. The bound is on progress: a watch fails only when
// no current value is handed over within it.
func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) {
	was := bus.StateTimeout
	bus.StateTimeout = 400 * time.Millisecond
	t.Cleanup(func() { bus.StateTimeout = was })

	conn, kv := owedWatcher(t)
	// A group of three pull requests judged last, after many older heads: their keys come last.
	const older = 30
	var keys []string
	for i := 0; i < older; i++ {
		keys = append(keys, fmt.Sprintf("novox/mesh-catalog.%d.%040x.status=mesh/check", 400+i, i))
	}
	for _, member := range []string{"novox/mesh-sdk.76.34f210e5", "novox/mesh-controller.453.c06f75d0", "novox/mesh-delivery.33.28ee22ef"} {
		keys = append(keys, member+strings.Repeat("0", 32)+".status=mesh/check")
	}
	for _, k := range keys {
		if _, err := kv.Put(k, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"success"}`)); err != nil {
			t.Fatal(err)
		}
	}

	var mu sync.Mutex
	var current, live []string
	// Each value takes 40 ms to be taken: 33 values take about 1.3 s, three times the bound.
	stop, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(ch bus.StateChange) error {
		if ch.Current {
			time.Sleep(40 * time.Millisecond)
		}
		mu.Lock()
		defer mu.Unlock()
		if ch.Current {
			current = append(current, ch.Key)
		} else {
			live = append(live, ch.Key)
		}
		return nil
	})
	if err != nil {
		t.Fatalf("a watch whose current values kept arriving was refused: %v", err)
	}
	t.Cleanup(stop)
	mu.Lock()
	got := append([]string(nil), current...)
	mu.Unlock()
	if len(got) != len(keys) {
		t.Fatalf("the watch returned after %d of %d current values", len(got), len(keys))
	}
	for i, k := range keys {
		if got[i] != k {
			t.Fatalf("current value %d is %s, want %s", i, got[i], k)
		}
	}

	// And it holds: a verdict written afterwards reaches it as a change.
	next := "novox/mesh-sdk.77." + strings.Repeat("a", 40) + ".status=mesh/check"
	if _, err := kv.Put(next, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"pending"}`)); err != nil {
		t.Fatal(err)
	}
	mt.Until(t, func() error {
		mu.Lock()
		defer mu.Unlock()
		if len(live) != 1 || live[0] != next {
			return fmt.Errorf("live changes so far: %v", live)
		}
		return nil
	})
}

// A watch whose bundle takes nothing for longer than the bound is still refused, so a bundle that hangs
// is said, not waited on for ever.
func TestAWatchIsRefusedWhenNoCurrentValueIsTakenWithinTheBound(t *testing.T) {
	was := bus.StateTimeout
	bus.StateTimeout = 300 * time.Millisecond
	t.Cleanup(func() { bus.StateTimeout = was })

	conn, kv := owedWatcher(t)
	for i := 0; i < 3; i++ {
		if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
			t.Fatal(err)
		}
	}
	release := make(chan struct{})
	t.Cleanup(func() { close(release) })
	began := time.Now()
	_, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(bus.StateChange) error {
		<-release
		return nil
	})
	if err == nil || !strings.Contains(err.Error(), "did not arrive in") {
		t.Fatalf("a watch whose first value was never taken: want refused, got %v", err)
	}
	if took := time.Since(began); took > 3*time.Second {
		t.Fatalf("the refusal took %s, far past the bound", took)
	}
}

// **A membership the grants part wrote in its old wrapper is served whole** (issue 774). Until 2026-10-30 the part wrote
// its pass record, the membership inside, into its state memberships; the tool runner read the value as a bare
// membership, took it over the assignments stream's copy, and served no tool and no state. This is what the live mesh
// held: read through mesh-sdk's codec, its tools are answered where the membership says and its state is reached.
func TestAMembershipTheGrantsPartWroteWrappedIsServed(t *testing.T) {
	mesh := mt.New(t)
	mesh.Bucket(t, "gitea_repos")
	gitea := mt.MembershipOf("gitea", "anchor", false, nil)
	gitea.State = []bus.StateIssued{{Name: "repos", Bucket: "gitea_repos", Writes: true}}
	mesh.IssueInGrantsWrapped(t, gitea, "published")
	// The old wrapper of a membership the part held back carries its unpublished composition: refused, so the
	// assignments stream's copy is served, never what the hold kept back.
	pages := mt.MembershipOf("pages", "anchor", false, nil)
	mesh.Issue(t, pages)
	heldBack := mt.MembershipOf("pages", "anchor", true, nil)
	mesh.IssueInGrantsWrapped(t, heldBack, "not published, a shadow difference: publishing would lose the last holder of x")
	conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
	if err != nil {
		t.Fatal(err)
	}
	conn.Logf = func(string, ...any) {}
	t.Cleanup(conn.Close)
	conn.Follow("gitea")
	mt.Until(t, func() error {
		if m := conn.Membership("gitea"); m == nil || len(m.Serves) == 0 {
			return fmt.Errorf("gitea's membership is %+v, want the one inside the wrapper", m)
		}
		return nil
	})
	if got, want := fmt.Sprint(conn.ServedOn("gitea", "gitea_list")), "[{mesh.mod.gitea.tool.gitea_list.anchor }]"; got != want {
		t.Errorf("gitea_list is served on %s, want %s", got, want)
	}
	if _, err := conn.StatePut("gitea", "repos", "novox.hq", []byte(`{"v":1}`)); err != nil {
		t.Fatalf("the state the membership issues is refused: %v", err)
	}
	if e, err := conn.StateGet("gitea", "repos", "novox.hq"); err != nil || e == nil || string(e.Value) != `{"v":1}` {
		t.Errorf("the state reads back %+v, %v", e, err)
	}
	conn.Follow("pages")
	mt.Until(t, func() error {
		if m := conn.Membership("pages"); m == nil || len(m.Serves) == 0 {
			return fmt.Errorf("pages' membership is %+v, want the assignments stream's", m)
		}
		return nil
	})
	if got, want := fmt.Sprint(conn.ServedOn("pages", "pages_list")), "[{mesh.mod.pages.tool.pages_list.anchor }]"; got != want {
		t.Errorf("a held-back membership's wrapper is served: pages_list on %s, want the stream's %s", got, want)
	}
}

// owedWatcher is the gitea module's view of the checker's owed: the bucket, and a runtime connection
// that serves gitea with the owed issued for reading.
func owedWatcher(t *testing.T) (*bus.Conn, interface {
	Put(string, []byte) (uint64, error)
}) {
	t.Helper()
	mesh := mt.New(t)
	kv := mesh.Bucket(t, "mesh-delivery-checker_owed")
	gitea := mt.MembershipOf("gitea", "anchor", false, nil)
	gitea.State = []bus.StateIssued{{Name: "mesh-delivery-checker.owed", Bucket: "mesh-delivery-checker_owed"}}
	mesh.Issue(t, gitea)
	conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
	if err != nil {
		t.Fatal(err)
	}
	conn.Logf = func(string, ...any) {}
	t.Cleanup(conn.Close)
	conn.Follow("gitea")
	mt.Until(t, func() error {
		if conn.Membership("gitea") == nil {
			return errors.New("gitea not issued yet")
		}
		return nil
	})
	return conn, kv
}

// **A watch given up, or stopped, hands over nothing more** (issue 774). The runtime's watch stopped its watcher
// when the current values did not arrive within the bound, but the watcher still held values, and its goroutine
// went on handing them over: to a watch id the bundle no longer held (on the provisioner, 51 failed watches were
// followed by 1257 orphan changes). Each row blocks the bundle on its first value, gives the watch up, lets the
// bundle go and counts what it is still handed.
func TestAWatchGivenUpHandsOverNothingMore(t *testing.T) {
	for _, tc := range []struct {
		name    string
		refused bool // the bound gives it up, or the caller stops it
	}{
		{"refused at the bound", true},
		{"stopped by its caller", false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			was := bus.StateTimeout
			bus.StateTimeout = 300 * time.Millisecond
			t.Cleanup(func() { bus.StateTimeout = was })
			conn, kv := owedWatcher(t)
			put := func(i int) {
				if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
					t.Fatal(err)
				}
			}
			if tc.refused {
				for i := 0; i < 4; i++ {
					put(i)
				}
			}
			var calls atomic.Int32
			first := make(chan struct{}, 1)
			release := make(chan struct{})
			deliver := func(ch bus.StateChange) error {
				if calls.Add(1) == 1 {
					first <- struct{}{}
					<-release
				}
				return nil
			}
			var stop func()
			if tc.refused {
				if _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err == nil {
					t.Fatal("the watch whose bundle took nothing was not refused")
				}
			} else {
				var err error
				if stop, err = conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err != nil {
					t.Fatal(err)
				}
				for i := 0; i < 4; i++ {
					put(i)
				}
				select {
				case <-first:
				case <-time.After(5 * time.Second):
					t.Fatal("the bundle was handed no change")
				}
				stop()
			}
			// Let the watcher's server side deliver what it held, then free the bundle.
			time.Sleep(300 * time.Millisecond)
			close(release)
			time.Sleep(500 * time.Millisecond)
			if n := calls.Load(); n != 1 {
				t.Fatalf("the bundle was handed %d changes of a watch given up, want 1 (the one in hand)", n)
			}
		})
	}
}