jschoubben
316a5a2fc3
Quick: check the watch context before each deliver in StateWatch. Thorough (chosen, same file kind): also LogWatch drops a message fetched as it stopped, and the bound restarts on arrival, not only after the bundle took a value.
265 lines
9.6 KiB
Go
265 lines
9.6 KiB
Go
//go:build integration
|
|
|
|
package bus_test
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
|
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
|
)
|
|
|
|
// **A watch whose current values take longer than the bound to hand over still holds** (novox/hq issue
|
|
// 1726). The gitea module watches the checker's owed: one key per pull request head, each a round trip
|
|
// to the bundle. With a few hundred keys the hand-over outlasted StateTimeout at every attempt, the
|
|
// watch was refused, and the newest keys, handed over last, never reached mesh/check: a group's passing
|
|
// verdict was never written on its members' heads. The bound is on progress: a watch fails only when
|
|
// no current value is handed over within it.
|
|
func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) {
|
|
was := bus.StateTimeout
|
|
bus.StateTimeout = 400 * time.Millisecond
|
|
t.Cleanup(func() { bus.StateTimeout = was })
|
|
|
|
conn, kv := owedWatcher(t)
|
|
// A group of three pull requests judged last, after many older heads: their keys come last.
|
|
const older = 30
|
|
var keys []string
|
|
for i := 0; i < older; i++ {
|
|
keys = append(keys, fmt.Sprintf("novox/mesh-catalog.%d.%040x.status=mesh/check", 400+i, i))
|
|
}
|
|
for _, member := range []string{"novox/mesh-sdk.76.34f210e5", "novox/mesh-controller.453.c06f75d0", "novox/mesh-delivery.33.28ee22ef"} {
|
|
keys = append(keys, member+strings.Repeat("0", 32)+".status=mesh/check")
|
|
}
|
|
for _, k := range keys {
|
|
if _, err := kv.Put(k, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"success"}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
var mu sync.Mutex
|
|
var current, live []string
|
|
// Each value takes 40 ms to be taken: 33 values take about 1.3 s, three times the bound.
|
|
stop, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(ch bus.StateChange) error {
|
|
if ch.Current {
|
|
time.Sleep(40 * time.Millisecond)
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if ch.Current {
|
|
current = append(current, ch.Key)
|
|
} else {
|
|
live = append(live, ch.Key)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("a watch whose current values kept arriving was refused: %v", err)
|
|
}
|
|
t.Cleanup(stop)
|
|
mu.Lock()
|
|
got := append([]string(nil), current...)
|
|
mu.Unlock()
|
|
if len(got) != len(keys) {
|
|
t.Fatalf("the watch returned after %d of %d current values", len(got), len(keys))
|
|
}
|
|
for i, k := range keys {
|
|
if got[i] != k {
|
|
t.Fatalf("current value %d is %s, want %s", i, got[i], k)
|
|
}
|
|
}
|
|
|
|
// And it holds: a verdict written afterwards reaches it as a change.
|
|
next := "novox/mesh-sdk.77." + strings.Repeat("a", 40) + ".status=mesh/check"
|
|
if _, err := kv.Put(next, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"pending"}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mt.Until(t, func() error {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if len(live) != 1 || live[0] != next {
|
|
return fmt.Errorf("live changes so far: %v", live)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// A watch whose bundle takes nothing for longer than the bound is still refused, so a bundle that hangs
|
|
// is said, not waited on for ever.
|
|
func TestAWatchIsRefusedWhenNoCurrentValueIsTakenWithinTheBound(t *testing.T) {
|
|
was := bus.StateTimeout
|
|
bus.StateTimeout = 300 * time.Millisecond
|
|
t.Cleanup(func() { bus.StateTimeout = was })
|
|
|
|
conn, kv := owedWatcher(t)
|
|
for i := 0; i < 3; i++ {
|
|
if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
release := make(chan struct{})
|
|
t.Cleanup(func() { close(release) })
|
|
began := time.Now()
|
|
_, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(bus.StateChange) error {
|
|
<-release
|
|
return nil
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "did not arrive in") {
|
|
t.Fatalf("a watch whose first value was never taken: want refused, got %v", err)
|
|
}
|
|
if took := time.Since(began); took > 3*time.Second {
|
|
t.Fatalf("the refusal took %s, far past the bound", took)
|
|
}
|
|
}
|
|
|
|
// **A membership the grants part wrote in its old wrapper is served whole** (issue 774). Until 2026-10-30 the part wrote
|
|
// its pass record, the membership inside, into its state memberships; the tool runner read the value as a bare
|
|
// membership, took it over the assignments stream's copy, and served no tool and no state. This is what the live mesh
|
|
// held: read through mesh-sdk's codec, its tools are answered where the membership says and its state is reached.
|
|
func TestAMembershipTheGrantsPartWroteWrappedIsServed(t *testing.T) {
|
|
mesh := mt.New(t)
|
|
mesh.Bucket(t, "gitea_repos")
|
|
gitea := mt.MembershipOf("gitea", "anchor", false, nil)
|
|
gitea.State = []bus.StateIssued{{Name: "repos", Bucket: "gitea_repos", Writes: true}}
|
|
mesh.IssueInGrantsWrapped(t, gitea, "published")
|
|
// The old wrapper of a membership the part held back carries its unpublished composition: refused, so the
|
|
// assignments stream's copy is served, never what the hold kept back.
|
|
pages := mt.MembershipOf("pages", "anchor", false, nil)
|
|
mesh.Issue(t, pages)
|
|
heldBack := mt.MembershipOf("pages", "anchor", true, nil)
|
|
mesh.IssueInGrantsWrapped(t, heldBack, "not published, a shadow difference: publishing would lose the last holder of x")
|
|
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
conn.Logf = func(string, ...any) {}
|
|
t.Cleanup(conn.Close)
|
|
conn.Follow("gitea")
|
|
mt.Until(t, func() error {
|
|
if m := conn.Membership("gitea"); m == nil || len(m.Serves) == 0 {
|
|
return fmt.Errorf("gitea's membership is %+v, want the one inside the wrapper", m)
|
|
}
|
|
return nil
|
|
})
|
|
if got, want := fmt.Sprint(conn.ServedOn("gitea", "gitea_list")), "[{mesh.mod.gitea.tool.gitea_list.anchor }]"; got != want {
|
|
t.Errorf("gitea_list is served on %s, want %s", got, want)
|
|
}
|
|
if _, err := conn.StatePut("gitea", "repos", "novox.hq", []byte(`{"v":1}`)); err != nil {
|
|
t.Fatalf("the state the membership issues is refused: %v", err)
|
|
}
|
|
if e, err := conn.StateGet("gitea", "repos", "novox.hq"); err != nil || e == nil || string(e.Value) != `{"v":1}` {
|
|
t.Errorf("the state reads back %+v, %v", e, err)
|
|
}
|
|
conn.Follow("pages")
|
|
mt.Until(t, func() error {
|
|
if m := conn.Membership("pages"); m == nil || len(m.Serves) == 0 {
|
|
return fmt.Errorf("pages' membership is %+v, want the assignments stream's", m)
|
|
}
|
|
return nil
|
|
})
|
|
if got, want := fmt.Sprint(conn.ServedOn("pages", "pages_list")), "[{mesh.mod.pages.tool.pages_list.anchor }]"; got != want {
|
|
t.Errorf("a held-back membership's wrapper is served: pages_list on %s, want the stream's %s", got, want)
|
|
}
|
|
}
|
|
|
|
// owedWatcher is the gitea module's view of the checker's owed: the bucket, and a runtime connection
|
|
// that serves gitea with the owed issued for reading.
|
|
func owedWatcher(t *testing.T) (*bus.Conn, interface {
|
|
Put(string, []byte) (uint64, error)
|
|
}) {
|
|
t.Helper()
|
|
mesh := mt.New(t)
|
|
kv := mesh.Bucket(t, "mesh-delivery-checker_owed")
|
|
gitea := mt.MembershipOf("gitea", "anchor", false, nil)
|
|
gitea.State = []bus.StateIssued{{Name: "mesh-delivery-checker.owed", Bucket: "mesh-delivery-checker_owed"}}
|
|
mesh.Issue(t, gitea)
|
|
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
conn.Logf = func(string, ...any) {}
|
|
t.Cleanup(conn.Close)
|
|
conn.Follow("gitea")
|
|
mt.Until(t, func() error {
|
|
if conn.Membership("gitea") == nil {
|
|
return errors.New("gitea not issued yet")
|
|
}
|
|
return nil
|
|
})
|
|
return conn, kv
|
|
}
|
|
|
|
// **A watch given up, or stopped, hands over nothing more** (issue 774). The runtime's watch stopped its watcher
|
|
// when the current values did not arrive within the bound, but the watcher still held values, and its goroutine
|
|
// went on handing them over: to a watch id the bundle no longer held (on the provisioner, 51 failed watches were
|
|
// followed by 1257 orphan changes). Each row blocks the bundle on its first value, gives the watch up, lets the
|
|
// bundle go and counts what it is still handed.
|
|
func TestAWatchGivenUpHandsOverNothingMore(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
refused bool // the bound gives it up, or the caller stops it
|
|
}{
|
|
{"refused at the bound", true},
|
|
{"stopped by its caller", false},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
was := bus.StateTimeout
|
|
bus.StateTimeout = 300 * time.Millisecond
|
|
t.Cleanup(func() { bus.StateTimeout = was })
|
|
conn, kv := owedWatcher(t)
|
|
put := func(i int) {
|
|
if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if tc.refused {
|
|
for i := 0; i < 4; i++ {
|
|
put(i)
|
|
}
|
|
}
|
|
var calls atomic.Int32
|
|
first := make(chan struct{}, 1)
|
|
release := make(chan struct{})
|
|
deliver := func(ch bus.StateChange) error {
|
|
if calls.Add(1) == 1 {
|
|
first <- struct{}{}
|
|
<-release
|
|
}
|
|
return nil
|
|
}
|
|
var stop func()
|
|
if tc.refused {
|
|
if _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err == nil {
|
|
t.Fatal("the watch whose bundle took nothing was not refused")
|
|
}
|
|
} else {
|
|
var err error
|
|
if stop, err = conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 4; i++ {
|
|
put(i)
|
|
}
|
|
select {
|
|
case <-first:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("the bundle was handed no change")
|
|
}
|
|
stop()
|
|
}
|
|
// Let the watcher's server side deliver what it held, then free the bundle.
|
|
time.Sleep(300 * time.Millisecond)
|
|
close(release)
|
|
time.Sleep(500 * time.Millisecond)
|
|
if n := calls.Load(); n != 1 {
|
|
t.Fatalf("the bundle was handed %d changes of a watch given up, want 1 (the one in hand)", n)
|
|
}
|
|
})
|
|
}
|
|
}
|