//go:build integration package bus_test import ( "errors" "fmt" "strings" "sync" "sync/atomic" "testing" "time" "github.com/novox/mesh-tools/node-tools/internal/bus" mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" ) // **A watch whose current values take longer than the bound to hand over still holds** (novox/hq issue // 1726). The gitea module watches the checker's owed: one key per pull request head, each a round trip // to the bundle. With a few hundred keys the hand-over outlasted StateTimeout at every attempt, the // watch was refused, and the newest keys, handed over last, never reached mesh/check: a group's passing // verdict was never written on its members' heads. The bound is on progress: a watch fails only when // no current value is handed over within it. func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) { was := bus.StateTimeout bus.StateTimeout = 400 * time.Millisecond t.Cleanup(func() { bus.StateTimeout = was }) conn, kv := owedWatcher(t) // A group of three pull requests judged last, after many older heads: their keys come last. const older = 30 var keys []string for i := 0; i < older; i++ { keys = append(keys, fmt.Sprintf("novox/mesh-catalog.%d.%040x.status=mesh/check", 400+i, i)) } for _, member := range []string{"novox/mesh-sdk.76.34f210e5", "novox/mesh-controller.453.c06f75d0", "novox/mesh-delivery.33.28ee22ef"} { keys = append(keys, member+strings.Repeat("0", 32)+".status=mesh/check") } for _, k := range keys { if _, err := kv.Put(k, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"success"}`)); err != nil { t.Fatal(err) } } var mu sync.Mutex var current, live []string // Each value takes 40 ms to be taken: 33 values take about 1.3 s, three times the bound. stop, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(ch bus.StateChange) error { if ch.Current { time.Sleep(40 * time.Millisecond) } mu.Lock() defer mu.Unlock() if ch.Current { current = append(current, ch.Key) } else { live = append(live, ch.Key) } return nil }) if err != nil { t.Fatalf("a watch whose current values kept arriving was refused: %v", err) } t.Cleanup(stop) mu.Lock() got := append([]string(nil), current...) mu.Unlock() if len(got) != len(keys) { t.Fatalf("the watch returned after %d of %d current values", len(got), len(keys)) } for i, k := range keys { if got[i] != k { t.Fatalf("current value %d is %s, want %s", i, got[i], k) } } // And it holds: a verdict written afterwards reaches it as a change. next := "novox/mesh-sdk.77." + strings.Repeat("a", 40) + ".status=mesh/check" if _, err := kv.Put(next, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"pending"}`)); err != nil { t.Fatal(err) } mt.Until(t, func() error { mu.Lock() defer mu.Unlock() if len(live) != 1 || live[0] != next { return fmt.Errorf("live changes so far: %v", live) } return nil }) } // A watch whose bundle takes nothing for longer than the bound is still refused, so a bundle that hangs // is said, not waited on for ever. func TestAWatchIsRefusedWhenNoCurrentValueIsTakenWithinTheBound(t *testing.T) { was := bus.StateTimeout bus.StateTimeout = 300 * time.Millisecond t.Cleanup(func() { bus.StateTimeout = was }) conn, kv := owedWatcher(t) for i := 0; i < 3; i++ { if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil { t.Fatal(err) } } release := make(chan struct{}) t.Cleanup(func() { close(release) }) began := time.Now() _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(bus.StateChange) error { <-release return nil }) if err == nil || !strings.Contains(err.Error(), "did not arrive in") { t.Fatalf("a watch whose first value was never taken: want refused, got %v", err) } if took := time.Since(began); took > 3*time.Second { t.Fatalf("the refusal took %s, far past the bound", took) } } // **A membership the grants part wrote in its old wrapper is served whole** (issue 774). Until 2026-10-30 the part wrote // its pass record, the membership inside, into its state memberships; the tool runner read the value as a bare // membership, took it over the assignments stream's copy, and served no tool and no state. This is what the live mesh // held: read through mesh-sdk's codec, its tools are answered where the membership says and its state is reached. func TestAMembershipTheGrantsPartWroteWrappedIsServed(t *testing.T) { mesh := mt.New(t) mesh.Bucket(t, "gitea_repos") gitea := mt.MembershipOf("gitea", "anchor", false, nil) gitea.State = []bus.StateIssued{{Name: "repos", Bucket: "gitea_repos", Writes: true}} mesh.IssueInGrantsWrapped(t, gitea, "published") // The old wrapper of a membership the part held back carries its unpublished composition: refused, so the // assignments stream's copy is served, never what the hold kept back. pages := mt.MembershipOf("pages", "anchor", false, nil) mesh.Issue(t, pages) heldBack := mt.MembershipOf("pages", "anchor", true, nil) mesh.IssueInGrantsWrapped(t, heldBack, "not published, a shadow difference: publishing would lose the last holder of x") conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"}) if err != nil { t.Fatal(err) } conn.Logf = func(string, ...any) {} t.Cleanup(conn.Close) conn.Follow("gitea") mt.Until(t, func() error { if m := conn.Membership("gitea"); m == nil || len(m.Serves) == 0 { return fmt.Errorf("gitea's membership is %+v, want the one inside the wrapper", m) } return nil }) if got, want := fmt.Sprint(conn.ServedOn("gitea", "gitea_list")), "[{mesh.mod.gitea.tool.gitea_list.anchor }]"; got != want { t.Errorf("gitea_list is served on %s, want %s", got, want) } if _, err := conn.StatePut("gitea", "repos", "novox.hq", []byte(`{"v":1}`)); err != nil { t.Fatalf("the state the membership issues is refused: %v", err) } if e, err := conn.StateGet("gitea", "repos", "novox.hq"); err != nil || e == nil || string(e.Value) != `{"v":1}` { t.Errorf("the state reads back %+v, %v", e, err) } conn.Follow("pages") mt.Until(t, func() error { if m := conn.Membership("pages"); m == nil || len(m.Serves) == 0 { return fmt.Errorf("pages' membership is %+v, want the assignments stream's", m) } return nil }) if got, want := fmt.Sprint(conn.ServedOn("pages", "pages_list")), "[{mesh.mod.pages.tool.pages_list.anchor }]"; got != want { t.Errorf("a held-back membership's wrapper is served: pages_list on %s, want the stream's %s", got, want) } } // owedWatcher is the gitea module's view of the checker's owed: the bucket, and a runtime connection // that serves gitea with the owed issued for reading. func owedWatcher(t *testing.T) (*bus.Conn, interface { Put(string, []byte) (uint64, error) }) { t.Helper() mesh := mt.New(t) kv := mesh.Bucket(t, "mesh-delivery-checker_owed") gitea := mt.MembershipOf("gitea", "anchor", false, nil) gitea.State = []bus.StateIssued{{Name: "mesh-delivery-checker.owed", Bucket: "mesh-delivery-checker_owed"}} mesh.Issue(t, gitea) conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"}) if err != nil { t.Fatal(err) } conn.Logf = func(string, ...any) {} t.Cleanup(conn.Close) conn.Follow("gitea") mt.Until(t, func() error { if conn.Membership("gitea") == nil { return errors.New("gitea not issued yet") } return nil }) return conn, kv } // **A watch given up, or stopped, hands over nothing more** (issue 774). The runtime's watch stopped its watcher // when the current values did not arrive within the bound, but the watcher still held values, and its goroutine // went on handing them over: to a watch id the bundle no longer held (on the provisioner, 51 failed watches were // followed by 1257 orphan changes). Each row blocks the bundle on its first value, gives the watch up, lets the // bundle go and counts what it is still handed. func TestAWatchGivenUpHandsOverNothingMore(t *testing.T) { for _, tc := range []struct { name string refused bool // the bound gives it up, or the caller stops it }{ {"refused at the bound", true}, {"stopped by its caller", false}, } { t.Run(tc.name, func(t *testing.T) { was := bus.StateTimeout bus.StateTimeout = 300 * time.Millisecond t.Cleanup(func() { bus.StateTimeout = was }) conn, kv := owedWatcher(t) put := func(i int) { if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil { t.Fatal(err) } } if tc.refused { for i := 0; i < 4; i++ { put(i) } } var calls atomic.Int32 first := make(chan struct{}, 1) release := make(chan struct{}) deliver := func(ch bus.StateChange) error { if calls.Add(1) == 1 { first <- struct{}{} <-release } return nil } var stop func() if tc.refused { if _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err == nil { t.Fatal("the watch whose bundle took nothing was not refused") } } else { var err error if stop, err = conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err != nil { t.Fatal(err) } for i := 0; i < 4; i++ { put(i) } select { case <-first: case <-time.After(5 * time.Second): t.Fatal("the bundle was handed no change") } stop() } // Let the watcher's server side deliver what it held, then free the bundle. time.Sleep(300 * time.Millisecond) close(release) time.Sleep(500 * time.Millisecond) if n := calls.Load(); n != 1 { t.Fatalf("the bundle was handed %d changes of a watch given up, want 1 (the one in hand)", n) } }) } }