Reading the catalog half by fingerprint, the node-engine no longer reads the builder's builds, so #224's refused key was never written there and a reader waited on a build that will never be done. The engine now hears the builder's build.ended stream, read whole at start then followed (as the assigner hears it); the last trunk end of a placed module that ended failed is written as the refused key with the event's words, a later done end deletes it, an end naming a branch moves nothing. The manifest consumes mesh-delivery-builder.build.ended.**. Builds: design 47 G A-293 (with §421, §628), ADR 0593 §6, issue 1966 U1/U8 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
236 lines
8.4 KiB
Go
236 lines
8.4 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ed25519"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
|
"github.com/nats-io/nats.go"
|
|
)
|
|
|
|
// quietLink is a link with no broker behind it.
|
|
type quietLink struct {
|
|
lost chan error
|
|
}
|
|
|
|
func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} }
|
|
|
|
func (l *quietLink) Lost() <-chan error { return l.lost }
|
|
func (l *quietLink) Close() {}
|
|
|
|
// runningLink hears the node seat's running events: the last of each subject first, then the durable consumer's.
|
|
type runningLink struct {
|
|
*quietLink
|
|
first, after [][]byte
|
|
}
|
|
|
|
func (l *runningLink) FirstRunning(_ context.Context, first func([]byte)) error {
|
|
for _, raw := range l.first {
|
|
first(raw)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *runningLink) HearRunning(_ string, heard func([]byte)) (func(), error) {
|
|
for _, raw := range l.after {
|
|
heard(raw)
|
|
}
|
|
return func() {}, nil
|
|
}
|
|
|
|
func (l *runningLink) HearBuildEnds(_ context.Context, heard func([]byte), readEnded func()) error {
|
|
for _, raw := range l.first {
|
|
heard(raw)
|
|
}
|
|
readEnded()
|
|
for _, raw := range l.after {
|
|
heard(raw)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
|
|
t.Helper()
|
|
public, private, err := ed25519.GenerateKey(nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return Membership{Node: "n1", Signer: public}, private
|
|
}
|
|
|
|
// Facts made with no link open are not kept, and nothing is said: the next link says them as they are then.
|
|
func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
|
|
m, _ := aMember(t)
|
|
l := newQuietLink()
|
|
s := &Session{Membership: m, Timeout: time.Second}
|
|
if s.SayFacts(context.Background(), Report{}) {
|
|
t.Fatal("facts were said with no link")
|
|
}
|
|
s.Attach(context.Background(), l)
|
|
s.Detach(l)
|
|
if s.Linked() || s.SayFacts(context.Background(), Report{}) {
|
|
t.Fatal("facts were said on a link that was let go")
|
|
}
|
|
|
|
// Design 47 G §199, §556: with no link the running events are not followed; with one, the last message of each subject
|
|
// comes first, the read ends, then the durable consumer's, and a body that is no running event is passed over.
|
|
if err := s.FollowRunning(context.Background(), func(nodeengine.Running) {}, func() {}); !errors.Is(err, ErrNoLink) {
|
|
t.Fatalf("followed with no link: %v", err)
|
|
}
|
|
rl := &runningLink{quietLink: newQuietLink(),
|
|
first: [][]byte{[]byte(`{"node":"a","module":"x","fingerprint":"f1"}`), []byte(`not json`), []byte(`{"module":"x"}`)},
|
|
after: [][]byte{[]byte(`{"node":"b","module":"y","stopped":true}`)}}
|
|
s.Attach(context.Background(), rl)
|
|
var order []string
|
|
ctx, stop := context.WithCancel(context.Background())
|
|
err := s.FollowRunning(ctx, func(ev nodeengine.Running) { order = append(order, ev.Node+"/"+ev.Module) },
|
|
func() { order = append(order, "read"); stop() })
|
|
if err != nil || strings.Join(order, " ") != "a/x b/y read" {
|
|
t.Fatalf("followed %v: %v, want a/x b/y read", order, err)
|
|
}
|
|
|
|
// Design 47 G A-293: the builder's build ends likewise, the last of each subject first, then the read's end, then the
|
|
// rest, a body that is no end of a module at a fingerprint passed over.
|
|
if err := (&Session{Membership: m}).FollowBuildEnds(context.Background(), func(builds.BuildEnded) {}, func() {}); !errors.Is(err, ErrNoLink) {
|
|
t.Fatalf("build ends followed with no link: %v", err)
|
|
}
|
|
rl.first = [][]byte{[]byte(`{"module":"x","fingerprint":"f1","outcome":"failed","error":"no go"}`), []byte(`not json`), []byte(`{"module":"x"}`)}
|
|
rl.after = [][]byte{[]byte(`{"module":"y","fingerprint":"f2","outcome":"done","branch":"b"}`)}
|
|
order = nil
|
|
err = s.FollowBuildEnds(context.Background(), func(e builds.BuildEnded) {
|
|
order = append(order, e.Module+"@"+e.Fingerprint+":"+e.Outcome+":"+e.Error+":"+e.Branch)
|
|
}, func() { order = append(order, "read") })
|
|
if err != nil || strings.Join(order, " ") != "x@f1:failed:no go: read y@f2:done::b" {
|
|
t.Fatalf("build ends followed %v: %v", order, err)
|
|
}
|
|
}
|
|
|
|
// A link opening says the node's facts, off the link's own loop.
|
|
func TestALinkOpeningSaysTheFacts(t *testing.T) {
|
|
m, _ := aMember(t)
|
|
l := newQuietLink()
|
|
opened := make(chan struct{}, 1)
|
|
s := &Session{Membership: m, Timeout: time.Second, OnLink: func(context.Context) { opened <- struct{}{} }}
|
|
s.Attach(context.Background(), l)
|
|
select {
|
|
case <-opened:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("a link opening did not ask for the facts")
|
|
}
|
|
}
|
|
|
|
// aBus is a connection to a real server, because some claims here are about one: set MESH_TEST_NATS to a server with
|
|
// JetStream, or the test is skipped.
|
|
func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) {
|
|
t.Helper()
|
|
url := os.Getenv("MESH_TEST_NATS")
|
|
if url == "" {
|
|
t.Skip("MESH_TEST_NATS unset")
|
|
}
|
|
conn, err := nats.Connect(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(conn.Close)
|
|
js, err := conn.JetStream()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return conn, js
|
|
}
|
|
|
|
// eventuallyHeld waits for a condition the bus reaches on its own time.
|
|
func eventuallyHeld(t *testing.T, what string, ok func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for !ok() {
|
|
if time.Now().After(deadline) {
|
|
t.Fatal(what)
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// refusingLink is a quiet link the bus refused subscriptions to.
|
|
type refusingLink struct {
|
|
*quietLink
|
|
refused chan string
|
|
}
|
|
|
|
func (l *refusingLink) Refused() <-chan string { return l.refused }
|
|
|
|
// The sweep of 2026-10-30: every engine was refused its node seat's subscriptions at 12:10, by a user list composed
|
|
// without them, and nats asks a refused subscription again only on a new connection; the seat answered nobody all
|
|
// afternoon. A refused subscription is read from the server's words, said, and the link opened again after a pause
|
|
// that grows while it is refused again; a refused publish is not one.
|
|
func TestASubscriptionTheBusRefusedIsAskedAgainOnANewLink(t *testing.T) {
|
|
// The server's words, as the client hands them on (journal of nox-mesh-host on novox, 2026-10-30 12:10:44).
|
|
for _, row := range []struct {
|
|
err error
|
|
want string
|
|
}{
|
|
{fmt.Errorf("%w: %s", nats.ErrPermissionViolation,
|
|
`Permissions Violation for Subscription to "mesh.seat.node-engine.tool.root-free.novox"`),
|
|
"mesh.seat.node-engine.tool.root-free.novox"},
|
|
{errors.Join(fmt.Errorf("%w: %s", nats.ErrPermissionViolation,
|
|
`Permissions Violation for Subscription to "$SRV.PING.node-engine"`), errors.New("stopping service: x")),
|
|
"$SRV.PING.node-engine"},
|
|
{fmt.Errorf("%w: %s", nats.ErrPermissionViolation,
|
|
`Permissions Violation for Publish to "$JS.API.STREAM.INFO.KV_mesh-host_health"`), ""},
|
|
{errors.New("nats: timeout"), ""},
|
|
} {
|
|
got, ok := RefusedSubscription(row.err)
|
|
if got != row.want || ok != (row.want != "") {
|
|
t.Errorf("%v reads as %q, %v", row.err, got, ok)
|
|
}
|
|
l := &natsLink{refused: make(chan string, 1)}
|
|
l.asyncError(nil, nil, row.err)
|
|
if heard := len(l.refused); heard != map[bool]int{true: 1}[row.want != ""] {
|
|
t.Errorf("%v handed %d refusals on", row.err, heard)
|
|
}
|
|
}
|
|
|
|
m, _ := aMember(t)
|
|
q := &Session{Membership: m, Timeout: time.Second, RefusedPause: 50 * time.Millisecond}
|
|
var said []string
|
|
var mu sync.Mutex
|
|
say := func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }
|
|
for _, want := range []time.Duration{50 * time.Millisecond, 100 * time.Millisecond} {
|
|
l := &refusingLink{quietLink: newQuietLink(), refused: make(chan string, 2)}
|
|
l.refused <- "mesh.seat.node-engine.tool.root-free.n1"
|
|
l.refused <- "$SRV.PING"
|
|
began := time.Now()
|
|
done := make(chan error, 1)
|
|
go func() { done <- serve(context.Background(), l, m, q, say, time.Second) }()
|
|
select {
|
|
case err := <-done:
|
|
if err == nil || !strings.Contains(err.Error(), "refused 2 of this node's subscriptions, "+
|
|
"mesh.seat.node-engine.tool.root-free.n1 first") {
|
|
t.Fatalf("the link ended with %v", err)
|
|
}
|
|
if took := time.Since(began); took < want {
|
|
t.Errorf("the link was dropped after %s, before its pause of %s", took, want)
|
|
}
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("a link the bus refused a subscription was kept, and the subscription never asked again")
|
|
}
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if n := strings.Count(strings.Join(said, "\n"), "the bus refused this node a subscription to "+
|
|
"mesh.seat.node-engine.tool.root-free.n1"); n != 2 {
|
|
t.Errorf("the refusal was said %d times in two links: %q", n, said)
|
|
}
|
|
if q.refusalsHeld(); q.refusedPause() != 50*time.Millisecond {
|
|
t.Error("a link that held its subscriptions did not start from the first pause again")
|
|
}
|
|
}
|