Files
mesh-tools/node-tools/internal/bus/bus.go
T
jschoubben eb174d853f H1 integration bound: the runner follows a membership without dropping a call
On every membership of a module, HandleCalled stopped each subscription
of a tool and subscribed again on the same subjects; a call the bus
routed in that window was dropped by the client in silence, and its
caller waited the runner's whole bound (30 s). On the build node this
was the 30 s of TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership
and of TestTheLedgerRepairsAreToolsOfEveryLedgerOwner, and most of the
integration level's 72 s. Now a membership keeps every subscription whose
subject and queue it still names, subscribes the new ones, and only then
stops those it no longer names (make before break).

The test asks the served tool once after each of five memberships, with
the tests' own call bound of 2 s (principle 11); it fails without the
change. The runtime tests' call helper and the ledger listing wait that
bound, not the runner's 30 s.

Builds: design 47 H §774, H-A-158; principle 11
2026-11-20 17:18:09 +01:00

1307 lines
46 KiB
Go

// Package bus is the node runtime's connection to the mesh bus, on NATS (novox/hq design 25, design
// 29, ADR 0160, ADR 0175). It is the Go port of node-tools' broker-nats.ts, and speaks the same
// wire: the same subjects, the same JSON request and reply bodies, the same event headers.
//
// mesh.mod.<module>.event.<type> an event a module emits
// mesh.mod.<module>.tool.<tool> a tool a module serves
// mesh.seat.<seat>.tool.<verb> a role's verb, answered by whoever holds the seat
// mesh.mod.<module>.call.<tool>.<caller>, mesh.seat.<seat>.call.<verb>.<caller>
// the same, naming the bus user that called (caller.go, hq issue 365)
package bus
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/membership"
"log"
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-tools/node-tools/internal/wire"
)
// RequestTimeout is how long a call waits for its answer when its verb's definition names no `timeout`: the tool runner's own
// bound, what modules already expect (design 47 H §760, H-A-43).
const RequestTimeout = 30 * time.Second
// waitOf is the bound a call waits: within where the verb's definition names one, else the tool runner's own.
func waitOf(within time.Duration) time.Duration {
if within > 0 {
return within
}
return RequestTimeout
}
// Credential is a broker credential as the mesh delivers it (novox/hq ADR 0120).
type Credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"`
Node string `json:"node,omitempty"`
Module string `json:"module,omitempty"`
User string `json:"user,omitempty"`
Password string `json:"password,omitempty"`
Claims []Claim `json:"claims,omitempty"`
}
// Claim is a seat a module claims, with the verbs it promises (novox/hq ADR 0159).
type Claim struct {
Seat string `json:"seat"`
Scope string `json:"scope,omitempty"`
Serves []string `json:"serves,omitempty"`
}
// Membership is what the mesh issued one assignment (novox/hq ADR 0160).
type Membership struct {
Node string `json:"node"`
Module string `json:"module"`
Serves []Served `json:"serves"`
Seats []SeatVerb `json:"seats,omitempty"`
Emits string `json:"emits"`
Reaches map[string][]string `json:"reaches,omitempty"`
Tools string `json:"tools"`
// State is every bucket the module's code may reach, by the name it uses (novox/hq ADR 0201).
State []StateIssued `json:"state,omitempty"`
// Logs is every log the module keeps, by its local name (novox/hq ADR 0297): its own alone.
Logs []LogIssued `json:"logs,omitempty"`
// SeatTraffic is what the module's code may submit, say, take, ask, answer and read on seats that name
// their caller or their kind (novox/hq ADR 0259): the runtime does for it only what is listed here.
SeatTraffic *SeatTraffic `json:"seat-traffic,omitempty"`
// Streams are the module's own streams (novox/hq ADR 0363 §2): an event one of them carries is published
// on that stream's subject, never on EVENTS (streams.go).
Streams []StreamIssued `json:"streams,omitempty"`
// Hears are the module streams the module hears, each through a durable consumer on that stream the
// runtime binds beside the one on EVENTS (ADR 0363 §8).
Hears []HeardIssued `json:"hears,omitempty"`
// Bindings are the module's bus state in the new form, by local name (novox/hq ADR 0595 §1, mesh-sdk
// membership.BusBinding): its object stores, streams, work queues and ledgers, its own alone (bindings.go).
Bindings []membership.BusBinding `json:"bindings,omitempty"`
}
// Served is an address a tool is answered on; `{tool}` stands for the tool's name.
type Served struct {
Subject string `json:"subject"`
Queue string `json:"queue,omitempty"`
}
// SeatVerb is one verb of a seat a module holds, where it is answered.
type SeatVerb struct {
Seat string `json:"seat"`
Verb string `json:"verb"`
Subject string `json:"subject"`
}
// Envelope is an event as a module emits it: the body is the payload, the metadata rides as headers
// (novox/hq ADR 0042).
type Envelope struct {
Key string `json:"key"`
Node string `json:"node,omitempty"`
Body json.RawMessage `json:"body"`
Headers map[string]string `json:"headers,omitempty"`
}
// Answered is a call's result and the machine that gave it (novox/hq ADR 0159).
type Answered struct {
Result json.RawMessage
Node string
}
// Handler answers one request body.
type Handler func(body json.RawMessage) (any, error)
// ErrPin is a bus whose certificate is not the one the mesh pinned: final, never retried.
var ErrPin = errors.New("the bus's certificate does not match the pin")
// Fatal is why a connection failure is final rather than "not yet", or "" when waiting may fix it —
// the same classification the TypeScript runtime makes.
func Fatal(err error) string {
if err == nil {
return ""
}
msg := err.Error()
if errors.Is(err, ErrPin) || strings.Contains(msg, ErrPin.Error()) {
return "the bus's certificate does not match the pin"
}
if regexp.MustCompile(`(?i)invalid url|no servers available for connection: .*url`).MatchString(msg) ||
strings.Contains(msg, "nats: invalid url") {
return "the bus address is not a usable URL"
}
if Refused(err) || regexp.MustCompile(`(?i)user authentication expired|permissions violation`).MatchString(msg) {
return "the bus refused this account"
}
return ""
}
// Refused says whether the bus refused the account a connection named (an authorization violation): final for a running
// runtime, but at a start possibly only "not yet", since the bus carries a new account after it reloads its users (issue
// 1496). Fatal still names it; the caller decides how long a start waits.
func Refused(err error) bool {
return err != nil && regexp.MustCompile(`(?i)authorization violation`).MatchString(err.Error())
}
func normalizeFingerprint(f string) string {
f = strings.TrimSpace(f)
if len(f) > 7 && strings.EqualFold(f[:7], "sha256:") {
f = f[7:]
}
return strings.ToLower(strings.ReplaceAll(f, ":", ""))
}
// pinned accepts exactly the certificate with this SHA-256 and no other. The pin is the only check:
// the bus's certificate names the seat, not the address a machine dials it by.
func pinned(want string) *tls.Config {
want = normalizeFingerprint(want)
return &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin, which is stricter
MinVersion: tls.VersionTLS12,
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
if len(raw) == 0 {
return fmt.Errorf("%w: it presented none", ErrPin)
}
sum := sha256.Sum256(raw[0])
if got := hex.EncodeToString(sum[:]); got != want {
return fmt.Errorf("%w: it presented %s, not the pinned %s", ErrPin, got, want)
}
return nil
},
}
}
// Conn is the runtime's connection: the module it is, the memberships it follows, the subjects it
// answers.
type Conn struct {
nc *nats.Conn
js nats.JetStreamContext
self string
node string
cred Credential
mu sync.Mutex
issued map[string]*Membership // module → membership; present with nil = followed, none issued
onNew []*func(Membership)
subs []*nats.Subscription
Logf func(format string, args ...any)
// answering is every subject this runtime answers on, so a subscription the bus refused can be
// asked for again (novox/hq issue 222).
answering map[string]*answered
// RetryAfter is how long after a refusal a subscription is asked for again, by attempt; past the
// last, it is given up and said. A variable so a test need not wait minutes.
RetryAfter []time.Duration
// calls are the caller-named calls in flight (caller.go, novox/hq issue 365).
calls callers
// waits are the waits in flight by subject, each told when the bus refuses its subscription (wait.go).
waits map[string]map[chan string]bool
// inboxSaid is when a refused answer publish was last said, by subject (answerRefused).
inboxSaid map[string]time.Time
}
// answered is one subject the runtime answers on: what to subscribe again with, and how often it was
// refused.
type answered struct {
subject, queue string
cb nats.MsgHandler
sub *nats.Subscription
refusals int
stopped bool
}
// retryAfter is the default wait before each new attempt: a push sends the bus's user list in the
// same breath as the machine that needs it, and the bus reloads it a moment later — five minutes
// covers a slow push, and a grant that never comes is said and left.
var retryAfter = []time.Duration{2 * time.Second, 5 * time.Second, 10 * time.Second, 20 * time.Second,
30 * time.Second, 60 * time.Second, 60 * time.Second, 120 * time.Second}
var (
refusedSubject = regexp.MustCompile(`Subscription to "(\S+)"`)
refusedQueue = regexp.MustCompile(`using queue "(\S+)"`)
)
func answeringKey(subject, queue string) string { return subject + "\x00" + queue }
// refused is the bus saying no to a subscription. **Asked again, not given up** (novox/hq issue 222):
// what an account may answer is the bus's user list, written on the bus's machine, and a push that
// assigns a module somewhere sends that machine and the bus's in the same breath — the runtime can
// subscribe in the moment before the bus has reloaded. Refused once, the subscription stayed dead
// until some later membership happened to re-serve it, and the module ran unreachable meanwhile.
func (c *Conn) refused(err error) {
if !errors.Is(err, nats.ErrPermissionViolation) {
return
}
if c.publishRefused(err) {
return
}
if c.answerRefused(err) {
return
}
if subject := subjectRefused(err); subject != "" && c.waitRefused(subject) {
return
}
m := refusedSubject.FindStringSubmatch(err.Error())
if len(m) < 2 {
return
}
queue := ""
if q := refusedQueue.FindStringSubmatch(err.Error()); len(q) >= 2 {
queue = q[1]
}
c.mu.Lock()
a := c.answering[answeringKey(m[1], queue)]
if a == nil || a.stopped {
c.mu.Unlock()
return
}
waits := c.RetryAfter
if waits == nil {
waits = retryAfter
}
if a.refusals >= len(waits) {
c.mu.Unlock()
c.Logf("[mesh-tools] the bus still refuses %s after %d attempts; not served here until the mesh issues it again", a.subject, a.refusals)
return
}
wait := waits[a.refusals]
a.refusals++
attempt := a.refusals
c.mu.Unlock()
c.Logf("[mesh-tools] the bus refused %s; asking again in %s (attempt %d)", a.subject, wait, attempt)
time.AfterFunc(wait, func() {
c.mu.Lock()
defer c.mu.Unlock()
if a.stopped {
return
}
if a.sub != nil {
_ = a.sub.Unsubscribe()
}
var sub *nats.Subscription
var err error
if a.queue != "" {
sub, err = c.nc.QueueSubscribe(a.subject, a.queue, a.cb)
} else {
sub, err = c.nc.Subscribe(a.subject, a.cb)
}
if err != nil {
c.Logf("[mesh-tools] asking again for %s failed: %v", a.subject, err)
return
}
a.sub = sub
c.subs = append(c.subs, sub)
})
}
// Connect dials the bus as the credential's module and follows its membership. A module's subjects come from its
// credential, never from its calls (ADR 0074).
func Connect(cred Credential) (*Conn, error) {
c, err := Dial(cred)
if err != nil {
return nil, err
}
c.Follow(cred.Module)
return c, nil
}
// Dial dials the bus as the credential's module and follows nothing: a connection that only publishes for its module,
// as a module's ledger connection does (novox/hq ADR 0597 §9).
func Dial(cred Credential) (*Conn, error) {
if cred.Module == "" {
return nil, errors.New("a broker credential with no module: the runtime derives its subjects " +
"from the account the mesh issued, and cannot guess which module it is")
}
node := cred.Node
if node == "" {
node = "?"
}
var c *Conn
opts := []nats.Option{
nats.Name(node + "." + cred.Module),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
if c != nil {
c.refused(err)
}
}),
// Reconnect forever: the bus restarting is an upgrade, not a reason to exit.
nats.MaxReconnects(-1),
}
if cred.User != "" {
opts = append(opts, nats.UserInfo(cred.User, cred.Password),
// Its own inbox: every user's inbox is private to it (design 25 §4).
nats.CustomInboxPrefix("_INBOX."+cred.User))
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
}
nc, err := nats.Connect(cred.URL, opts...)
if err != nil {
return nil, err
}
js, err := nc.JetStream()
if err != nil {
nc.Close()
return nil, err
}
c = &Conn{nc: nc, js: js, self: cred.Module, node: cred.Node, cred: cred,
issued: map[string]*Membership{}, Logf: log.Printf, answering: map[string]*answered{}}
return c, nil
}
// Module is what this connection is.
func (c *Conn) Module() string { return c.self }
// Node is the machine this connection's account is scoped to.
func (c *Conn) Node() string { return c.node }
// Credential is what this connection was opened with.
func (c *Conn) Credential() Credential { return c.cred }
// Follow reads a module's membership on this machine once and follows it live, so its tools are
// served where the mesh issued them (ADR 0175). It is read from the grants part's state memberships, its key
// `<node>.<module>` (novox/hq ADR 0571 clause 7 and 10, mesh-sdk membership.GrantsBucket), and nowhere else:
// the old assignments stream is not read and its subject is not heard.
func (c *Conn) Follow(module string) {
c.mu.Lock()
if c.node == "" {
c.mu.Unlock()
return
}
if _, has := c.issued[module]; has {
c.mu.Unlock()
return
}
c.issued[module] = nil
c.mu.Unlock()
c.followGrants(module)
if c.Membership(module) == nil {
c.Logf("[mesh-tools] no membership issued for %s on %s yet; serving the derived shape until one arrives", module, c.node)
}
}
// GrantsPatience bounds the first read of a membership from the grants part's state.
var GrantsPatience = 2 * time.Second
// GrantsRetry is the first wait before the memberships state is read again after it could not be; it doubles up to
// GrantsRetryMost.
var (
GrantsRetry = time.Second
GrantsRetryMost = 30 * time.Second
)
// issue applies a membership that arrived for module, live, and re-serves on it.
func (c *Conn) issue(module string, m Membership) {
c.mu.Lock()
c.issued[module] = &m
handlers := make([]func(Membership), 0, len(c.onNew))
for _, h := range c.onNew {
handlers = append(handlers, *h)
}
c.mu.Unlock()
c.Logf("[mesh-tools] %s on %s was issued a new membership; re-serving on it", module, c.node)
for _, h := range handlers {
h(m)
}
_ = c.nc.Flush()
}
// followGrants reads module's key of the grants part's state memberships and watches it: the value there now is applied
// before Follow returns, each change after it live. A state that is not there yet, or not readable (a bus answering 503
// right after its boot, a timeout), is said once and read again, with a growing wait, until it can be or the
// connection closes: Follow runs once per module, so nothing else would ever read it again.
func (c *Conn) followGrants(module string) {
if c.watchGrants(module, true) {
return
}
go func() {
wait := GrantsRetry
for !c.nc.IsClosed() {
time.Sleep(wait)
if c.nc.IsClosed() {
return
}
if c.watchGrants(module, false) {
c.Logf("[mesh-tools] the grants part's memberships are read again; %s on %s follows its key", module, c.node)
return
}
if wait *= 2; wait > GrantsRetryMost {
wait = GrantsRetryMost
}
}
}()
}
// watchGrants is one attempt of followGrants: true once the key is watched. A failure is said when say is true.
func (c *Conn) watchGrants(module string, say bool) bool {
logf := func(format string, a ...any) {
if say {
c.Logf(format, a...)
}
}
key := membership.GrantsKey(c.node, module)
// First the key's direct get, which the account is granted alone: a bus that refuses it, or a state not there yet,
// answers at once, where binding the bucket would wait out its bound on a refusal.
bucket := membership.GrantsBucket
probe, err := c.nc.Request("$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+key, nil, GrantsPatience)
if err != nil || probe.Header.Get("Status") == "503" {
why := "it did not answer"
if err != nil {
why = err.Error()
}
logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%s); %s has no membership "+
"until it can be", bucket, why, module)
return false
}
js, err := jetstream.New(c.nc)
if err != nil {
return false
}
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
defer cancel()
kv, err := js.KeyValue(ctx, bucket)
if err != nil {
logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%v); %s has no membership "+
"until it can be", membership.GrantsBucket, err, module)
return false
}
w, err := kv.Watch(context.Background(), key)
if err != nil {
logf("[mesh-tools] %s's key %s of the grants part's memberships cannot be watched (%v); %s has no membership "+
"until it can be", module, key, err, module)
return false
}
apply := func(e jetstream.KeyValueEntry) {
if e.Operation() != jetstream.KeyValuePut {
return // a membership withdrawn is the stream's, or the module's unassignment's, to say
}
// Read through mesh-sdk's one codec of this state, the grants part's writer's (issue 774): it takes the bare
// membership and a wrapper the part wrote before of one published or unchanged, and refuses a held-back one's
// wrapper or a value naming no node or module, which then leaves the module with no membership.
// A refused value arriving after a good one keeps the last good membership: nothing here clears it.
var m Membership
body, err := membership.BareGrantsMembership(e.Value())
if err == nil {
err = json.Unmarshal(body, &m)
}
if err != nil {
c.Logf("[mesh-tools] the membership at %s is not one: %v", key, err)
return
}
c.issue(module, m)
}
timer := time.NewTimer(StateTimeout)
defer timer.Stop()
for current := true; current; {
select {
case e, ok := <-w.Updates():
if !ok {
return false
}
if e == nil {
current = false
continue
}
apply(e)
case <-timer.C:
_ = w.Stop()
logf("[mesh-tools] %s's key %s of the grants part's memberships did not answer in %s; %s has no membership "+
"until it can be", module, key, StateTimeout, module)
return false
}
}
go func() {
for e := range w.Updates() {
if e != nil {
apply(e)
}
}
}()
return true
}
// Membership is what the mesh issued a module here, or nil when nothing has been issued.
func (c *Conn) Membership(module string) *Membership {
c.mu.Lock()
defer c.mu.Unlock()
return c.issued[module]
}
// Following says whether this connection follows a module's membership.
func (c *Conn) Following(module string) bool {
c.mu.Lock()
defer c.mu.Unlock()
_, has := c.issued[module]
return has
}
// Serving is every module whose membership this connection follows.
func (c *Conn) Serving() []string {
c.mu.Lock()
defer c.mu.Unlock()
out := make([]string, 0, len(c.issued))
for m := range c.issued {
out = append(out, m)
}
return out
}
// OnMembership is called with every new membership any followed module is issued.
func (c *Conn) OnMembership(h func(Membership)) {
c.mu.Lock()
c.onNew = append(c.onNew, &h)
c.mu.Unlock()
}
// onMembership is OnMembership until the answered function is called, which drops the handler.
func (c *Conn) onMembership(h func(Membership)) func() {
mine := &h
c.mu.Lock()
c.onNew = append(c.onNew, mine)
c.mu.Unlock()
return func() {
c.mu.Lock()
defer c.mu.Unlock()
for i, had := range c.onNew {
if had == mine {
c.onNew = append(c.onNew[:i:i], c.onNew[i+1:]...)
return
}
}
}
}
func (c *Conn) track(s *nats.Subscription) {
c.mu.Lock()
c.subs = append(c.subs, s)
c.mu.Unlock()
}
// servedOn is where a served module's tool is answered: its membership's subjects when issued, the
// derived shape otherwise (the shape the mesh issues on day one).
func (c *Conn) servedOn(module, tool string) []Served {
if m := c.Membership(module); m != nil {
out := make([]Served, 0, len(m.Serves)+1)
for _, s := range m.Serves {
out = append(out, Served{Subject: strings.ReplaceAll(s.Subject, "{tool}", tool), Queue: s.Queue})
}
if tool == "tools" && m.Tools != "" {
found := false
for _, s := range out {
found = found || s.Subject == m.Tools
}
if !found {
out = append([]Served{{Subject: m.Tools, Queue: "serve." + module}}, out...)
}
}
return out
}
base := "mesh.mod." + module + ".tool." + tool
out := []Served{{Subject: base, Queue: "serve." + module}}
if c.node != "" {
out = append(out, Served{Subject: base + "." + c.node})
}
return out
}
type reply struct {
Result any `json:"result,omitempty"`
Error string `json:"error,omitempty"`
Node string `json:"node,omitempty"`
}
// answerOn answers one subject with one handler, and says which machine answered (ADR 0159).
func (c *Conn) answerOn(subject, queue string, h Handler) (func(), error) {
return c.answerCalledOn(subject, queue, func(_ string, body json.RawMessage) (any, error) { return h(body) })
}
// answerCalledOn answers one subject, handing the handler the caller the subject a call arrived on names:
// under a pattern calledPattern made, its last token; on any other subject, "" — none was proven (novox/hq
// issue 365). Never from the body: a body is the caller's to write.
func (c *Conn) answerCalledOn(subject, queue string, h CalledHandler) (func(), error) {
cb := func(msg *nats.Msg) {
go func() {
var r reply
result, err := h(callerOn(subject, msg.Subject), json.RawMessage(msg.Data))
if err != nil {
r.Error = err.Error()
} else {
r.Result = nullable(result)
}
r.Node = c.node
body, _ := wire.Marshal(r)
// Within what the bus carries, or said to be too large — never a send the library
// refuses and the caller waits out (novox/hq issue 314).
if err := msg.Respond(c.fits(subject, body)); err != nil {
c.Logf("[mesh-tools] could not answer on %s: %v", subject, err)
}
}()
}
var sub *nats.Subscription
var err error
if queue != "" {
sub, err = c.nc.QueueSubscribe(subject, queue, cb)
} else {
sub, err = c.nc.Subscribe(subject, cb)
}
if err != nil {
return func() {}, err
}
c.track(sub)
a := &answered{subject: subject, queue: queue, cb: cb, sub: sub}
c.mu.Lock()
c.answering[answeringKey(subject, queue)] = a
c.mu.Unlock()
return func() {
c.mu.Lock()
a.stopped = true
if c.answering[answeringKey(subject, queue)] == a {
delete(c.answering, answeringKey(subject, queue))
}
current := a.sub
c.mu.Unlock()
_ = current.Unsubscribe()
}, nil
}
// nullable keeps a nil result as JSON null rather than dropping the key: the TypeScript reply always
// carries `result` when the handler did not throw.
func nullable(v any) any {
if v == nil {
return json.RawMessage("null")
}
return v
}
// HandleSubject answers a subject outright: a seat's verb where the mesh issued it.
func (c *Conn) HandleSubject(subject string, h Handler) (func(), error) {
return c.HandleSubjectCalled(subject, func(_ string, body json.RawMessage) (any, error) { return h(body) })
}
// HandleSubjectCalled answers a tool subject and its caller-named counterpart (novox/hq issue 365): a call on
// the counterpart is handed the caller its subject names, one on the subject itself "" — none proven. The
// subject that names no caller is served beside the other for one release, so callers move without a gap
// (its retirement: hq issue 464).
func (c *Conn) HandleSubjectCalled(subject string, h CalledHandler) (func(), error) {
return c.answerBoth(subject, "", h)
}
// HandleSubjectCalledQueue is HandleSubjectCalled in one queue group: of every holder that serves the subject in the
// group, one answers (lead answer A-393: a node-scoped seat's verb asked with no machine).
func (c *Conn) HandleSubjectCalledQueue(subject, queue string, h CalledHandler) (func(), error) {
return c.answerBoth(subject, queue, h)
}
// answerBoth answers a subject and, when it is a tool's, the caller-named pattern beside it, in one queue.
func (c *Conn) answerBoth(subject, queue string, h CalledHandler) (func(), error) {
stop, err := c.answerCalledOn(subject, queue, h)
if err != nil {
return func() {}, err
}
pattern := calledPattern(subject)
if pattern == "" {
return stop, nil
}
stopCalled, err := c.answerCalledOn(pattern, queue, h)
if err != nil {
c.Logf("[mesh-tools] cannot serve the caller-named %s: %v; its callers are asked on %s alone", pattern, err, subject)
return stop, nil
}
return func() { stop(); stopCalled() }, nil
}
// Handle serves `<module>.<tool>` where the mesh issued that module, and follows its membership:
// when a new one arrives, it serves where it now says and stops where it no longer does.
func (c *Conn) Handle(key string, h Handler) (func(), error) {
return c.HandleCalled(key, func(_ string, body json.RawMessage) (any, error) { return h(body) })
}
// HandleCalled is Handle telling the handler who called: the bus user a caller-named subject names, or "" for
// a call on the subject that names none (novox/hq issue 365).
func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) {
if strings.HasPrefix(key, "seat:") {
subject, err := ToolSubject(key, c.self)
if err != nil {
return func() {}, err
}
return c.answerBoth(subject, "", h)
}
module, tool := c.self, key
if dot := strings.Index(key, "."); dot >= 0 {
module, tool = key[:dot], key[dot+1:]
}
if module != c.self && !c.Following(module) {
return func() {}, fmt.Errorf("%s cannot serve %s: a module serves its own tools, and a runtime "+
"those of the modules it follows", c.self, key)
}
var mu sync.Mutex
stops := map[Served]func(){}
stopped := false
// **Make before break** (novox/hq design 47 H, H-A-158): a membership issued again keeps every subscription
// whose subject and queue it still names, subscribes those it newly names, and only then stops those it no
// longer names — a call the bus routes while the membership is followed is never dropped in silence.
serve := func() {
mu.Lock()
defer mu.Unlock()
if stopped {
return
}
named := map[Served]bool{}
for _, s := range c.servedOn(module, tool) {
named[s] = true
if _, kept := stops[s]; kept {
continue
}
if stop, err := c.answerBoth(s.Subject, s.Queue, h); err == nil {
stops[s] = stop
} else {
c.Logf("[mesh-tools] cannot serve %s on %s: %v", key, s.Subject, err)
}
}
for s, stop := range stops {
if !named[s] {
stop()
delete(stops, s)
}
}
}
serve()
// Its stop drops the membership handler too: a handler left behind would bind the stopped tool again
// on the module's next membership, and call a handler whose bundle was replaced (issue 1945).
unfollow := c.onMembership(func(m Membership) {
if m.Module == module {
serve()
}
})
return func() {
unfollow()
mu.Lock()
defer mu.Unlock()
stopped = true
for s, stop := range stops {
stop()
delete(stops, s)
}
}, nil
}
// reachedAt is where a call by key goes: a subject this connection's own membership says it
// reaches — the machine's when named — else the derived shape.
func (c *Conn) reachedAt(key string) (string, error) {
name, wanted, _ := strings.Cut(key, "@")
if m := c.Membership(c.self); m != nil {
if reach := m.Reaches[name]; len(reach) > 0 {
if wanted == "" {
return reach[0], nil
}
for _, s := range reach {
if strings.HasSuffix(s, "."+wanted) {
return s, nil
}
}
}
}
return ToolSubject(key, c.self)
}
// Ask calls a tool by key — or on a subject the mesh listed for it — and learns which machine
// answered. Core request/reply: a tool call is never persisted (design 25 §3).
func (c *Conn) Ask(key string, body any, on string) (Answered, error) {
return c.AskWithin(key, body, on, 0)
}
// AskWithin is Ask that waits within for its answer, the `timeout` its verb's definition names (design 47 H §760, H-A-43);
// zero waits the tool runner's own bound, RequestTimeout.
func (c *Conn) AskWithin(key string, body any, on string, within time.Duration) (Answered, error) {
subject := on
if subject == "" {
s, err := c.reachedAt(key)
if err != nil {
return Answered{}, err
}
subject = s
}
data, err := wire.Marshal(body)
if err != nil {
return Answered{}, err
}
// **Named by its caller first** (novox/hq issue 365): on the caller-named subject, which the bus lets this
// account publish with its own name alone; asked again on the subject that names no caller only when the
// first reached nobody — nothing serves it yet, or the bus does not grant it yet — so no call is done twice.
if called := CalledSubject(subject, c.cred.User); called != "" {
answered, fallBack, err := c.askCalled(called, data, within)
if !fallBack {
return answered, err
}
}
return c.askOnce(subject, data, within)
}
// ErrNoResponders is the bus's own refusal: nothing subscribes to the subject asked. It is told apart
// from a tool's own error by errors.Is, never by its text (a tool may say "503" itself, issue 1529).
var ErrNoResponders = errors.New("503 no responders")
// askOnce asks once.
func (c *Conn) askOnce(subject string, data []byte, within time.Duration) (Answered, error) {
msg, err := c.nc.Request(subject, data, waitOf(within))
if err != nil {
if errors.Is(err, nats.ErrNoResponders) {
return Answered{}, ErrNoResponders
}
if errors.Is(err, nats.ErrTimeout) {
return Answered{}, errors.New("timeout")
}
return Answered{}, err
}
return answerOf(msg.Data)
}
// answerOf reads one answer.
func answerOf(data []byte) (Answered, error) {
var r struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
Node string `json:"node"`
}
if err := json.Unmarshal(data, &r); err != nil {
return Answered{}, err
}
if r.Error != "" {
return Answered{}, errors.New(r.Error)
}
return Answered{Result: r.Result, Node: r.Node}, nil
}
// PublishAs emits an event as a module: published into JetStream and awaited, de-duplicated by its
// own id (ADR 0042). The body is the payload; the metadata rides as headers.
func (c *Conn) PublishAs(module string, env Envelope) error {
_, err := c.PublishAsAcked(module, env)
return err
}
// PublishAsAcked is PublishAs that also answers the sequence the bus acknowledged the event at.
func (c *Conn) PublishAsAcked(module string, env Envelope) (uint64, error) {
// On the module's own stream when its membership names one carrying the event (ADR 0363 §2), on EVENTS
// otherwise — and only there, as before, for a membership that names no stream.
msg := nats.NewMsg(eventSubject(c.Membership(module), module, env.Key))
for k, v := range env.Headers {
msg.Header.Set(k, v)
}
if env.Headers["content-type"] == "" {
msg.Header.Set("content-type", "application/json")
}
if env.Node != "" {
msg.Header.Set("x-node", env.Node)
}
body := env.Body
if len(body) == 0 {
body = json.RawMessage("null")
}
msg.Data = body
var opts []nats.PubOpt
if id := env.Headers["x-event-id"]; id != "" {
opts = append(opts, nats.MsgId(id))
}
ack, err := c.js.PublishMsg(msg, opts...)
if err != nil {
return 0, err
}
return ack.Sequence, nil
}
// Say publishes one message on core NATS, kept by nobody: a heartbeat, whose loss is the next one.
func (c *Conn) Say(subject string, body []byte) error { return c.nc.Publish(subject, body) }
// ServedOn is where a served module's tool is answered right now: the membership's subjects when
// issued, the derived shape otherwise — what Handle subscribes, for what announces it (ADR 0197).
func (c *Conn) ServedOn(module, tool string) []Served { return c.servedOn(module, tool) }
// Raw answers one subject with a function of the request, not a tool's reply envelope: the NATS
// services protocol's discovery subjects answer in their own format (novox/hq ADR 0197). A nil
// answer is no reply — the request was for another service.
func (c *Conn) Raw(subject string, answer func(subject string, data []byte) []byte) (func(), error) {
sub, err := c.nc.Subscribe(subject, func(msg *nats.Msg) {
if body := answer(msg.Subject, msg.Data); body != nil {
// Said, never dropped: an answer the bus will not carry was silence before (2026-10-04).
if err := msg.Respond(body); err != nil {
c.Logf("[mesh-tools] could not answer %s (%d bytes): %v", msg.Subject, len(body), err)
}
}
})
if err != nil {
return func() {}, err
}
c.track(sub)
return func() { _ = sub.Unsubscribe() }, nil
}
// Arrival is one answer to a request, and when it arrived.
type Arrival struct {
Data []byte
At time.Time
}
// Collect publishes one request and hands every answer to the channel as it arrives, until stopped:
// for a caller that decides for itself when it has heard enough — discovery waits for every service
// that said it is there, not for a fixed moment (2026-10-05). Answers beyond what the caller has
// taken are kept up to the channel's room; an empty answer is no answer.
func (c *Conn) Collect(subject string, body []byte) (<-chan Arrival, func(), error) {
out, ask, stop, err := c.Collector(subject)
if err != nil {
return nil, func() {}, err
}
if err := ask(subject, body); err != nil {
stop()
return nil, func() {}, err
}
return out, stop, nil
}
// Collector is one inbox for many requests: every answer to every request `ask` publishes arrives on
// the one channel, until stopped — for discovery, which asks each service that said it is there for
// what it serves, as each says so (novox/hq issue 554). `what` names the requests in a log line.
func (c *Conn) Collector(what string) (<-chan Arrival, func(subject string, body []byte) error, func(), error) {
inbox := c.nc.NewRespInbox()
out := make(chan Arrival, 1024)
var mu sync.Mutex
stopped := false
sub, err := c.nc.Subscribe(inbox, func(msg *nats.Msg) {
if len(msg.Data) == 0 {
return
}
a := Arrival{Data: msg.Data, At: time.Now()}
mu.Lock()
defer mu.Unlock()
if stopped {
return
}
select {
case out <- a:
default:
c.Logf("[mesh-tools] more answers to %s than are kept; one (%d bytes) dropped", what, len(msg.Data))
}
})
if err != nil {
return nil, nil, func() {}, err
}
stop := func() {
_ = sub.Unsubscribe()
mu.Lock()
stopped = true
mu.Unlock()
}
ask := func(subject string, body []byte) error { return c.nc.PublishRequest(subject, inbox, body) }
return out, ask, stop, nil
}
// MaxPayload is the largest message the bus carries, as the server told this connection.
func (c *Conn) MaxPayload() int64 { return c.nc.MaxPayload() }
// Flush waits until the bus has every subscription made so far, so what is served is answerable
// when this returns.
func (c *Conn) Flush() { _ = c.nc.Flush() }
// Close unsubscribes everything and drains, so an in-flight reply is finished rather than dropped.
func (c *Conn) Close() {
c.mu.Lock()
subs := c.subs
c.subs = nil
c.mu.Unlock()
for _, s := range subs {
_ = s.Unsubscribe()
}
_ = c.nc.Drain()
}
// ToolSubject is a tool's subject. A bare name is this module's own; `<module>.<tool>` another's;
// `seat:<seat>.<verb>` a role's, with `@<node>` for a node-scoped seat (design 33 §4).
func ToolSubject(key, self string) (string, error) {
if strings.HasPrefix(key, "seat:") {
rest := strings.TrimPrefix(key, "seat:")
dot := strings.Index(rest, ".")
if dot < 0 {
return "", fmt.Errorf("%q names a seat and no verb: seat:<seat>.<verb>", key)
}
seat := rest[:dot]
verb, node, _ := strings.Cut(rest[dot+1:], "@")
if node != "" {
return "mesh.seat." + seat + ".tool." + verb + "." + node, nil
}
return "mesh.seat." + seat + ".tool." + verb, nil
}
name, node, _ := strings.Cut(key, "@")
var base string
if dot := strings.Index(name, "."); dot < 0 {
base = "mesh.mod." + self + ".tool." + name
} else {
base = "mesh.mod." + name[:dot] + ".tool." + name[dot+1:]
}
if node != "" {
return base + "." + node, nil
}
return base, nil
}
// SeatToolSubject is a seat's verb as its holder serves it: flat for a mesh seat, carrying the
// machine for a node-scoped one.
func SeatToolSubject(seat, verb, scope, node string) string {
base := "mesh.seat." + seat + ".tool." + verb
if scope == "node" && node != "" {
return base + "." + node
}
return base
}
// AskAs calls a tool on a module's behalf (novox/hq ADR 0198): a bare key is that module's own tool,
// `<module>.<tool>` another's, `seat:<seat>.<verb>[@<node>]` a role's — resolved through what the
// mesh issued that module to reach, as its own runtime resolved it, else the derived shape.
func (c *Conn) AskAs(module, key string, body any) (Answered, error) {
return c.AskAsWithin(module, key, body, 0)
}
// AskAsWithin is AskAs that waits within for its answer (design 47 H §760); zero waits RequestTimeout.
func (c *Conn) AskAsWithin(module, key string, body any, within time.Duration) (Answered, error) {
name, wanted, _ := strings.Cut(key, "@")
subject := ""
if m := c.Membership(module); m != nil {
if reach := m.Reaches[name]; len(reach) > 0 {
subject = reach[0]
if wanted != "" {
subject = ""
for _, s := range reach {
if strings.HasSuffix(s, "."+wanted) {
subject = s
}
}
}
}
}
if subject == "" {
s, err := ToolSubject(key, module)
if err != nil {
return Answered{}, err
}
subject = s
}
return c.AskWithin(key, body, subject, within)
}
// EventsStream is where every module's events land, and ConsumerOf the durable consumer the
// mesh makes for a module on a machine: the same names the module's own runtime bound
// (the retired TypeScript runtime's broker-nats.ts), so moving the module into the node's runtime neither loses an
// event nor sees one twice.
const EventsStream = "EVENTS"
// ConsumerOf is a module's durable consumer on a machine: `<node>_<module>`.
func ConsumerOf(node, module string) string {
if node == "" {
node = "?"
}
return node + "_" + module
}
// ToolsSuffix ends the name of the tool runner's own durable consumer of a module: the runner and the module's program each
// need every event, so each reads under a name of its own (lead answer A-397); only a work queue shares one name.
const ToolsSuffix = "_tools"
// ToolsConsumerOf is the tool runner's durable consumer of a module on a machine: `<node>_<module>_tools`.
func ToolsConsumerOf(node, module string) string { return ConsumerOf(node, module) + ToolsSuffix }
// NakDelay is how long an event a handler failed waits before it is offered again: a transient cause
// gets another attempt, a permanent one exhausts the consumer's max-deliver rather than spinning. The
// consumer then gives the event up, and the mesh keeps it in DEAD_LETTERS until a person delivers
// it again — under `mesh.again.<consumer>.…`, from which keyOf reads the same key — or drops it
// (novox/hq issue 330).
var NakDelay = 5 * time.Second
// EventHeartbeat is how often an event a bundle is still handling is said to be in progress, well inside the
// consumer's ack wait. The client holds one event at a time (PullMaxMessages(1)): events are handed over in
// order, one at a time, so an event waiting in the client's buffer would have its ack wait run out before it is
// handed over, and a burst would end in max-deliver with nothing failed (novox/hq issue 387).
var EventHeartbeat = 10 * time.Second
// ConsumeAs reads a module's durable consumer on EVENTS, and each one on a module's stream its
// membership names (novox/hq ADR 0363 §8), and hands each event to deliver one at a time (novox/hq
// ADR 0198): acknowledged when deliver returns nil, negatively acknowledged after NakDelay when it
// returns an error, terminated when it is not an event at all. The consumers are the mesh's to
// create; this binds to them and never makes one. Runs until stopped.
func (c *Conn) ConsumeAs(module string, deliver func(Envelope) error) (func(), error) {
return c.ConsumeAsBeating(module, deliver, nil)
}
// IdleHeartbeat is how often the runtime tells a bundle that its module's consumer is bound and the bus answers,
// while no event arrives (stdio.OnHeartbeat), so a subscription's silence is told from a quiet source.
var IdleHeartbeat = 15 * time.Second
// Beat is the runtime's word on a module's consumer between events: it is bound and the bus answers (At), or the
// bus's heartbeat stopped arriving or the bus ended the loop (Missed, with Why). Its JSON is the stdio Heartbeat.
type Beat struct {
At time.Time `json:"at"`
Missed bool `json:"missed,omitempty"`
Why string `json:"why,omitempty"`
}
// ConsumeAsBeating is ConsumeAs that also says, to beat, how the module's consumer on EVENTS stands: a Beat at each
// IdleHeartbeat in which no event arrived and the bus answered about the consumer, and one Missed (said once) when
// the bus's own heartbeat to the loop stopped arriving, the bus ended the loop or no longer answers. A Beat that is
// not Missed follows the first answer after one that was. A nil beat says nothing.
func (c *Conn) ConsumeAsBeating(module string, deliver func(Envelope) error, beat func(Beat)) (func(), error) {
return c.ConsumeAsBeatingWhen(module, deliver, beat, nil)
}
// tickHook, when set before a consume starts, is called after each idle heartbeat interval was handled (tests only,
// through SetTickHook), so a test counts beats instead of sleeping. A consume reads it once, when it starts.
var tickHook func()
// ConsumeAsBeatingWhen is ConsumeAsBeating that asks the bus about the consumer, and says anything, only while wanted
// answers true (nil is always): a module whose children asked for no heartbeat costs the bus no consumer-info request.
func (c *Conn) ConsumeAsBeatingWhen(module string, deliver func(Envelope) error, beat func(Beat), wanted func() bool) (func(), error) {
asked := func() bool { return beat != nil && (wanted == nil || wanted()) }
durable := ToolsConsumerOf(c.node, module)
js, err := jetstream.New(c.nc)
if err != nil {
return nil, err
}
var lastEvent atomic.Int64
var missed atomic.Bool
say := func(why string) {
if asked() && missed.CompareAndSwap(false, true) {
beat(Beat{At: time.Now().UTC(), Missed: true, Why: why})
}
}
h := newHearing(c, js, module, func(env Envelope) error {
lastEvent.Store(time.Now().UnixNano())
missed.Store(false) // an event arrived: the consumer is heard, whatever was said before
return deliver(env)
})
// Bound by name, never created and never matched against a subject: the consumer and its
// filters are the mesh's (design 29 §3), exactly as the module's own runtime bound it.
events, _, doing, err := h.bind(EventsStream, durable, func(err error) {
if errors.Is(err, jetstream.ErrNoHeartbeat) {
say("the bus's heartbeat to the consumer " + durable + " stopped arriving")
}
})
if err != nil {
return nil, fmt.Errorf("%s %s's consumer %s: %w", doing, module, durable, err)
}
// And every consumer on a module's stream its membership names (novox/hq ADR 0363 §8), followed as the
// membership changes. A membership naming none binds nothing more: the module hears EVENTS alone.
h.start()
if beat != nil {
every := IdleHeartbeat
hook := tickHook
tickOnce := func() {
select {
case <-events.Closed():
say("the bus ended reading the consumer " + durable)
return
default:
}
if !asked() {
return
}
ctx, cancel := context.WithTimeout(context.Background(), every)
_, err := js.Consumer(ctx, EventsStream, durable)
cancel()
if err != nil {
say("the bus does not answer about the consumer " + durable + ": " + err.Error())
return
}
if time.Since(time.Unix(0, lastEvent.Load())) < every {
return // an event arrived in this interval: it was the word
}
missed.Store(false)
beat(Beat{At: time.Now().UTC()})
}
go func() {
tick := time.NewTicker(every)
defer tick.Stop()
for {
select {
case <-h.done:
return
case <-tick.C:
}
tickOnce()
if hook != nil {
hook()
}
}
}()
}
var once sync.Once
return func() {
once.Do(func() {
h.stop()
events.Stop()
})
}, nil
}
// envelopeOf rebuilds the envelope a module sees from a delivered event: its key is the emitter and
// the event, recovered from the subject; its metadata rides as headers (novox/hq ADR 0042).
func envelopeOf(subject string, header nats.Header, data []byte) (Envelope, bool) {
if !json.Valid(data) {
return Envelope{}, false
}
headers := map[string]string{}
for k := range header {
headers[k] = header.Get(k)
}
return Envelope{Key: keyOf(subject), Node: headers["x-node"], Body: json.RawMessage(data), Headers: headers}, true
}
// keyOf is the key a module sees for an event subject: `mesh.mod.<emitter>.event.<event>` is
// `<emitter>.<event>` — the vocabulary its manifest names what it consumes in.
func keyOf(subject string) string {
before, event, found := strings.Cut(subject, ".event.")
if !found {
return subject
}
parts := strings.Split(before, ".")
if emitter := parts[len(parts)-1]; emitter != "" {
return emitter + "." + event
}
return event
}
// answerRefused says a refused publish of an answer (to a caller's inbox: a permission, or the single-use
// allowance of a reply expired) once per subject per minute, never silently (issue 1966: a caller waited out its
// time for an answer the bus had refused). It reports whether the refusal was one.
func (c *Conn) answerRefused(err error) bool {
m := refusedPublish.FindStringSubmatch(err.Error())
if len(m) < 2 || !strings.HasPrefix(strings.Trim(m[1], `"`), "_INBOX.") {
return false
}
// Keyed without the reply's last token (a new one per request), so a caller's burst is one line.
subject := strings.Trim(m[1], `"`)
if dot := strings.LastIndex(subject, "."); dot > len("_INBOX.") {
subject = subject[:dot]
}
c.mu.Lock()
if c.inboxSaid == nil {
c.inboxSaid = map[string]time.Time{}
}
now := time.Now()
for k, at := range c.inboxSaid {
if now.Sub(at) >= time.Minute {
delete(c.inboxSaid, k)
}
}
_, seen := c.inboxSaid[subject]
say := !seen
if say {
c.inboxSaid[subject] = now
}
c.mu.Unlock()
if say && c.Logf != nil {
c.Logf("[mesh-tools] the bus refused this account an answer to %s (%v); the caller gets none", subject, err)
}
return true
}