Files
mesh-sdk/go/process/call.go
T
jschoubben 4e338436da Kept-call pace read once and atomically: no data race with the kept calls reading it
ReadPace wrote the pace at every Connect and Serve while kept calls on other
goroutines read it. The pace is now held atomically and read once at the
start: a later read of the same pace changes nothing, one of another pace is
refused naming both. KeptPace becomes a function.

Builds: design 47 H2 §782, §775
2026-11-22 02:21:53 +01:00

291 lines
12 KiB
Go

package process
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/approval"
"git.novox.be/novox/mesh-sdk/go/busretry"
"git.novox.be/novox/mesh-sdk/go/calladdress"
)
// CallTimeout bounds a call to a tool or a seat verb (mesh-tools node-tools/internal/bus/bus.go:35,
// RequestTimeout): what a module already expects.
var CallTimeout = 30 * time.Second
// Call invokes a tool or a seat verb as this process's own account (novox/hq ADR 0436 clause 4), on the
// caller-named subject, falling back to the subject that names no caller when nothing answers it —
// exactly the provisioner client's own pattern (mesh-catalog modules/mesh-provisioner-client/cmd/
// mesh-provisioner-client/secrets.go:59-61, 93-97, 119, 125-133). key is "seat:<seat>.<verb>" for a seat's
// verb, or "<module>.<tool>" for a module's own tool, as a process's bus.invokes names it. The reply is
// {result, error, node} (mesh-tools node-tools/internal/bus/bus.go answerOf); a non-empty error is the
// call's failure. A process answers calls through AnswerSeat (serve.go). A key naming its machine after '@' (`seat:node-service-manager.stop@novox`) is CallOn of that
// machine; any other malformed key is refused before anything is published (calladdress; novox/hq issue 774).
func (c *Conn) Call(key string, body any) (json.RawMessage, error) {
called, fallback, err := callSubjectsOn(key, "", c.callerToken())
if err != nil {
return nil, err
}
return c.callOn(called, fallback, body)
}
// CallOn is Call to the tool or seat verb of one named machine (novox/hq ADR 0469 §3; issue 1470): the tool
// runner of that node alone answers, on mesh.mod.<module>.call.<tool>.<node>.<token> (a seat verb:
// mesh.seat.<seat>.call.<verb>.<node>.<token>), falling back to the subject that names no caller
// (mesh-tools node-tools/internal/bus/caller.go:15-19). node is one subject token: an empty one, or one
// holding '.', '*', '>' or white space, is refused before anything is published.
func (c *Conn) CallOn(node, key string, body any) (json.RawMessage, error) {
if err := calladdress.CheckNode(node); err != nil {
return nil, err
}
called, fallback, err := callSubjectsOn(key, node, c.callerToken())
if err != nil {
return nil, err
}
return c.callOn(called, fallback, body)
}
// CallKept is Call for a call whose answer the caller keeps and which is safe to make again (a read, or a write the
// callee deduplicates): mesh-sdk's kept-call form (design 47 H2 §775), asked again through a transport fault, a timeout
// or no responders at the kept-call pace (busretry.KeptPace(), jittered), whether or not the bus is blacked out, until it
// is answered or busretry.Bound has passed (novox/hq ADR 0668, extending ADR 0615 §6; issue 1966). A refusal returns at
// once. A caller with an ask id and a bound of its own uses CallKeptAsk.
func (c *Conn) CallKept(key string, body any) (json.RawMessage, error) {
return c.CallKeptAsk(KeptAsk{}, key, body)
}
// CallKeptOn is CallKept to the tool or seat verb of one named machine, as CallOn is Call's.
func (c *Conn) CallKeptOn(node, key string, body any) (json.RawMessage, error) {
return c.CallKeptAskOn(KeptAsk{}, node, key, body)
}
// KeptAsk is what a caller names of its kept call (design 47 H2 §775): the ask's id, which the call carries in the
// header stdio.AskHeader for the holder to take it once by (H-A-351), and the caller's bound (zero is busretry.Bound).
type KeptAsk struct {
ID string
Bound time.Duration
}
// CallKeptAsk is CallKept carrying ask's id and stopping at ask's bound: past it the error is a *busretry.Unanswered
// naming every call's time and words.
func (c *Conn) CallKeptAsk(ask KeptAsk, key string, body any) (json.RawMessage, error) {
return c.callKeptAskOn(ask, "", key, body, nil)
}
// CallKeptAskOn is CallKeptAsk to the tool or seat verb of one named machine.
func (c *Conn) CallKeptAskOn(ask KeptAsk, node, key string, body any) (json.RawMessage, error) {
if err := calladdress.CheckNode(node); err != nil {
return nil, err
}
return c.callKeptAskOn(ask, node, key, body, nil)
}
// CallKeptWhileOpen is CallKept for an ask the asker can close or replace: it is asked again only while open says its
// key is still open (design 47 G §351, §460).
func (c *Conn) CallKeptWhileOpen(open busretry.Open, key string, body any) (json.RawMessage, error) {
if open.Open == nil {
return nil, fmt.Errorf("the call on %s: a kept call names its key's open function", key)
}
return c.callKeptAskOn(KeptAsk{}, "", key, body, &open)
}
func (c *Conn) callKeptAskOn(ask KeptAsk, node, key string, body any, open *busretry.Open) (json.RawMessage, error) {
called, fallback, err := callSubjectsOn(key, node, c.callerToken())
if err != nil {
return nil, err
}
data, err := json.Marshal(body)
if err != nil {
return nil, err
}
return callKeptWith(keptOf(key, ask, open), c.requestAsking(ask.ID), called, fallback, data)
}
// keptOf is the paced form's naming of a call on key: its seat and verb, or its module and tool.
func keptOf(key string, ask KeptAsk, open *busretry.Open) busretry.Kept {
k := busretry.Kept{Ask: ask.ID, Bound: ask.Bound, Open: open}
if a, err := calladdress.Parse(key); err == nil {
k.Seat, k.Verb = a.Seat, a.Verb
if a.Seat == "" {
k.Seat, k.Verb = a.Module, a.Tool
}
}
return k
}
// callKeptWith is callWith for a kept call: busretry.DoPaced with no blackout probe.
func callKeptWith(k busretry.Kept, request func(string, []byte) (json.RawMessage, error), called, fallback string,
data []byte) (json.RawMessage, error) {
ask := askBoth(request, called, fallback, data)
return busretry.DoPaced(k, func() (json.RawMessage, error) {
result, err := ask()
if err == nil {
// A holder's answer of the kind away is a transport fault (§776): called again at the pace.
if away := busretry.AwayIn(result); away != nil {
return result, away
}
}
return result, err
})
}
// askBoth asks called, then fallback when nothing answers it.
func askBoth(request func(string, []byte) (json.RawMessage, error), called, fallback string,
data []byte) func() (json.RawMessage, error) {
return func() (json.RawMessage, error) {
start := time.Now()
result, err := request(called, data)
if !errors.Is(err, nats.ErrNoResponders) {
return result, unansweredOn(err, start, called)
}
first := err
start2 := time.Now()
result, err = request(fallback, data)
if err != nil && !isAnswered(err) {
return result, fmt.Errorf("asked on %s (%v), then on %s, unanswered after %s: %w", called, first, fallback,
time.Since(start2).Round(time.Millisecond), err)
}
return result, err
}
}
// unansweredOn names the subject a transport fault was met on and how long it waited, so a log of a call nobody
// answered says where it was asked; a callee's answer keeps its own words.
func unansweredOn(err error, start time.Time, subject string) error {
if err == nil || isAnswered(err) {
return err
}
return fmt.Errorf("asked on %s, unanswered after %s: %w", subject, time.Since(start).Round(time.Millisecond), err)
}
func isAnswered(err error) bool {
var a *busretry.Answered
return errors.As(err, &a)
}
// callOn asks called, then fallback when nothing answers it; while the bus itself answers nothing (a user-list reload
// drops the account's imports for a few seconds, busretry) both are asked again, within busretry.Bound. A call nobody
// answers on a bus that answers is not asked again: its callee is not running. A call that may have been delivered (a
// timeout, any answer) is never asked again.
func (c *Conn) callOn(called, fallback string, body any) (json.RawMessage, error) {
data, err := json.Marshal(body)
if err != nil {
return nil, err
}
return callWith(c.request, c.blackedOut, called, fallback, data)
}
// callWith is callOn over a request and a blackout probe. Only the transport's own errors can count as undelivered: a
// callee's answer is *busretry.Answered, whatever its words, so a call that reached its callee is never asked again.
func callWith(request func(string, []byte) (json.RawMessage, error), blackout func() bool, called, fallback string,
data []byte) (json.RawMessage, error) {
return busretry.Do("the call on "+fallback, askBoth(request, called, fallback, data), blackout)
}
// answered is the error a callee answered with: delivered, never asked again (busretry.Answered).
func answered(words string) error { return &busretry.Answered{Words: words} }
// blackedOut says the bus answers this account nothing at all: its JetStream API, which every account imports, has no
// responder or says it is not enabled.
func (c *Conn) blackedOut() bool {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
msg, err := c.nc.RequestWithContext(ctx, "$JS.API.INFO", nil)
if err != nil {
return busretry.Undelivered(err)
}
return strings.Contains(string(msg.Data), `"err_code":10039`) || strings.Contains(string(msg.Data), "jetstream not enabled")
}
// callerToken is this account's user name as the last token of a call it publishes, each dot written ~
// (mesh-tools node-tools/internal/bus/caller.go:43, CallerToken).
func (c *Conn) callerToken() string { return strings.ReplaceAll(c.account.User, ".", "~") }
// callSubjects is the caller-named subject key is called on, and its plain fallback (mesh-tools
// node-tools/internal/bus/caller.go CalledSubject, toolShape, calledPattern).
func callSubjects(key, token string) (called, fallback string, err error) {
return callSubjectsOn(key, "", token)
}
// callSubjectsOn is callSubjects for one named node ("" names none, and then the key's own '@' names it): the node is a
// token between the tool and the caller on the called subject, and the last one of the fallback. A key naming another
// machine than node is refused.
func callSubjectsOn(key, node, token string) (called, fallback string, err error) {
a, err := calladdress.Parse(key)
if err != nil {
return "", "", err
}
switch {
case a.Node != "" && node != "" && a.Node != node:
return "", "", fmt.Errorf("%q names the machine %s, and it is asked of %s", key, a.Node, node)
case a.Node != "":
node = a.Node
}
at := ""
if node != "" {
at = "." + node
}
if a.Seat != "" {
return "mesh.seat." + a.Seat + ".call." + a.Verb + at + "." + token, "mesh.seat." + a.Seat + ".tool." + a.Verb + at, nil
}
return "mesh.mod." + a.Module + ".call." + a.Tool + at + "." + token, "mesh.mod." + a.Module + ".tool." + a.Tool + at, nil
}
// request asks one subject once and reads its reply ({result, error, node}); a reply naming an error is
// the call's failure, not a transport one.
func (c *Conn) request(subject string, data []byte) (json.RawMessage, error) {
return c.requestAsk(subject, data, "")
}
// requestAsking is request carrying an ask's id in the header stdio.AskHeader (H-A-351); empty carries none.
func (c *Conn) requestAsking(ask string) func(string, []byte) (json.RawMessage, error) {
return func(subject string, data []byte) (json.RawMessage, error) { return c.requestAsk(subject, data, ask) }
}
// askMsg is the request a call publishes: its ask's id in stdio.AskHeader when it carries one (H-A-351).
func askMsg(subject string, data []byte, ask string) *nats.Msg {
req := nats.NewMsg(subject)
req.Data = data
if ask != "" {
req.Header.Set(stdio.AskHeader, ask)
}
return req
}
func (c *Conn) requestAsk(subject string, data []byte, ask string) (json.RawMessage, error) {
ctx, cancel := context.WithTimeout(context.Background(), CallTimeout)
defer cancel()
req := askMsg(subject, data, ask)
if strings.HasPrefix(subject, "mesh.seat.") {
// Every seat call says it waits for a held call's end; a verb that is not marked ignores it (lead answer A-244).
req.Header.Set(approval.WaitHeader, approval.WaitHeld)
}
msg, err := c.nc.RequestMsgWithContext(ctx, req)
if err != nil {
return nil, err
}
var r struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
Node string `json:"node"`
Held *approval.Held `json:"held"`
}
if err := json.Unmarshal(msg.Data, &r); err != nil {
return nil, fmt.Errorf("%s answered something that is not a seat's answer: %w", subject, err)
}
if r.Error != "" {
return nil, answered(r.Error)
}
if r.Held != nil {
return c.awaitGated(subject, *r.Held)
}
return r.Result, nil
}