The kept call is made again on a transport fault after kept-call-pace-first (1 s) doubling to kept-call-pace-max (30 s), each wait within 20 %, until the caller's bound; past it the answer names every call's time and words. One journal line per seat at the first fault, then at most once a minute. The ask id travels in the header x-ask; a holder reads it with process.AskOf, a bundle with stdio.AskIDOf (TypeScript askIdOf) from the call's _meta mesh/ask. Builds: design 47 H2 §775; H-A-351
281 lines
12 KiB
Go
281 lines
12 KiB
Go
package process
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/approval"
|
|
"git.novox.be/novox/mesh-sdk/go/busretry"
|
|
"git.novox.be/novox/mesh-sdk/go/calladdress"
|
|
)
|
|
|
|
// CallTimeout bounds a call to a tool or a seat verb (mesh-tools node-tools/internal/bus/bus.go:35,
|
|
// RequestTimeout): what a module already expects.
|
|
var CallTimeout = 30 * time.Second
|
|
|
|
// Call invokes a tool or a seat verb as this process's own account (novox/hq ADR 0436 clause 4), on the
|
|
// caller-named subject, falling back to the subject that names no caller when nothing answers it —
|
|
// exactly the provisioner client's own pattern (mesh-catalog modules/mesh-provisioner-client/cmd/
|
|
// mesh-provisioner-client/secrets.go:59-61, 93-97, 119, 125-133). key is "seat:<seat>.<verb>" for a seat's
|
|
// verb, or "<module>.<tool>" for a module's own tool, as a process's bus.invokes names it. The reply is
|
|
// {result, error, node} (mesh-tools node-tools/internal/bus/bus.go answerOf); a non-empty error is the
|
|
// call's failure. A process answers calls through AnswerSeat (serve.go). A key naming its machine after '@' (`seat:node-service-manager.stop@novox`) is CallOn of that
|
|
// machine; any other malformed key is refused before anything is published (calladdress; novox/hq issue 774).
|
|
func (c *Conn) Call(key string, body any) (json.RawMessage, error) {
|
|
called, fallback, err := callSubjectsOn(key, "", c.callerToken())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return c.callOn(called, fallback, body)
|
|
}
|
|
|
|
// CallOn is Call to the tool or seat verb of one named machine (novox/hq ADR 0469 §3; issue 1470): the tool
|
|
// runner of that node alone answers, on mesh.mod.<module>.call.<tool>.<node>.<token> (a seat verb:
|
|
// mesh.seat.<seat>.call.<verb>.<node>.<token>), falling back to the subject that names no caller
|
|
// (mesh-tools node-tools/internal/bus/caller.go:15-19). node is one subject token: an empty one, or one
|
|
// holding '.', '*', '>' or white space, is refused before anything is published.
|
|
func (c *Conn) CallOn(node, key string, body any) (json.RawMessage, error) {
|
|
if err := calladdress.CheckNode(node); err != nil {
|
|
return nil, err
|
|
}
|
|
called, fallback, err := callSubjectsOn(key, node, c.callerToken())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return c.callOn(called, fallback, body)
|
|
}
|
|
|
|
// CallKept is Call for a call whose answer the caller keeps and which is safe to make again (a read, or a write the
|
|
// callee deduplicates): mesh-sdk's kept-call form (design 47 H2 §775), asked again through a transport fault, a timeout
|
|
// or no responders at the kept-call pace (busretry.KeptPace, jittered), whether or not the bus is blacked out, until it
|
|
// is answered or busretry.Bound has passed (novox/hq ADR 0668, extending ADR 0615 §6; issue 1966). A refusal returns at
|
|
// once. A caller with an ask id and a bound of its own uses CallKeptAsk.
|
|
func (c *Conn) CallKept(key string, body any) (json.RawMessage, error) {
|
|
return c.CallKeptAsk(KeptAsk{}, key, body)
|
|
}
|
|
|
|
// CallKeptOn is CallKept to the tool or seat verb of one named machine, as CallOn is Call's.
|
|
func (c *Conn) CallKeptOn(node, key string, body any) (json.RawMessage, error) {
|
|
return c.CallKeptAskOn(KeptAsk{}, node, key, body)
|
|
}
|
|
|
|
// KeptAsk is what a caller names of its kept call (design 47 H2 §775): the ask's id, which the call carries in the
|
|
// header stdio.AskHeader for the holder to take it once by (H-A-351), and the caller's bound (zero is busretry.Bound).
|
|
type KeptAsk struct {
|
|
ID string
|
|
Bound time.Duration
|
|
}
|
|
|
|
// CallKeptAsk is CallKept carrying ask's id and stopping at ask's bound: past it the error is a *busretry.Unanswered
|
|
// naming every call's time and words.
|
|
func (c *Conn) CallKeptAsk(ask KeptAsk, key string, body any) (json.RawMessage, error) {
|
|
return c.callKeptAskOn(ask, "", key, body, nil)
|
|
}
|
|
|
|
// CallKeptAskOn is CallKeptAsk to the tool or seat verb of one named machine.
|
|
func (c *Conn) CallKeptAskOn(ask KeptAsk, node, key string, body any) (json.RawMessage, error) {
|
|
if err := calladdress.CheckNode(node); err != nil {
|
|
return nil, err
|
|
}
|
|
return c.callKeptAskOn(ask, node, key, body, nil)
|
|
}
|
|
|
|
// CallKeptWhileOpen is CallKept for an ask the asker can close or replace: it is asked again only while open says its
|
|
// key is still open (design 47 G §351, §460).
|
|
func (c *Conn) CallKeptWhileOpen(open busretry.Open, key string, body any) (json.RawMessage, error) {
|
|
if open.Open == nil {
|
|
return nil, fmt.Errorf("the call on %s: a kept call names its key's open function", key)
|
|
}
|
|
return c.callKeptAskOn(KeptAsk{}, "", key, body, &open)
|
|
}
|
|
|
|
func (c *Conn) callKeptAskOn(ask KeptAsk, node, key string, body any, open *busretry.Open) (json.RawMessage, error) {
|
|
called, fallback, err := callSubjectsOn(key, node, c.callerToken())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
data, err := json.Marshal(body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return callKeptWith(keptOf(key, ask, open), c.requestAsking(ask.ID), called, fallback, data)
|
|
}
|
|
|
|
// keptOf is the paced form's naming of a call on key: its seat and verb, or its module and tool.
|
|
func keptOf(key string, ask KeptAsk, open *busretry.Open) busretry.Kept {
|
|
k := busretry.Kept{Ask: ask.ID, Bound: ask.Bound, Open: open}
|
|
if a, err := calladdress.Parse(key); err == nil {
|
|
k.Seat, k.Verb = a.Seat, a.Verb
|
|
if a.Seat == "" {
|
|
k.Seat, k.Verb = a.Module, a.Tool
|
|
}
|
|
}
|
|
return k
|
|
}
|
|
|
|
// callKeptWith is callWith for a kept call: busretry.DoPaced with no blackout probe.
|
|
func callKeptWith(k busretry.Kept, request func(string, []byte) (json.RawMessage, error), called, fallback string,
|
|
data []byte) (json.RawMessage, error) {
|
|
return busretry.DoPaced(k, askBoth(request, called, fallback, data))
|
|
}
|
|
|
|
// askBoth asks called, then fallback when nothing answers it.
|
|
func askBoth(request func(string, []byte) (json.RawMessage, error), called, fallback string,
|
|
data []byte) func() (json.RawMessage, error) {
|
|
return func() (json.RawMessage, error) {
|
|
start := time.Now()
|
|
result, err := request(called, data)
|
|
if !errors.Is(err, nats.ErrNoResponders) {
|
|
return result, unansweredOn(err, start, called)
|
|
}
|
|
first := err
|
|
start2 := time.Now()
|
|
result, err = request(fallback, data)
|
|
if err != nil && !isAnswered(err) {
|
|
return result, fmt.Errorf("asked on %s (%v), then on %s, unanswered after %s: %w", called, first, fallback,
|
|
time.Since(start2).Round(time.Millisecond), err)
|
|
}
|
|
return result, err
|
|
}
|
|
}
|
|
|
|
// unansweredOn names the subject a transport fault was met on and how long it waited, so a log of a call nobody
|
|
// answered says where it was asked; a callee's answer keeps its own words.
|
|
func unansweredOn(err error, start time.Time, subject string) error {
|
|
if err == nil || isAnswered(err) {
|
|
return err
|
|
}
|
|
return fmt.Errorf("asked on %s, unanswered after %s: %w", subject, time.Since(start).Round(time.Millisecond), err)
|
|
}
|
|
|
|
func isAnswered(err error) bool {
|
|
var a *busretry.Answered
|
|
return errors.As(err, &a)
|
|
}
|
|
|
|
// callOn asks called, then fallback when nothing answers it; while the bus itself answers nothing (a user-list reload
|
|
// drops the account's imports for a few seconds, busretry) both are asked again, within busretry.Bound. A call nobody
|
|
// answers on a bus that answers is not asked again: its callee is not running. A call that may have been delivered (a
|
|
// timeout, any answer) is never asked again.
|
|
func (c *Conn) callOn(called, fallback string, body any) (json.RawMessage, error) {
|
|
data, err := json.Marshal(body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return callWith(c.request, c.blackedOut, called, fallback, data)
|
|
}
|
|
|
|
// callWith is callOn over a request and a blackout probe. Only the transport's own errors can count as undelivered: a
|
|
// callee's answer is *busretry.Answered, whatever its words, so a call that reached its callee is never asked again.
|
|
func callWith(request func(string, []byte) (json.RawMessage, error), blackout func() bool, called, fallback string,
|
|
data []byte) (json.RawMessage, error) {
|
|
return busretry.Do("the call on "+fallback, askBoth(request, called, fallback, data), blackout)
|
|
}
|
|
|
|
// answered is the error a callee answered with: delivered, never asked again (busretry.Answered).
|
|
func answered(words string) error { return &busretry.Answered{Words: words} }
|
|
|
|
// blackedOut says the bus answers this account nothing at all: its JetStream API, which every account imports, has no
|
|
// responder or says it is not enabled.
|
|
func (c *Conn) blackedOut() bool {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
msg, err := c.nc.RequestWithContext(ctx, "$JS.API.INFO", nil)
|
|
if err != nil {
|
|
return busretry.Undelivered(err)
|
|
}
|
|
return strings.Contains(string(msg.Data), `"err_code":10039`) || strings.Contains(string(msg.Data), "jetstream not enabled")
|
|
}
|
|
|
|
// callerToken is this account's user name as the last token of a call it publishes, each dot written ~
|
|
// (mesh-tools node-tools/internal/bus/caller.go:43, CallerToken).
|
|
func (c *Conn) callerToken() string { return strings.ReplaceAll(c.account.User, ".", "~") }
|
|
|
|
// callSubjects is the caller-named subject key is called on, and its plain fallback (mesh-tools
|
|
// node-tools/internal/bus/caller.go CalledSubject, toolShape, calledPattern).
|
|
func callSubjects(key, token string) (called, fallback string, err error) {
|
|
return callSubjectsOn(key, "", token)
|
|
}
|
|
|
|
// callSubjectsOn is callSubjects for one named node ("" names none, and then the key's own '@' names it): the node is a
|
|
// token between the tool and the caller on the called subject, and the last one of the fallback. A key naming another
|
|
// machine than node is refused.
|
|
func callSubjectsOn(key, node, token string) (called, fallback string, err error) {
|
|
a, err := calladdress.Parse(key)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
switch {
|
|
case a.Node != "" && node != "" && a.Node != node:
|
|
return "", "", fmt.Errorf("%q names the machine %s, and it is asked of %s", key, a.Node, node)
|
|
case a.Node != "":
|
|
node = a.Node
|
|
}
|
|
at := ""
|
|
if node != "" {
|
|
at = "." + node
|
|
}
|
|
if a.Seat != "" {
|
|
return "mesh.seat." + a.Seat + ".call." + a.Verb + at + "." + token, "mesh.seat." + a.Seat + ".tool." + a.Verb + at, nil
|
|
}
|
|
return "mesh.mod." + a.Module + ".call." + a.Tool + at + "." + token, "mesh.mod." + a.Module + ".tool." + a.Tool + at, nil
|
|
}
|
|
|
|
// request asks one subject once and reads its reply ({result, error, node}); a reply naming an error is
|
|
// the call's failure, not a transport one.
|
|
func (c *Conn) request(subject string, data []byte) (json.RawMessage, error) {
|
|
return c.requestAsk(subject, data, "")
|
|
}
|
|
|
|
// requestAsking is request carrying an ask's id in the header stdio.AskHeader (H-A-351); empty carries none.
|
|
func (c *Conn) requestAsking(ask string) func(string, []byte) (json.RawMessage, error) {
|
|
return func(subject string, data []byte) (json.RawMessage, error) { return c.requestAsk(subject, data, ask) }
|
|
}
|
|
|
|
// askMsg is the request a call publishes: its ask's id in stdio.AskHeader when it carries one (H-A-351).
|
|
func askMsg(subject string, data []byte, ask string) *nats.Msg {
|
|
req := nats.NewMsg(subject)
|
|
req.Data = data
|
|
if ask != "" {
|
|
req.Header.Set(stdio.AskHeader, ask)
|
|
}
|
|
return req
|
|
}
|
|
|
|
func (c *Conn) requestAsk(subject string, data []byte, ask string) (json.RawMessage, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), CallTimeout)
|
|
defer cancel()
|
|
req := askMsg(subject, data, ask)
|
|
if strings.HasPrefix(subject, "mesh.seat.") {
|
|
// Every seat call says it waits for a held call's end; a verb that is not marked ignores it (lead answer A-244).
|
|
req.Header.Set(approval.WaitHeader, approval.WaitHeld)
|
|
}
|
|
msg, err := c.nc.RequestMsgWithContext(ctx, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var r struct {
|
|
Result json.RawMessage `json:"result"`
|
|
Error string `json:"error"`
|
|
Node string `json:"node"`
|
|
Held *approval.Held `json:"held"`
|
|
}
|
|
if err := json.Unmarshal(msg.Data, &r); err != nil {
|
|
return nil, fmt.Errorf("%s answered something that is not a seat's answer: %w", subject, err)
|
|
}
|
|
if r.Error != "" {
|
|
return nil, answered(r.Error)
|
|
}
|
|
if r.Held != nil {
|
|
return c.awaitGated(subject, *r.Held)
|
|
}
|
|
return r.Result, nil
|
|
}
|