Compare commits
41
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e2bc0678c6 | ||
|
|
d76d9e0b1a | ||
|
|
8ee4fdfb8e | ||
|
|
05deea5952 | ||
|
|
3850eb8eac | ||
|
|
eb174d853f | ||
|
|
b1d10dd67e | ||
|
|
fb4fe282c7 | ||
|
|
13ff8041b8 | ||
|
|
f92ec7d24f | ||
|
|
8a534ba436 | ||
|
|
d9fe2568de | ||
|
|
81663a4d9e | ||
|
|
a118559515 | ||
|
|
f2a05fb440 | ||
|
|
5d88eecea9 | ||
|
|
618d8953ed | ||
|
|
5314231c0d | ||
|
|
76df3cb420 | ||
|
|
d1dc482b05 | ||
|
|
2f9690fe80 | ||
|
|
30b2a72f9b | ||
|
|
470e57124e | ||
|
|
e554402e47 | ||
|
|
ef0a61d348 | ||
|
|
42a752f052 | ||
|
|
4043dd1176 | ||
|
|
b232e39c96 | ||
|
|
20b6b02341 | ||
|
|
72b0d4ca4e | ||
|
|
4333d5b944 | ||
|
|
9e8bf6dc0c | ||
|
|
45fbc813df | ||
|
|
3d6daad925 | ||
|
|
903ac4ce93 | ||
|
|
9fa29ad801 | ||
|
|
30027347e9 | ||
|
|
7a6174bb3e | ||
|
|
faa1064214 | ||
|
|
27a45d414d | ||
|
|
a96df478d4 |
@@ -1,4 +1,6 @@
|
||||
// node-tools — the node's tool runtime, in Go (novox/hq ADR 0175, ADR 0193; to-be 38 WP4d).
|
||||
// (Its pipeline is asked again by this line, 2026-11-16 23:45: main.981 passed all but composition, whose revision 433 was removed; the assigner now waits 3 minutes for composing nodes.)
|
||||
// Design 47 H1, proof 2 once more: the importer half of the edge mesh-delivery-sdk to node-tools.
|
||||
//
|
||||
// One process per machine: it connects to the bus on the node's credential, launches every assigned
|
||||
// module's tools bundle and serves its tools and its seats' verbs; as the node-tools module — or
|
||||
@@ -52,6 +54,17 @@ func main() {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %q is not a mode; this runtime serves (novox/hq ADR 0193)\n", os.Args[1])
|
||||
os.Exit(2)
|
||||
}
|
||||
// Stopped by the service manager's signal; the runtime itself is serve, which a whole test starts by its entry (design 47 H §774).
|
||||
ctx, stopSignals := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
|
||||
defer stopSignals()
|
||||
if err := serve(ctx); err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// serve is the tool runner: it reads its credential and serves until ctx ends, then shuts down in order. It is the entry a whole test
|
||||
// starts, beside the process's own (design 47 H §774, H-A-61).
|
||||
func serve(ctx context.Context) error {
|
||||
until := time.Now().Add(startWait)
|
||||
cred := credential(until)
|
||||
conn := connectPatiently(cred, until)
|
||||
@@ -60,11 +73,11 @@ func main() {
|
||||
|
||||
served, err := runtime.ServedModulesFrom(os.Getenv(runtime.ToolModules), cred.Module)
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
return err
|
||||
}
|
||||
envs, err := runtime.TakeToolEnvs()
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
return err
|
||||
}
|
||||
// Its own secret ledgers: each module's ledger connection, the only one its history is published on (novox/hq ADR
|
||||
// 0597 §9). Followed, so a credential minted or retired is taken with no restart.
|
||||
@@ -77,7 +90,10 @@ func main() {
|
||||
// The indexer's and the assigner's states the console indexes the mesh by, and the gate reads its marks from, come to it as
|
||||
// first views, asked of the owners' own tools and followed on their views families, never read from their state (novox/hq
|
||||
// ADR 0615 §1; design 46 7-approval-mark §5).
|
||||
source, stopViews := firstview.Start(context.Background(), conn, log.Printf)
|
||||
// In a module's own runtime the follower hears on the module's one consumer, bound by the runtime for its bundle (design 47
|
||||
// H §752, H-A-132).
|
||||
source, stopViews, hearing := startViews(context.Background(), firstview.DepsOf(conn, log.Printf), served, cred.Module, log.Printf)
|
||||
opts = append(opts, hearing...)
|
||||
defer stopViews()
|
||||
approvals := approvalsState{conn: conn, module: cred.Module}
|
||||
gate := approval.New(approval.Deps{Node: conn.Node(), Runner: cred.Module, Catalog: source.ApprovalCatalog(definitionsOf(seatList)), Wait: source.WaitCatalog,
|
||||
@@ -89,7 +105,8 @@ func main() {
|
||||
Build: os.Getenv(runtime.FingerprintWord), Hear: hearOne(conn)})
|
||||
// A failed read or write of the state approvals is made again at the next membership change the runner hears (A-157).
|
||||
conn.OnMembership(func(bus.Membership) { go gate.StateAgain() })
|
||||
opts = append(opts, runtime.WithGate(gate), runtime.WithSeatDefinitions(definitionsOf(seatList)))
|
||||
opts = append(opts, runtime.WithGate(gate), runtime.WithSeatDefinitions(definitionsOf(seatList)),
|
||||
runtime.WithSeatDefinitionsAgain(rereadOf(seatList)))
|
||||
// A run still unrecorded is asked again at each start and when the hand-act seat's holder is heard again (lead answer A-157).
|
||||
if stopBounds, err := gate.WatchBounds(); err != nil {
|
||||
log.Printf("[mesh-tools] the gate's bound keys are not watched (%v): an unanswered hand-act is said in the log only", err)
|
||||
@@ -100,7 +117,7 @@ func main() {
|
||||
go gate.Recover()
|
||||
stop, err := runtime.Run(conn, served, envs, log.Printf, opts...)
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
return err
|
||||
}
|
||||
// And says it is there (novox/hq to-be 45 S11): the machine's runtime, on its machine's name. A
|
||||
// runtime on another credential — a module's own, in a test — is not the machine's.
|
||||
@@ -125,17 +142,16 @@ func main() {
|
||||
}
|
||||
who := node + "." + cred.Module
|
||||
surface := console.NewSurface(conn, source, who)
|
||||
surface.Definitions = definitionsOf(seatList)
|
||||
surface.Approvals = gateTools{Gate: gate, state: approvalsState{conn: conn, module: cred.Module}}
|
||||
up, err = console.Serve(surface, listen)
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
return err
|
||||
}
|
||||
log.Printf("mesh console listening on http://%s/mcp as %s", up.Address, who)
|
||||
}
|
||||
|
||||
signals := make(chan os.Signal, 1)
|
||||
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
|
||||
<-signals
|
||||
<-ctx.Done()
|
||||
// First of all: the service manager signalled every bundle with this process, and none is started again
|
||||
// (issue 1126).
|
||||
launch.ShutDown()
|
||||
@@ -146,6 +162,7 @@ func main() {
|
||||
_ = up.Close()
|
||||
}
|
||||
conn.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// startWait is how long a runtime waits, from its start, for its broker credential to be on the machine and for the bus
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/firstview"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
// ownHearer is the branch follower's hearing in a module's own runtime: the runtime binds the module's consumer once, for
|
||||
// its bundle, and hands each event to it after the bundle took it (runtime.WithHearer), so no second pull loop reads the
|
||||
// same durable and each event reaches both (design 47 H §752, H-A-113, H-A-132).
|
||||
type ownHearer struct {
|
||||
mu sync.Mutex
|
||||
deliver func(bus.Envelope) error
|
||||
}
|
||||
|
||||
// consume is the follower's Consume: it binds nothing, and holds deliver until the stop is called.
|
||||
func (h *ownHearer) consume(deliver func(bus.Envelope) error) (func(), error) {
|
||||
h.mu.Lock()
|
||||
h.deliver = deliver
|
||||
h.mu.Unlock()
|
||||
return func() {
|
||||
h.mu.Lock()
|
||||
h.deliver = nil
|
||||
h.mu.Unlock()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// hear hands one event of the module's consumer to the follower, while it hears.
|
||||
func (h *ownHearer) hear(env bus.Envelope) error {
|
||||
h.mu.Lock()
|
||||
deliver := h.deliver
|
||||
h.mu.Unlock()
|
||||
if deliver == nil {
|
||||
return nil
|
||||
}
|
||||
return deliver(env)
|
||||
}
|
||||
|
||||
// followerOnOwnConsumer is the first views' reach in a runtime that serves its own account's module: the follower hears on
|
||||
// the consumer the runtime binds for the module's bundle, through the hearer answered, which the runtime is given
|
||||
// (runtime.WithHearer). A runtime that serves another module's account (the machine's runtime) keeps the follower's own
|
||||
// binding, and answers no hearer.
|
||||
func followerOnOwnConsumer(d firstview.Deps, served []runtime.Served, own string) (firstview.Deps, *ownHearer) {
|
||||
for _, s := range served {
|
||||
if s.Module == own {
|
||||
h := &ownHearer{}
|
||||
d.Consume = h.consume
|
||||
return d, h
|
||||
}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// startViews is serve's first views: it starts the follower over d and answers the options runtime.Run is given, so that on a
|
||||
// module's own account the follower binds nothing and hears on the module's one consumer (runtime.WithHearer), and on the
|
||||
// machine's account the follower binds its own (design 47 H §752, H-A-132, H-A-133).
|
||||
func startViews(ctx context.Context, d firstview.Deps, served []runtime.Served, own string, logf func(string, ...any)) (*firstview.Views, func(), []runtime.Option) {
|
||||
followed, hearer := followerOnOwnConsumer(d, served, own)
|
||||
source, stop := firstview.StartWith(ctx, followed, logf)
|
||||
if hearer == nil {
|
||||
return source, stop, nil
|
||||
}
|
||||
return source, stop, []runtime.Option{runtime.WithHearer(own, hearer.hear)}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/firstview"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
// In a module's own runtime the branch follower binds no consumer of its own and hears each event the runtime hands it from the
|
||||
// module's one consumer; a runtime that is not the module's own keeps the follower's own binding (design 47 H §752, H-A-132).
|
||||
func TestTheFollowerHearsOnTheOwnModulesOneConsumer(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
served []runtime.Served
|
||||
own string
|
||||
binds bool
|
||||
}{
|
||||
{"a module's own runtime", []runtime.Served{{Module: "mesh-delivery-indexer"}}, "mesh-delivery-indexer", false},
|
||||
{"the machine's runtime", []runtime.Served{{Module: "mesh-delivery-indexer"}}, runtime.NodeRuntimeModule, true},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
bound := 0
|
||||
d := firstview.Deps{Consume: func(func(bus.Envelope) error) (func(), error) { bound++; return func() {}, nil }}
|
||||
got, hearer := followerOnOwnConsumer(d, row.served, row.own)
|
||||
var heard []string
|
||||
stop, err := got.Consume(func(env bus.Envelope) error { heard = append(heard, env.Key); return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if (bound == 1) != row.binds || (hearer == nil) != row.binds {
|
||||
t.Fatalf("bound its own consumer %d time(s), hearer given to the runtime %v; want its own binding %v", bound, hearer != nil, row.binds)
|
||||
}
|
||||
if row.binds {
|
||||
return
|
||||
}
|
||||
env := bus.Envelope{Key: "mesh-catalog.catalog-revision.main", Body: json.RawMessage(`{}`)}
|
||||
if err := hearer.hear(env); err != nil || len(heard) != 1 || heard[0] != env.Key {
|
||||
t.Fatalf("the event handed by the runtime: err %v, the follower heard %q", err, heard)
|
||||
}
|
||||
stop()
|
||||
if err := hearer.hear(env); err != nil || len(heard) != 1 {
|
||||
t.Fatalf("after the follower stopped: err %v, heard %q", err, heard)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// serve's first views: on a module's own account the follower binds nothing and the runtime is given the hearer; on the
|
||||
// machine's account the follower binds its own consumer and the runtime is given nothing (design 47 H §752, H-A-132).
|
||||
func TestServesFollowerBindsOnlyOnTheMachinesAccount(t *testing.T) {
|
||||
served := []runtime.Served{{Module: "mesh-delivery-indexer"}}
|
||||
for _, row := range []struct {
|
||||
own string
|
||||
binds bool
|
||||
}{{"mesh-delivery-indexer", false}, {runtime.NodeRuntimeModule, true}} {
|
||||
t.Run(row.own, func(t *testing.T) {
|
||||
bound := 0
|
||||
d := firstview.Deps{Module: row.own,
|
||||
Ask: func(string, any) (json.RawMessage, error) { return nil, errors.New("no owner here") },
|
||||
Consume: func(func(bus.Envelope) error) (func(), error) { bound++; return func() {}, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
_, stop, opts := startViews(ctx, d, served, row.own, t.Logf)
|
||||
cancel()
|
||||
stop()
|
||||
if (bound == 1) != row.binds || (len(opts) == 1) == row.binds {
|
||||
t.Fatalf("the follower bound %d consumer(s), the runtime is given %d option(s); want the follower's own binding %v",
|
||||
bound, len(opts), row.binds)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -63,11 +63,25 @@ func startSeatListOn(b seatlist.Bus, bound <-chan time.Time, logf func(string, .
|
||||
return nil
|
||||
}
|
||||
catalogue.UseSeats(reader.Seats())
|
||||
// a first read that ended without an answer is no failure of the reader, which holds the bootstrap and waits for the seat's next
|
||||
// event: so it is said here, where nothing else would say it, whether it ends before the bound or after it (a read the bus does
|
||||
// not answer is asked again for about 30 s, so it ends after the bound); the runner then reads again where a bundle asks a verb of
|
||||
// a seat the list defines that its definition does not name (design 47 G A-490m)
|
||||
sayUnanswered := func() {
|
||||
if st := reader.Status(); !st.Heard {
|
||||
logf("node-tools: the first read of the seat list ended without an answer (%s); the runner serves with the bootstrapped seats and reads again when a bundle asks a verb of a seat the list defines that its definition does not name", st.LastError)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-reader.Ready():
|
||||
catalogue.UseSeats(reader.Seats())
|
||||
sayUnanswered()
|
||||
case <-bound:
|
||||
logf("node-tools: the seat list is not read yet; the runner serves with the bootstrapped seats and reads on")
|
||||
go func() {
|
||||
<-reader.Ready()
|
||||
sayUnanswered()
|
||||
}()
|
||||
}
|
||||
return reader
|
||||
}
|
||||
@@ -82,3 +96,14 @@ func definitionsOf(reader *seatlist.Reader) func() map[string]seatdef.Definition
|
||||
return reader.Definitions()
|
||||
}
|
||||
}
|
||||
|
||||
// rereadOf is how the runner reads the seat list again where a verb is not named by the definitions it holds; it reads nothing
|
||||
// while the reader did not start.
|
||||
func rereadOf(reader *seatlist.Reader) func() error {
|
||||
return func() error {
|
||||
if reader == nil {
|
||||
return fmt.Errorf("the seat list reader did not start")
|
||||
}
|
||||
return reader.Reload()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -32,19 +33,23 @@ func TestTheRunnerWaitsForTheFirstReadOfTheSeatListOnlyToItsBound(t *testing.T)
|
||||
wantSaid string
|
||||
wantReady bool
|
||||
}{
|
||||
{"the first read ends before the bound", true, "", true},
|
||||
{"the first read ends before the bound, unanswered", true, "the first read of the seat list ended without an answer", true},
|
||||
{"the bound passes with the seat silent", false, "the seat list is not read yet", false},
|
||||
{"the bound passes, then the read ends unanswered", false, "the first read of the seat list ended without an answer", false},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
b := &busOfAStandIn{release: make(chan struct{})}
|
||||
bound := make(chan time.Time, 1)
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
saidAgain := make(chan struct{}, 4)
|
||||
say := func(f string, a ...any) { mu.Lock(); said = append(said, f); mu.Unlock(); saidAgain <- struct{}{} }
|
||||
if row.release {
|
||||
close(b.release)
|
||||
} else {
|
||||
bound <- time.Time{}
|
||||
}
|
||||
r := startSeatListOn(b, bound, func(f string, a ...any) { said = append(said, f) })
|
||||
r := startSeatListOn(b, bound, say)
|
||||
if r == nil {
|
||||
t.Fatal("no reader")
|
||||
}
|
||||
@@ -57,12 +62,25 @@ func TestTheRunnerWaitsForTheFirstReadOfTheSeatListOnlyToItsBound(t *testing.T)
|
||||
if ended != row.wantReady {
|
||||
t.Errorf("first read ended = %v, want %v", ended, row.wantReady)
|
||||
}
|
||||
if got := strings.Join(said, ";"); !strings.Contains(got, row.wantSaid) || (row.wantSaid == "" && got != "") {
|
||||
t.Errorf("said %q, want %q", got, row.wantSaid)
|
||||
}
|
||||
if !row.release {
|
||||
close(b.release)
|
||||
<-r.Ready()
|
||||
// the read that ends after the bound is said by a goroutine of its own: wait for it, to a deadline that only a fault reaches
|
||||
if strings.Contains(row.wantSaid, "first read") {
|
||||
for i := 0; i < 2; i++ {
|
||||
select {
|
||||
case <-saidAgain:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the unanswered end of the first read was never said")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
got := strings.Join(said, ";")
|
||||
mu.Unlock()
|
||||
if !strings.Contains(got, row.wantSaid) {
|
||||
t.Errorf("said %q, want %q", got, row.wantSaid)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// catalogStandIn is the seat mesh-catalog's verb `seat-definitions`, as the seat list's reader asks it: the seat node-desk whose
|
||||
// first listing names the verbs who and slow (slow with a timeout of one second) and, from the second listing on, also the verb
|
||||
// fresh, which sends its answer in chunks.
|
||||
type catalogStandIn struct {
|
||||
mu sync.Mutex
|
||||
listings int
|
||||
}
|
||||
|
||||
func (c *catalogStandIn) definition(withFresh bool) json.RawMessage {
|
||||
one := 1
|
||||
verbs := []seatdef.Verb{{Name: "who"}, {Name: "slow", Timeout: &one}}
|
||||
if withFresh {
|
||||
verbs = append(verbs, seatdef.Verb{Name: "fresh", Chunks: true})
|
||||
}
|
||||
raw, _ := json.Marshal(seatdef.Definition{Format: seatdef.Format, Seat: "node-desk", Scope: seatdef.ScopeNode, Grain: seatdef.SeatList,
|
||||
Decision: "ADR 0001", Domain: seatdef.Domains[0], Verbs: verbs})
|
||||
return raw
|
||||
}
|
||||
|
||||
func (c *catalogStandIn) handle(body json.RawMessage) (any, error) {
|
||||
var asked struct {
|
||||
Seat string `json:"seat"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &asked)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if asked.Seat == "" {
|
||||
c.listings++
|
||||
return seatlist.Listing{Revision: uint64(c.listings), Seats: []seatlist.Entry{{Seat: "node-desk", Source: seatlist.SourceTrunk,
|
||||
Digest: map[bool]string{false: "d1", true: "d2"}[c.listings > 1]}}}, nil
|
||||
}
|
||||
withFresh := c.listings > 1
|
||||
return seatlist.Answer{Seat: asked.Seat, Found: true, Revision: uint64(c.listings), Entry: &seatlist.Entry{Seat: asked.Seat,
|
||||
Source: seatlist.SourceTrunk, Digest: map[bool]string{false: "d1", true: "d2"}[withFresh], Definition: c.definition(withFresh)}}, nil
|
||||
}
|
||||
|
||||
// **The tool runner, started whole by its entry** (design 47 H §774, H-A-16, H-A-61): the process's own `serve`, on an embedded bus with
|
||||
// JetStream, with a credential file and the environment the node-engine writes, serving one bundle's tool and seat verbs, over the
|
||||
// seat list read from a stand-in of the seat mesh-catalog.
|
||||
// - the bundle's tool and its seat verb are answered on the bus (§734's way of serving a verb from a bundle);
|
||||
// - a verb whose definition names a timeout of one second is waited one second by the serving runner, not the runner's own
|
||||
// thirty (design 47 H §760, H-A-43);
|
||||
// - a bundle's ask of a seat verb its first seat list does not name is asked after one reread of the list, with its call token
|
||||
// for a verb that sends chunks (design 47 G A-490m).
|
||||
func TestTheToolRunnerStartedByItsEntryServesABundleOverTheSeatListItReads(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", true, map[string][]string{"node-desk": {"who", "slow"}}))
|
||||
dir := t.TempDir()
|
||||
credential, _ := json.Marshal(bus.Credential{URL: mt.URL(t), Module: runtimeModule, Node: "anchor"})
|
||||
file := filepath.Join(dir, "broker")
|
||||
if err := os.WriteFile(file, credential, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("MESH_BROKER_FILE", file)
|
||||
t.Setenv("MESH_TOOL_MODULES", "ledger="+mt.Fixture("whole-bundle.py"))
|
||||
t.Setenv("MESH_CONSOLE_LISTEN", "127.0.0.1:0")
|
||||
|
||||
open := func(module, node string) *bus.Conn {
|
||||
c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: module, Node: node, User: node + "." + module, Password: "unused"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Logf = func(string, ...any) {}
|
||||
t.Cleanup(c.Close)
|
||||
return c
|
||||
}
|
||||
catalog := &catalogStandIn{}
|
||||
holder := open("mesh-catalog", "anchor")
|
||||
if _, err := holder.HandleSubject("mesh.seat.mesh-catalog.tool.seat-definitions", func(body json.RawMessage) (any, error) { return catalog.handle(body) }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var fresh []map[string]any
|
||||
if _, err := holder.HandleSubject("mesh.seat.node-desk.tool.fresh", func(body json.RawMessage) (any, error) {
|
||||
var got map[string]any
|
||||
_ = json.Unmarshal(body, &got)
|
||||
mu.Lock()
|
||||
fresh = append(fresh, got)
|
||||
mu.Unlock()
|
||||
return map[string]any{"ok": true}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holder.Flush()
|
||||
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
ended := make(chan error, 1)
|
||||
go func() { ended <- serve(ctx) }()
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
select {
|
||||
case err := <-ended:
|
||||
if err != nil {
|
||||
t.Errorf("serve ended with %v", err)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Error("the tool runner did not stop within 30s")
|
||||
}
|
||||
})
|
||||
// a module's account, which the approval gate runs unmarked while the catalog half is unread (the stand-in serves no catalog half)
|
||||
asker := open("shop", "workstation")
|
||||
ask := func(key string) (string, error) {
|
||||
answered, err := asker.Ask(key, map[string]any{}, "")
|
||||
return string(answered.Result), err
|
||||
}
|
||||
|
||||
// the bundle's tool and its seat verb are served
|
||||
for _, key := range []string{"ledger.who", "seat:node-desk.who@anchor"} {
|
||||
mt.Until(t, func() error {
|
||||
got, err := ask(key)
|
||||
if err == nil && !strings.Contains(got, `"tool":`) {
|
||||
t.Errorf("%s answered %s, want the bundle's", key, got)
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// a verb whose definition names a timeout of one second is waited one second
|
||||
began := time.Now()
|
||||
if _, err := ask("seat:node-desk.slow@anchor"); err == nil {
|
||||
t.Error("the verb that answers after three seconds was answered within its timeout of one second")
|
||||
}
|
||||
if took := time.Since(began); took < time.Second || took > 2500*time.Millisecond {
|
||||
t.Errorf("the verb of a timeout of one second was waited %s, want about one second, not the runner's own %s", took, bus.RequestTimeout)
|
||||
}
|
||||
|
||||
// the bundle's ask of a verb its first list does not name is asked after one reread, with its call token
|
||||
mt.Until(t, func() error {
|
||||
got, err := ask("ledger.reach")
|
||||
if err == nil && strings.Contains(got, "error") {
|
||||
t.Errorf("the bundle's ask of node-desk.fresh failed: %s", got)
|
||||
}
|
||||
return err
|
||||
})
|
||||
catalog.mu.Lock()
|
||||
listings := catalog.listings
|
||||
catalog.mu.Unlock()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if listings != 2 || len(fresh) != 1 || fresh[0]["call"] == nil || fresh[0]["call"] == "" {
|
||||
t.Errorf("the list was read %d times (want 2) and the holder of node-desk.fresh was asked %v (want once, with a call token)", listings, fresh)
|
||||
}
|
||||
}
|
||||
@@ -49,13 +49,20 @@ func Version() string {
|
||||
// Keep says the runtime is there now and every interval until the returned function is called. A
|
||||
// heartbeat that cannot be said is logged when that starts and when it stops — not every minute.
|
||||
func Keep(bus Sayer, node string, every time.Duration, logf func(string, ...any)) func() {
|
||||
tick := time.NewTicker(every)
|
||||
stop := keep(bus, node, every, tick.C, logf)
|
||||
return func() { stop(); tick.Stop() }
|
||||
}
|
||||
|
||||
// keep is Keep saying it again on every tick: the interval is told in the beat, the ticks are the caller's.
|
||||
// Its stop returns once nothing more is said.
|
||||
func keep(bus Sayer, node string, every time.Duration, ticks <-chan time.Time, logf func(string, ...any)) func() {
|
||||
body, _ := json.Marshal(Beat{Node: node, IntervalSeconds: int(every / time.Second), Version: Version()})
|
||||
done := make(chan struct{})
|
||||
done, ended := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
go func() {
|
||||
defer close(ended)
|
||||
failing := ""
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
why := ""
|
||||
if err := bus.Say(Subject(node), body); err != nil {
|
||||
@@ -72,9 +79,12 @@ func Keep(bus Sayer, node string, every time.Duration, logf func(string, ...any)
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
case <-ticks:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { once.Do(func() { close(done) }) }
|
||||
return func() {
|
||||
once.Do(func() { close(done) })
|
||||
<-ended
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@ package alive
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -13,11 +15,29 @@ type said struct {
|
||||
subs []string
|
||||
body [][]byte
|
||||
fail error
|
||||
// tried is told every Say, said or not, so a test waits on the heartbeat itself and never on the clock.
|
||||
tried chan struct{}
|
||||
// held, when set, keeps each Say from returning until it is closed; inSay says one is held there now.
|
||||
held chan struct{}
|
||||
inSay atomic.Bool
|
||||
}
|
||||
|
||||
func newSaid(fail error) *said { return &said{fail: fail, tried: make(chan struct{}, 16)} }
|
||||
|
||||
func (s *said) Say(subject string, body []byte) error {
|
||||
if s.held != nil {
|
||||
s.inSay.Store(true)
|
||||
s.tried <- struct{}{}
|
||||
<-s.held
|
||||
defer s.inSay.Store(false)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
defer func() {
|
||||
s.mu.Unlock()
|
||||
if s.held == nil {
|
||||
s.tried <- struct{}{}
|
||||
}
|
||||
}()
|
||||
if s.fail != nil {
|
||||
return s.fail
|
||||
}
|
||||
@@ -32,39 +52,85 @@ func (s *said) count() int {
|
||||
}
|
||||
|
||||
// **The runtime says it is there at once and every interval, as its machine, with the interval**: the
|
||||
// controller's bound is three of them (to-be 45 S11).
|
||||
// controller's bound is three of them (to-be 45 S11). The ticks are the test's own, so a loaded machine
|
||||
// cannot make it miss one (principle 11; design 47 H §774); stopped, it says nothing more.
|
||||
func TestTheRuntimeSaysItIsThereEveryInterval(t *testing.T) {
|
||||
s := &said{}
|
||||
stop := Keep(s, "anchor", 20*time.Millisecond, t.Logf)
|
||||
time.Sleep(70 * time.Millisecond)
|
||||
s := newSaid(nil)
|
||||
ticks := make(chan time.Time)
|
||||
stop := keep(s, "anchor", 20*time.Millisecond, ticks, t.Logf)
|
||||
<-s.tried
|
||||
for i := 0; i < 2; i++ {
|
||||
ticks <- time.Time{}
|
||||
<-s.tried
|
||||
}
|
||||
stop()
|
||||
stop()
|
||||
if n := s.count(); n < 3 {
|
||||
t.Fatalf("said %d times in three intervals", n)
|
||||
if n := s.count(); n != 3 {
|
||||
t.Fatalf("said %d times at once and on two ticks", n)
|
||||
}
|
||||
var beat Beat
|
||||
if err := json.Unmarshal(s.body[0], &beat); err != nil || beat.Node != "anchor" || beat.IntervalSeconds != 0 ||
|
||||
s.subs[0] != "mesh.control.anchor.tools-alive" {
|
||||
t.Fatalf("%s on %s: %v", s.body[0], s.subs[0], err)
|
||||
}
|
||||
n := s.count()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if s.count() != n {
|
||||
select {
|
||||
case ticks <- time.Time{}:
|
||||
t.Fatal("a tick was taken after it was stopped")
|
||||
default:
|
||||
}
|
||||
if s.count() != 3 {
|
||||
t.Fatal("said after it was stopped")
|
||||
}
|
||||
// Stopped while a beat is being said, stop returns only once that Say has returned: nothing is said after it.
|
||||
slow := newSaid(nil)
|
||||
slow.held = make(chan struct{})
|
||||
stopSlow := keep(slow, "anchor", 20*time.Millisecond, make(chan time.Time), t.Logf)
|
||||
<-slow.tried
|
||||
returned := make(chan bool)
|
||||
go func() {
|
||||
stopSlow()
|
||||
returned <- slow.inSay.Load()
|
||||
}()
|
||||
// The stop is given its turn before the Say is let go (no clock: a returned stop is seen at once).
|
||||
for i := 0; i < 1000; i++ {
|
||||
select {
|
||||
case stillSaying := <-returned:
|
||||
close(slow.held)
|
||||
t.Fatalf("stop returned while a beat was still being said (held: %v)", stillSaying)
|
||||
default:
|
||||
runtime.Gosched()
|
||||
}
|
||||
}
|
||||
close(slow.held)
|
||||
if stillSaying := <-returned; stillSaying {
|
||||
t.Fatal("stop returned while a beat was still being said")
|
||||
}
|
||||
// Keep itself keeps the interval on a ticker of its own.
|
||||
kept := newSaid(nil)
|
||||
stopKept := Keep(kept, "anchor", time.Hour, t.Logf)
|
||||
<-kept.tried
|
||||
stopKept()
|
||||
if kept.count() != 1 {
|
||||
t.Fatalf("Keep said %d times before its first interval", kept.count())
|
||||
}
|
||||
}
|
||||
|
||||
// **A heartbeat the bus will not take is said once, and again when it is taken**: not a line a minute.
|
||||
func TestAHeartbeatThatCannotBeSaidIsLoggedOnce(t *testing.T) {
|
||||
s := &said{fail: errors.New("permissions violation")}
|
||||
s := newSaid(errors.New("permissions violation"))
|
||||
var mu sync.Mutex
|
||||
var lines []string
|
||||
stop := Keep(s, "anchor", 10*time.Millisecond, func(f string, a ...any) {
|
||||
ticks := make(chan time.Time)
|
||||
stop := keep(s, "anchor", 10*time.Millisecond, ticks, func(f string, a ...any) {
|
||||
mu.Lock()
|
||||
lines = append(lines, f)
|
||||
mu.Unlock()
|
||||
})
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
<-s.tried
|
||||
for i := 0; i < 4; i++ {
|
||||
ticks <- time.Time{}
|
||||
<-s.tried
|
||||
}
|
||||
stop()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
@@ -218,16 +218,6 @@ func TestTheRefusalsAndAnswers(t *testing.T) {
|
||||
t.Fatalf("asked %d, %v, runs %d", len(w.asked), err, w.runs)
|
||||
}
|
||||
}},
|
||||
{"§17: an ask the router holds back as the fourth open one still waits", func(t *testing.T, w *world) {
|
||||
_, _ = w.do(call("wipe", `{}`))
|
||||
a := w.asked[0]
|
||||
rec := w.records["shell."+a.ID]
|
||||
rec.State, rec.HeldSince = "held-back", operatorchannelclient.FormatTime(w.now)
|
||||
w.records["shell."+a.ID] = rec
|
||||
if _, err := w.do(call("wipe", `{}`)); err == nil || !strings.Contains(err.Error(), "still waiting") || len(w.asked) != 1 {
|
||||
t.Fatalf("%v, asked %d", err, len(w.asked))
|
||||
}
|
||||
}},
|
||||
{"§14, §15: approved, the same call runs once; made again it is already run and asks anew", func(t *testing.T, w *world) {
|
||||
_, _ = w.do(call("wipe", `{"dir":"scratch"}`))
|
||||
w.decide(gate.OptionApprove, w.now.Add(time.Minute))
|
||||
@@ -321,9 +311,9 @@ func TestTheRefusalsAndAnswers(t *testing.T) {
|
||||
_, _ = w.do(call("wipe", `{}`))
|
||||
a := w.asked[0]
|
||||
rec := w.records["shell."+a.ID]
|
||||
rec.State, rec.Answer = "refused", "the asker has too many open asks"
|
||||
rec.State, rec.Answer = "refused", "the asker is not known to the router"
|
||||
w.records["shell."+a.ID] = rec
|
||||
if _, err := w.do(call("wipe", `{}`)); err == nil || !strings.Contains(err.Error(), "the asker has too many open asks") {
|
||||
if _, err := w.do(call("wipe", `{}`)); err == nil || !strings.Contains(err.Error(), "the asker is not known to the router") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}},
|
||||
|
||||
@@ -62,8 +62,28 @@ func (c *Conn) Bindings(module string) []membership.BusBinding {
|
||||
// A-16, A-27).
|
||||
func (c *Conn) JetStream() (jetstream.JetStream, error) { return jetstream.New(c.nc) }
|
||||
|
||||
func (c *Conn) objectStore(ctx context.Context, module, local string) (jetstream.ObjectStore, membership.BusBinding, error) {
|
||||
// objectBucketOf is the bus name of the module's object store of a local name: its binding's when it has one, else, for a
|
||||
// name no binding of any type holds, the store grants creates for a module's old `objects` entry, `<module>_<name>`
|
||||
// (catalogue.ObjectsBucket; design 47 G lead answers A-380): the bus's rights decide whether it is reached. A name bound
|
||||
// with another storage type is refused as the binding refuses it.
|
||||
func (c *Conn) objectBucketOf(module, local string) (membership.BusBinding, error) {
|
||||
b, err := c.Binding(module, local, catalogue.StorageObjectStore)
|
||||
if err == nil {
|
||||
return b, nil
|
||||
}
|
||||
if m := c.Membership(module); m != nil {
|
||||
for _, held := range m.Bindings {
|
||||
if held.Local == local {
|
||||
return b, err
|
||||
}
|
||||
}
|
||||
return membership.BusBinding{Local: local, Storage: catalogue.StorageObjectStore, ObjectStore: catalogue.ObjectsBucket(module, local)}, nil
|
||||
}
|
||||
return b, err
|
||||
}
|
||||
|
||||
func (c *Conn) objectStore(ctx context.Context, module, local string) (jetstream.ObjectStore, membership.BusBinding, error) {
|
||||
b, err := c.objectBucketOf(module, local)
|
||||
if err != nil {
|
||||
return nil, b, err
|
||||
}
|
||||
@@ -73,6 +93,10 @@ func (c *Conn) objectStore(ctx context.Context, module, local string) (jetstream
|
||||
}
|
||||
obs, err := js.ObjectStore(ctx, b.ObjectStore)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
if b.Template == "" {
|
||||
return nil, b, fmt.Errorf("%s was bound no object-store called %q, and the store %s grants creates for an old `objects` entry of that name is not on the bus",
|
||||
module, local, b.ObjectStore)
|
||||
}
|
||||
return nil, b, fmt.Errorf("%s's objects %q are bound and their store %s is not on the bus", module, local, b.ObjectStore)
|
||||
}
|
||||
return obs, b, err
|
||||
@@ -178,7 +202,7 @@ func (c *Conn) ObjectsInfo(module string, r buscalls.ObjectsRequest) (buscalls.F
|
||||
if err != nil {
|
||||
return buscalls.Fill{}, err
|
||||
}
|
||||
b, _ := c.Binding(module, r.Objects, catalogue.StorageObjectStore)
|
||||
b, _ := c.objectBucketOf(module, r.Objects)
|
||||
fill, err := c.streamFill("OBJ_" + b.ObjectStore)
|
||||
if err != nil {
|
||||
return buscalls.Fill{}, err
|
||||
|
||||
@@ -33,6 +33,9 @@ func TestABundleReachesItsOwnBoundObjectsAndStreamsOnly(t *testing.T) {
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: "m_files"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: "m_old"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "m_events", Subjects: []string{"mesh.state.m.events.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -92,6 +95,13 @@ func TestABundleReachesItsOwnBoundObjectsAndStreamsOnly(t *testing.T) {
|
||||
if listed, err := conn.ObjectsList("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || len(listed.Objects) != 1 {
|
||||
t.Fatalf("listed %+v, %v", listed, err)
|
||||
}
|
||||
// A-380: an old `objects` entry is reached by the store grants creates for it, `<module>_<name>`, with no binding of that name.
|
||||
if _, err := conn.ObjectsPut("m", buscalls.ObjectsPutRequest{Objects: "old", Name: "r", Data: []byte("old section")}); err != nil {
|
||||
t.Fatalf("put into the old objects section: %v", err)
|
||||
}
|
||||
if listed, err := conn.ObjectsList("m", buscalls.ObjectsRequest{Objects: "old"}); err != nil || len(listed.Objects) != 1 {
|
||||
t.Fatalf("listed the old objects section %+v, %v", listed, err)
|
||||
}
|
||||
page, err := conn.StreamRead("m", buscalls.StreamReadRequest{Stream: "events", Limit: 1})
|
||||
if err != nil || len(page.Messages) != 1 || page.Next != 2 || page.Last != 2 {
|
||||
t.Fatalf("read %+v, %v", page, err)
|
||||
@@ -121,7 +131,7 @@ func TestABundleReachesItsOwnBoundObjectsAndStreamsOnly(t *testing.T) {
|
||||
name, module, refusal string
|
||||
call func(module string) error
|
||||
}{
|
||||
{"a local name not bound", "m", `bound no object-store called "photos"`, func(module string) error {
|
||||
{"a local name not bound", "m", `bound no object-store called "photos", and the store m_photos grants creates`, func(module string) error {
|
||||
_, err := conn.ObjectsGet(module, buscalls.ObjectsNameRequest{Objects: "photos", Name: "a"})
|
||||
return err
|
||||
}},
|
||||
|
||||
@@ -32,9 +32,18 @@ import (
|
||||
"github.com/novox/mesh-tools/node-tools/internal/wire"
|
||||
)
|
||||
|
||||
// RequestTimeout is how long a call waits for its answer — what modules already expect.
|
||||
// RequestTimeout is how long a call waits for its answer when its verb's definition names no `timeout`: the tool runner's own
|
||||
// bound, what modules already expect (design 47 H §760, H-A-43).
|
||||
const RequestTimeout = 30 * time.Second
|
||||
|
||||
// waitOf is the bound a call waits: within where the verb's definition names one, else the tool runner's own.
|
||||
func waitOf(within time.Duration) time.Duration {
|
||||
if within > 0 {
|
||||
return within
|
||||
}
|
||||
return RequestTimeout
|
||||
}
|
||||
|
||||
// Credential is a broker credential as the mesh delivers it (novox/hq ADR 0120).
|
||||
type Credential struct {
|
||||
URL string `json:"url"`
|
||||
@@ -682,6 +691,12 @@ func (c *Conn) HandleSubjectCalled(subject string, h CalledHandler) (func(), err
|
||||
return c.answerBoth(subject, "", h)
|
||||
}
|
||||
|
||||
// HandleSubjectCalledQueue is HandleSubjectCalled in one queue group: of every holder that serves the subject in the
|
||||
// group, one answers (lead answer A-393: a node-scoped seat's verb asked with no machine).
|
||||
func (c *Conn) HandleSubjectCalledQueue(subject, queue string, h CalledHandler) (func(), error) {
|
||||
return c.answerBoth(subject, queue, h)
|
||||
}
|
||||
|
||||
// answerBoth answers a subject and, when it is a tool's, the caller-named pattern beside it, in one queue.
|
||||
func (c *Conn) answerBoth(subject, queue string, h CalledHandler) (func(), error) {
|
||||
stop, err := c.answerCalledOn(subject, queue, h)
|
||||
@@ -725,25 +740,35 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) {
|
||||
"those of the modules it follows", c.self, key)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var stops []func()
|
||||
stops := map[Served]func(){}
|
||||
stopped := false
|
||||
// **Make before break** (novox/hq design 47 H, H-A-158): a membership issued again keeps every subscription
|
||||
// whose subject and queue it still names, subscribes those it newly names, and only then stops those it no
|
||||
// longer names — a call the bus routes while the membership is followed is never dropped in silence.
|
||||
serve := func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if stopped {
|
||||
return
|
||||
}
|
||||
for _, s := range stops {
|
||||
s()
|
||||
}
|
||||
stops = nil
|
||||
named := map[Served]bool{}
|
||||
for _, s := range c.servedOn(module, tool) {
|
||||
named[s] = true
|
||||
if _, kept := stops[s]; kept {
|
||||
continue
|
||||
}
|
||||
if stop, err := c.answerBoth(s.Subject, s.Queue, h); err == nil {
|
||||
stops = append(stops, stop)
|
||||
stops[s] = stop
|
||||
} else {
|
||||
c.Logf("[mesh-tools] cannot serve %s on %s: %v", key, s.Subject, err)
|
||||
}
|
||||
}
|
||||
for s, stop := range stops {
|
||||
if !named[s] {
|
||||
stop()
|
||||
delete(stops, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
serve()
|
||||
// Its stop drops the membership handler too: a handler left behind would bind the stopped tool again
|
||||
@@ -758,10 +783,10 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
stopped = true
|
||||
for _, s := range stops {
|
||||
s()
|
||||
for s, stop := range stops {
|
||||
stop()
|
||||
delete(stops, s)
|
||||
}
|
||||
stops = nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -787,6 +812,12 @@ func (c *Conn) reachedAt(key string) (string, error) {
|
||||
// Ask calls a tool by key — or on a subject the mesh listed for it — and learns which machine
|
||||
// answered. Core request/reply: a tool call is never persisted (design 25 §3).
|
||||
func (c *Conn) Ask(key string, body any, on string) (Answered, error) {
|
||||
return c.AskWithin(key, body, on, 0)
|
||||
}
|
||||
|
||||
// AskWithin is Ask that waits within for its answer, the `timeout` its verb's definition names (design 47 H §760, H-A-43);
|
||||
// zero waits the tool runner's own bound, RequestTimeout.
|
||||
func (c *Conn) AskWithin(key string, body any, on string, within time.Duration) (Answered, error) {
|
||||
subject := on
|
||||
if subject == "" {
|
||||
s, err := c.reachedAt(key)
|
||||
@@ -803,12 +834,12 @@ func (c *Conn) Ask(key string, body any, on string) (Answered, error) {
|
||||
// account publish with its own name alone; asked again on the subject that names no caller only when the
|
||||
// first reached nobody — nothing serves it yet, or the bus does not grant it yet — so no call is done twice.
|
||||
if called := CalledSubject(subject, c.cred.User); called != "" {
|
||||
answered, fallBack, err := c.askCalled(called, data)
|
||||
answered, fallBack, err := c.askCalled(called, data, within)
|
||||
if !fallBack {
|
||||
return answered, err
|
||||
}
|
||||
}
|
||||
return c.askOnce(subject, data)
|
||||
return c.askOnce(subject, data, within)
|
||||
}
|
||||
|
||||
// ErrNoResponders is the bus's own refusal: nothing subscribes to the subject asked. It is told apart
|
||||
@@ -816,8 +847,8 @@ func (c *Conn) Ask(key string, body any, on string) (Answered, error) {
|
||||
var ErrNoResponders = errors.New("503 no responders")
|
||||
|
||||
// askOnce asks once.
|
||||
func (c *Conn) askOnce(subject string, data []byte) (Answered, error) {
|
||||
msg, err := c.nc.Request(subject, data, RequestTimeout)
|
||||
func (c *Conn) askOnce(subject string, data []byte, within time.Duration) (Answered, error) {
|
||||
msg, err := c.nc.Request(subject, data, waitOf(within))
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return Answered{}, ErrNoResponders
|
||||
@@ -1030,6 +1061,11 @@ func SeatToolSubject(seat, verb, scope, node string) string {
|
||||
// `<module>.<tool>` another's, `seat:<seat>.<verb>[@<node>]` a role's — resolved through what the
|
||||
// mesh issued that module to reach, as its own runtime resolved it, else the derived shape.
|
||||
func (c *Conn) AskAs(module, key string, body any) (Answered, error) {
|
||||
return c.AskAsWithin(module, key, body, 0)
|
||||
}
|
||||
|
||||
// AskAsWithin is AskAs that waits within for its answer (design 47 H §760); zero waits RequestTimeout.
|
||||
func (c *Conn) AskAsWithin(module, key string, body any, within time.Duration) (Answered, error) {
|
||||
name, wanted, _ := strings.Cut(key, "@")
|
||||
subject := ""
|
||||
if m := c.Membership(module); m != nil {
|
||||
@@ -1052,7 +1088,7 @@ func (c *Conn) AskAs(module, key string, body any) (Answered, error) {
|
||||
}
|
||||
subject = s
|
||||
}
|
||||
return c.Ask(key, body, subject)
|
||||
return c.AskWithin(key, body, subject, within)
|
||||
}
|
||||
|
||||
// EventsStream is where every module's events land, and ConsumerOf the durable consumer the
|
||||
@@ -1069,6 +1105,13 @@ func ConsumerOf(node, module string) string {
|
||||
return node + "_" + module
|
||||
}
|
||||
|
||||
// ToolsSuffix ends the name of the tool runner's own durable consumer of a module: the runner and the module's program each
|
||||
// need every event, so each reads under a name of its own (lead answer A-397); only a work queue shares one name.
|
||||
const ToolsSuffix = "_tools"
|
||||
|
||||
// ToolsConsumerOf is the tool runner's durable consumer of a module on a machine: `<node>_<module>_tools`.
|
||||
func ToolsConsumerOf(node, module string) string { return ConsumerOf(node, module) + ToolsSuffix }
|
||||
|
||||
// NakDelay is how long an event a handler failed waits before it is offered again: a transient cause
|
||||
// gets another attempt, a permanent one exhausts the consumer's max-deliver rather than spinning. The
|
||||
// consumer then gives the event up, and the mesh keeps it in DEAD_LETTERS until a person delivers
|
||||
@@ -1119,7 +1162,7 @@ var tickHook func()
|
||||
// answers true (nil is always): a module whose children asked for no heartbeat costs the bus no consumer-info request.
|
||||
func (c *Conn) ConsumeAsBeatingWhen(module string, deliver func(Envelope) error, beat func(Beat), wanted func() bool) (func(), error) {
|
||||
asked := func() bool { return beat != nil && (wanted == nil || wanted()) }
|
||||
durable := ConsumerOf(c.node, module)
|
||||
durable := ToolsConsumerOf(c.node, module)
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSubjectsAreTheOnesTheTypeScriptRuntimeUses(t *testing.T) {
|
||||
@@ -60,3 +61,12 @@ func TestAnEventDeliveredAgainHasItsOwnKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §760: a call waits the bound its verb's definition names, else the tool runner's own.
|
||||
func TestWaitOfIsTheVerbsBoundElseTheRunnersOwn(t *testing.T) {
|
||||
for within, want := range map[time.Duration]time.Duration{0: RequestTimeout, 40 * time.Second: 40 * time.Second, 600 * time.Second: 600 * time.Second} {
|
||||
if got := waitOf(within); got != want {
|
||||
t.Errorf("waitOf(%s) = %s, want %s", within, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
@@ -150,9 +151,9 @@ func (c *Conn) publishRefused(err error) bool {
|
||||
// askCalled asks once on a caller-named subject. fallBack says the call reached nobody — nothing serves that
|
||||
// subject yet, or the bus refused this user its publish — so it may be asked on the subject that names no
|
||||
// caller without being done twice.
|
||||
func (c *Conn) askCalled(subject string, data []byte) (answered Answered, fallBack bool, err error) {
|
||||
func (c *Conn) askCalled(subject string, data []byte, within time.Duration) (answered Answered, fallBack bool, err error) {
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
ctx, stop := context.WithTimeout(ctx, RequestTimeout)
|
||||
ctx, stop := context.WithTimeout(ctx, waitOf(within))
|
||||
defer stop()
|
||||
c.calls.mu.Lock()
|
||||
if c.calls.pending == nil {
|
||||
|
||||
@@ -216,3 +216,30 @@ func TestAFallBackForWantOfAResponderIsSaidOncePerSubject(t *testing.T) {
|
||||
t.Errorf("the fall-back was said %d times, want once: %v", n, said)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §760: a call waits the bound its verb's definition names, and ends there with the timeout; the caller of a verb whose
|
||||
// definition names none waits the tool runner's own bound, RequestTimeout.
|
||||
func TestACallWaitsTheBoundItsVerbNamesAndTimesOutThere(t *testing.T) {
|
||||
url := grantedBus(t, map[string][]string{"anchor.node-tools": {"_INBOX.>"}, "shanks.node-tools": {"mesh.mod.*.tool.>"}})
|
||||
server := dial(t, url, "node-tools", "anchor", "anchor.node-tools")
|
||||
release := make(chan struct{})
|
||||
stop, err := server.HandleSubjectCalled("mesh.mod.ledger.tool.slow", func(string, json.RawMessage) (any, error) {
|
||||
<-release
|
||||
return map[string]any{}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
defer close(release)
|
||||
server.Flush()
|
||||
asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools")
|
||||
began := time.Now()
|
||||
_, err = asker.AskWithin("ledger.slow", map[string]any{}, "", 200*time.Millisecond)
|
||||
if err == nil || err.Error() != "timeout" {
|
||||
t.Fatalf("a call whose verb names a bound the holder outlasts: %v, want the timeout", err)
|
||||
}
|
||||
if took := time.Since(began); took > bus.RequestTimeout/2 {
|
||||
t.Errorf("the call waited %s, want its own bound of 200ms, not the tool runner's %s", took, bus.RequestTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,11 @@ const FileField = "file"
|
||||
// counts with buscalls.SeatChunks, each acknowledged before the next, and answers the verb's own answer with the whole
|
||||
// file added as FileField. The asking bundle receives no chunk itself.
|
||||
func (c *Conn) AskAsChunked(module, key string, body any) (Answered, error) {
|
||||
return c.AskAsChunkedWithin(module, key, body, 0)
|
||||
}
|
||||
|
||||
// AskAsChunkedWithin is AskAsChunked that waits within for the verb's answer (design 47 H §760); zero waits RequestTimeout.
|
||||
func (c *Conn) AskAsChunkedWithin(module, key string, body any, within time.Duration) (Answered, error) {
|
||||
// The verb may be asked of the machine that ran the ask (`@<node>`, A-254): the chunk subjects name the seat, the verb and
|
||||
// the call alone.
|
||||
name, _, _ := strings.Cut(key, "@")
|
||||
@@ -52,7 +57,7 @@ func (c *Conn) AskAsChunked(module, key string, body any) (Answered, error) {
|
||||
return Answered{}, fmt.Errorf("the chunks of %s cannot be heard: %w", key, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
answered, err := c.AskAs(module, key, args)
|
||||
answered, err := c.AskAsWithin(module, key, args, within)
|
||||
if err != nil {
|
||||
return answered, err
|
||||
}
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -174,64 +176,50 @@ func (c *Conn) StatePage(module, name, after string, maxBytes int, budget time.D
|
||||
each := func() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), StateTimeout)
|
||||
}
|
||||
var keys []string
|
||||
var get func(key string) (*StateEntry, error)
|
||||
// Every current value is read in one stream per bucket (currentOf), never one bus request per key: a page of the
|
||||
// assignments (about 330 keys) read key by key took 13 s from a node far from the bus (2026-11-14).
|
||||
current := map[string]*StateEntry{}
|
||||
if s.PerNode {
|
||||
fam := c.family(s)
|
||||
if keys, err = fam.Keys(ctx); err != nil {
|
||||
keys, err := fam.Keys(ctx)
|
||||
if err != nil {
|
||||
return StatePage{}, err
|
||||
}
|
||||
buckets := map[string]jetstream.KeyValue{} // each node's bucket bound once
|
||||
get = func(key string) (*StateEntry, error) {
|
||||
node, err := pernode.NodeOf(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
nodes := map[string]bool{}
|
||||
for _, k := range keys {
|
||||
if node, err := pernode.NodeOf(k); err == nil {
|
||||
nodes[node] = true
|
||||
}
|
||||
kv, bound := buckets[node]
|
||||
if !bound {
|
||||
octx, ocancel := each() // the open is a bus request in the loop: its own deadline, never ctx's
|
||||
kv, err = fam.Open(octx, node)
|
||||
ocancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buckets[node] = kv
|
||||
}
|
||||
for node := range nodes {
|
||||
octx, ocancel := each() // each node's bucket is a bus request of its own: its own deadline
|
||||
kv, err := fam.Open(octx, node)
|
||||
ocancel()
|
||||
if err != nil {
|
||||
return StatePage{}, err
|
||||
}
|
||||
if kv == nil {
|
||||
return nil, nil
|
||||
}
|
||||
gctx, gcancel := each()
|
||||
defer gcancel()
|
||||
e, err := kv.Get(gctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
continue
|
||||
}
|
||||
wctx, wcancel := each()
|
||||
got, err := currentOf(wctx, kv)
|
||||
wcancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return StatePage{}, err
|
||||
}
|
||||
return &StateEntry{Key: e.Key(), Value: valueOf(e.Value()), Revision: e.Revision(), At: e.Created().UTC()}, nil
|
||||
maps.Copy(current, got)
|
||||
}
|
||||
} else {
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return StatePage{}, err
|
||||
}
|
||||
if keys, err = keysOf(ctx, kv); err != nil {
|
||||
if current, err = currentOf(ctx, kv); err != nil {
|
||||
return StatePage{}, err
|
||||
}
|
||||
get = func(key string) (*StateEntry, error) {
|
||||
gctx, gcancel := each()
|
||||
defer gcancel()
|
||||
e, err := kv.Get(gctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &StateEntry{Key: e.Key(), Value: valueOf(e.Value()), Revision: e.Revision(), At: e.Created().UTC()}, nil
|
||||
}
|
||||
}
|
||||
keys := slices.Collect(maps.Keys(current))
|
||||
get := func(key string) (*StateEntry, error) { return current[key], nil }
|
||||
sort.Strings(keys)
|
||||
page := StatePage{Entries: []StateEntry{}}
|
||||
started := PageClock()
|
||||
@@ -719,3 +707,28 @@ func providerReference(v any) bool {
|
||||
|
||||
// referenceName is a node's or a module's name as mesh-sdk names them: lower case, digits and dashes.
|
||||
var referenceName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
// currentOf is every current value of a bucket, read in one stream (a watch of every key, deletes left out, ended at its
|
||||
// first nil): the values a page answers, without a bus request per key.
|
||||
func currentOf(ctx context.Context, kv jetstream.KeyValue) (map[string]*StateEntry, error) {
|
||||
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
out := map[string]*StateEntry{}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("the values of %s are not all read: %w", kv.Bucket(), ctx.Err())
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return out, nil
|
||||
}
|
||||
if e == nil {
|
||||
return out, nil
|
||||
}
|
||||
out[e.Key()] = &StateEntry{Key: e.Key(), Value: valueOf(e.Value()), Revision: e.Revision(), At: e.Created().UTC()}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
)
|
||||
|
||||
// **A watch whose current values take longer than the bound to hand over still holds** (novox/hq issue
|
||||
// 1726). The gitea module watches the checker's owed: one key per pull request head, each a round trip
|
||||
// 1726). The gitea module watches another module's state (the assigner's pins stand for it): one key per pull request head, each a round trip
|
||||
// to the bundle. With a few hundred keys the hand-over outlasted StateTimeout at every attempt, the
|
||||
// watch was refused, and the newest keys, handed over last, never reached mesh/check: a group's passing
|
||||
// verdict was never written on its members' heads. The bound is on progress: a watch fails only when
|
||||
@@ -45,7 +45,7 @@ func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var current, live []string
|
||||
// Each value takes 40 ms to be taken: 33 values take about 1.3 s, three times the bound.
|
||||
stop, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(ch bus.StateChange) error {
|
||||
stop, err := conn.StateWatch("gitea", "mesh-delivery-assigner.pins", "", func(ch bus.StateChange) error {
|
||||
if ch.Current {
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
}
|
||||
@@ -127,12 +127,12 @@ func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) {
|
||||
}
|
||||
var stop func()
|
||||
if tc.refused {
|
||||
if _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err == nil {
|
||||
if _, err := conn.StateWatch("gitea", "mesh-delivery-assigner.pins", "", deliver); err == nil {
|
||||
t.Fatal("the watch whose bundle took nothing was not refused")
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
if stop, err = conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err != nil {
|
||||
if stop, err = conn.StateWatch("gitea", "mesh-delivery-assigner.pins", "", deliver); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
@@ -173,7 +173,7 @@ func TestAWatchIsRefusedWhenNoCurrentValueIsTakenWithinTheBound(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
t.Cleanup(func() { close(release) })
|
||||
began := time.Now()
|
||||
_, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(bus.StateChange) error {
|
||||
_, err := conn.StateWatch("gitea", "mesh-delivery-assigner.pins", "", func(bus.StateChange) error {
|
||||
<-release
|
||||
return nil
|
||||
})
|
||||
@@ -229,16 +229,16 @@ func TestAMembershipTheGrantsPartWroteWrappedIsServed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// owedWatcher is the gitea module's view of the checker's owed: the bucket, and a runtime connection
|
||||
// that serves gitea with the owed issued for reading.
|
||||
// owedWatcher is the gitea module's view of another module's state: the bucket, and a runtime connection
|
||||
// that serves gitea with that state issued for reading.
|
||||
func owedWatcher(t *testing.T) (*bus.Conn, interface {
|
||||
Put(string, []byte) (uint64, error)
|
||||
}) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
kv := mesh.Bucket(t, "mesh-delivery-checker_owed")
|
||||
kv := mesh.Bucket(t, "mesh-delivery-assigner_pins")
|
||||
gitea := mt.MembershipOf("gitea", "anchor", false, nil)
|
||||
gitea.State = []bus.StateIssued{{Name: "mesh-delivery-checker.owed", Bucket: "mesh-delivery-checker_owed"}}
|
||||
gitea.State = []bus.StateIssued{{Name: "mesh-delivery-assigner.pins", Bucket: "mesh-delivery-assigner_pins"}}
|
||||
mesh.Issue(t, gitea)
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
|
||||
@@ -269,6 +269,8 @@ func (h *hearing) reconcile() {
|
||||
if m := h.c.Membership(h.module); m != nil {
|
||||
for _, heard := range m.Hears {
|
||||
if heard.Stream != "" && heard.Consumer != "" {
|
||||
// The runner's own durable beside the program's (A-397).
|
||||
heard.Consumer += ToolsSuffix
|
||||
want[heard] = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -927,10 +927,23 @@ func schemaWithoutNode(raw json.RawMessage) map[string]any {
|
||||
}
|
||||
|
||||
// answerText is a discovery verb's answer as MCP content: JSON, indented.
|
||||
// withinOf is how long a call of the target waits: the `timeout` its seat verb's definition names (design 47 H §760), else the
|
||||
// runner's own bound, RequestTimeout.
|
||||
func (s *Surface) withinOf(t target) time.Duration {
|
||||
if t.Seat && s.Definitions != nil {
|
||||
for _, v := range s.Definitions()[t.Tool.Module].Verbs {
|
||||
if v.Name == t.Tool.Name && v.Timeout != nil {
|
||||
return time.Duration(*v.Timeout) * time.Second
|
||||
}
|
||||
}
|
||||
}
|
||||
return bus.RequestTimeout
|
||||
}
|
||||
|
||||
// callFailed is why a call failed, as mesh_call says it: a marked tool or verb that reaches no responder names `withheld`
|
||||
// and the grants part's read (design 46 7-approval-mark §6), and a target known only from an earlier round says so.
|
||||
func (s *Surface) callFailed(t target, err error) string {
|
||||
why := whyItFailed(t.Address, err)
|
||||
why := whyItFailedWithin(t.Address, err, s.withinOf(t))
|
||||
if ok, _ := s.markedTool(t); ok && errors.Is(err, bus.ErrNoResponders) {
|
||||
// A marked subject withheld from a machine whose tool runner does not honour the mark (§6).
|
||||
why += "; it needs the operator's approval, so it may be withheld where the machine's tool runner does not state " +
|
||||
@@ -1147,7 +1160,7 @@ func (s *Surface) discover(ctx context.Context, name string, args map[string]any
|
||||
}
|
||||
var got bus.Answered
|
||||
if t.Seat {
|
||||
got, err = s.conn.Ask(t.Key, callArgs, "")
|
||||
got, err = s.conn.AskWithin(t.Key, callArgs, "", s.withinOf(t))
|
||||
} else {
|
||||
got, err = callTool(s.conn, t.Key, callArgs, nil, x.Listing)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
@@ -120,6 +121,11 @@ var (
|
||||
|
||||
// whyItFailed says why a call failed, so the remedy is in the words.
|
||||
func whyItFailed(key string, err error) string {
|
||||
return whyItFailedWithin(key, err, bus.RequestTimeout)
|
||||
}
|
||||
|
||||
// whyItFailedWithin is whyItFailed for a call that waited within, the bound its verb's definition names or the runner's own.
|
||||
func whyItFailedWithin(key string, err error, within time.Duration) string {
|
||||
if err == nil {
|
||||
err = errors.New("failed")
|
||||
}
|
||||
@@ -139,7 +145,7 @@ func whyItFailed(key string, err error) string {
|
||||
case timedOut.MatchString(msg):
|
||||
// **A timeout is not a failure** (novox/hq issue 265). Something took the call, so it may
|
||||
// still be running and may already have done what was asked: a push said this while it pushed.
|
||||
said := key + " did not answer within " + bus.RequestTimeout.String() + ". That is not a failure: " +
|
||||
said := key + " did not answer within " + within.String() + ". That is not a failure: " +
|
||||
"something is serving it, so it may still be running and may already have done what was " +
|
||||
"asked — check before repeating a call that changes the mesh."
|
||||
return said
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -65,6 +66,9 @@ type Surface struct {
|
||||
Record func(args map[string]any) error
|
||||
// Approvals is this machine's approval gate, read by mesh_approvals and mesh_describe; nil when the runner has none.
|
||||
Approvals Approvals
|
||||
// Definitions is the seat definitions this runner holds, by seat, which name a verb's `timeout` (design 47 H §760); nil when it
|
||||
// holds none, and a call then waits the runner's own bound.
|
||||
Definitions func() map[string]seatdef.Definition
|
||||
}
|
||||
|
||||
// MethodSeats is the console's JSON-RPC method beside MCP's that answers every seat's verbs and holders, in the shape
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -74,6 +75,27 @@ func TestARefusalIsSaidOnlyForTheBussOwn(t *testing.T) {
|
||||
// the grammar's example, the handshake's instructions and a timeout's remedy named the controller's verbs.
|
||||
func TestATimeoutIsNotAFailure(t *testing.T) {
|
||||
got := whyItFailed("seat:mesh-delivery.deliveries", errors.New("timeout"))
|
||||
// Design 47 H §760: mesh_call waits the `timeout` the verb's definition names and a timeout says that bound; a verb naming none,
|
||||
// a module's tool and a surface holding no definitions wait and say the runner's own 30s.
|
||||
forty := 40
|
||||
s := &Surface{Definitions: func() map[string]seatdef.Definition {
|
||||
return map[string]seatdef.Definition{"mesh-broker": {Verbs: []seatdef.Verb{{Name: "rates", Timeout: &forty}, {Name: "plain"}}}}
|
||||
}}
|
||||
for _, row := range []struct {
|
||||
t target
|
||||
want string
|
||||
}{
|
||||
{target{Address: "mesh-broker.rates", Seat: true, Tool: Tool{Module: "mesh-broker", Name: "rates"}}, "within 40s"},
|
||||
{target{Address: "mesh-broker.plain", Seat: true, Tool: Tool{Module: "mesh-broker", Name: "plain"}}, "within 30s"},
|
||||
{target{Address: "mesh-broker.rates", Tool: Tool{Module: "mesh-broker", Name: "rates"}}, "within 30s"},
|
||||
} {
|
||||
if said := s.callFailed(row.t, errors.New("timeout")); !strings.Contains(said, row.want) {
|
||||
t.Errorf("%s: a timeout says %q, want it to say %q", row.t.Address, said, row.want)
|
||||
}
|
||||
}
|
||||
if said := (&Surface{}).callFailed(target{Seat: true, Tool: Tool{Module: "mesh-broker", Name: "rates"}}, errors.New("timeout")); !strings.Contains(said, "within 30s") {
|
||||
t.Errorf("a surface that holds no definitions says %q, want 30s", said)
|
||||
}
|
||||
for _, want := range []string{"not a failure", "may already have done"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("a seat's timeout does not say %q: %s", want, got)
|
||||
|
||||
@@ -58,18 +58,19 @@ func (s Spec) Name() string { return AssignerSeat + "." + s.State }
|
||||
|
||||
// Views is the first views: the assigner's two, and the catalog half's per watched branch.
|
||||
type Views struct {
|
||||
logf func(string, ...any)
|
||||
mu sync.Mutex
|
||||
followers map[string]*views.Follower // by `<owner>.<state>`
|
||||
byState map[string]*views.Follower // by the state an event names, the owners' views families
|
||||
adopted map[string]*adoption // by `<owner>.<state>`, the views the branch follower adopted
|
||||
hearers map[string][]*adoption // event key `<seat>.<event>` -> the adopted views that hear it
|
||||
branches *meshcatalogclient.Branches
|
||||
holder *holding // the module that claims the assigner seat, once the catalog half names it
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
deps Deps
|
||||
logf func(string, ...any)
|
||||
mu sync.Mutex
|
||||
followers map[string]*views.Follower // by `<owner>.<state>`
|
||||
byState map[string]*views.Follower // by the state an event names, the owners' views families
|
||||
adopted map[string]*adoption // by `<owner>.<state>`, the views the branch follower adopted
|
||||
unheardKeys map[string]bool // event keys said as heard by no view
|
||||
hearers map[string][]*adoption // event key `<seat>.<event>` -> the adopted views that hear it
|
||||
branches *meshcatalogclient.Branches
|
||||
holder *holding // the module that claims the assigner seat, once the catalog half names it
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
deps Deps
|
||||
// heard is closed and made again at every event heard, so a wait for the picture wakes on events, never on a timer.
|
||||
heard chan struct{}
|
||||
// onKey are the hearers of one key of an owner's state, by `<state> <key>` (OnSeatHolder).
|
||||
@@ -207,7 +208,7 @@ func StartWith(parent context.Context, d Deps, logf func(string, ...any)) (*View
|
||||
func (v *Views) hear(env bus.Envelope) error {
|
||||
defer v.heardOne()
|
||||
v.mu.Lock()
|
||||
hs := append([]*adoption(nil), v.hearers[env.Key]...)
|
||||
hs := v.hearersOf(env.Key)
|
||||
v.mu.Unlock()
|
||||
if len(hs) > 0 { // an event of the catalog half or the gate's source, heard by the views the branch follower adopted
|
||||
for _, a := range hs {
|
||||
@@ -219,6 +220,7 @@ func (v *Views) hear(env bus.Envelope) error {
|
||||
}
|
||||
ev, err := views.DecodeEvent(env.Body)
|
||||
if err != nil {
|
||||
v.unheard(env.Key)
|
||||
return nil // not a views event: nothing to hold, nothing to retry
|
||||
}
|
||||
v.mu.Lock()
|
||||
@@ -235,6 +237,37 @@ func (v *Views) hear(env bus.Envelope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// hearersOf are the adopted views that hear the event key `<seat>.<event>` or, as its subject carries further tokens
|
||||
// (`<event>.<branch>`, the mesh-catalog seat's catalog-revision), the key's shorter prefixes: the way mesh-sdk's views reader
|
||||
// finds its adopted hears. The caller holds v.mu.
|
||||
func (v *Views) hearersOf(key string) []*adoption {
|
||||
for k := key; k != ""; {
|
||||
if hs := v.hearers[k]; len(hs) > 0 {
|
||||
return append([]*adoption(nil), hs...)
|
||||
}
|
||||
i := strings.LastIndex(k, ".")
|
||||
if i <= 0 {
|
||||
break
|
||||
}
|
||||
k = k[:i]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unheard says once per key an event no adopted view and no owner's state hears, never dropping it silently.
|
||||
func (v *Views) unheard(key string) {
|
||||
v.mu.Lock()
|
||||
if v.unheardKeys == nil {
|
||||
v.unheardKeys = map[string]bool{}
|
||||
}
|
||||
seen := v.unheardKeys[key]
|
||||
v.unheardKeys[key] = true
|
||||
v.mu.Unlock()
|
||||
if !seen {
|
||||
v.logf("[mesh-tools] an event %s is heard by no view of this tool runner: it is dropped (said once per key)", key)
|
||||
}
|
||||
}
|
||||
|
||||
// finding writes or clears the runner's finding of a first view unanswered past its bound, through its writer (design 47 G A-28).
|
||||
func (v *Views) finding(f condseat.Finding, tool string, raised bool) {
|
||||
if v.deps.Findings == nil {
|
||||
|
||||
@@ -156,10 +156,14 @@ func TestTheRunnerHoldsTheFourStatesFromTheOwnersFirstViewsAndTheirEvents(t *tes
|
||||
o.catalog, o.modules = 4, []string{"web", "db", fixtureHolder}
|
||||
o.mu.Unlock()
|
||||
ev, _ := json.Marshal(catalogseat.CatalogRevisionEvent{V: 1, Branch: "main", Kind: meshcatalogclient.KindTrunk, Revision: 4})
|
||||
if err := o.deliver(bus.Envelope{Key: catalogseat.Seat + "." + catalogseat.EventCatalogRevision, Body: ev}); err != nil {
|
||||
if err := o.deliver(bus.Envelope{Key: catalogseat.Seat + "." + catalogseat.EventCatalogRevision + ".main", Body: ev}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "the catalog-revision was not taken", func() bool { m, err := v.Modules(); return err == nil && len(m) == 3 })
|
||||
// An event no view hears is said once per key, not dropped silently.
|
||||
for i := 0; i < 2; i++ {
|
||||
_ = o.deliver(bus.Envelope{Key: "some-seat.unknown-event.x", Body: json.RawMessage(`"x"`)})
|
||||
}
|
||||
// And the halves by branch, with their revision, which mesh_wait names checking an emitter (design 34 every-event §14).
|
||||
if h, err := v.Halves(); err != nil || h["main"].Revision.Revision != 4 || len(h["main"].Entries) != 3 {
|
||||
t.Errorf("the halves: %+v, %v", h, err)
|
||||
@@ -207,6 +211,9 @@ func TestTheRunnerHoldsTheFourStatesFromTheOwnersFirstViewsAndTheirEvents(t *tes
|
||||
}
|
||||
lmu.Lock()
|
||||
defer lmu.Unlock()
|
||||
if n := strings.Count(strings.Join(logs, "\n"), "heard by no view"); n != 1 {
|
||||
t.Errorf("an unheard event was said %d times, want once: %v", n, logs)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logs, "\n"), "a gap in") {
|
||||
t.Errorf("the gap was not said in the log: %v", logs)
|
||||
}
|
||||
|
||||
@@ -46,6 +46,20 @@ type Tool struct {
|
||||
// Run calls the tool for caller: the bus user the call's subject named, or "" when it named none. A
|
||||
// caller is handed to the bundle in the call's `_meta` (MetaCaller), never among its arguments.
|
||||
Run func(caller string, args json.RawMessage) (json.RawMessage, error)
|
||||
// RunWithin is Run that waits within for the bundle's answer, the `timeout` the verb's seat definition names (design 47 H §760,
|
||||
// H-A-43); zero waits CallTimeout. A tool the runtime serves itself has none: Run is its call.
|
||||
RunWithin func(caller string, args json.RawMessage, within time.Duration) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// Within is the tool as called for a verb whose definition names a timeout: its Run waits within for the bundle's answer. A zero
|
||||
// within, and a tool that has no RunWithin, are the tool as it is.
|
||||
func (t Tool) Within(within time.Duration) Tool {
|
||||
if within <= 0 || t.RunWithin == nil {
|
||||
return t
|
||||
}
|
||||
run := t.RunWithin
|
||||
t.Run = func(caller string, args json.RawMessage) (json.RawMessage, error) { return run(caller, args, within) }
|
||||
return t
|
||||
}
|
||||
|
||||
// MetaCaller is where in a `tools/call`'s `_meta` the runtime names the caller it took from the subject the
|
||||
@@ -799,13 +813,17 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
input = json.RawMessage("{}")
|
||||
}
|
||||
tool := Tool{Name: name, Description: t.Description, Input: input,
|
||||
Run: func(caller string, args json.RawMessage) (json.RawMessage, error) {
|
||||
Run: nil, RunWithin: func(caller string, args json.RawMessage, within time.Duration) (json.RawMessage, error) {
|
||||
if len(args) == 0 || string(args) == "null" {
|
||||
args = json.RawMessage("{}")
|
||||
}
|
||||
call, end := mesh.Begin(caller)
|
||||
defer end()
|
||||
res, err := asking("tools/call", callParams(full, caller, call, args), CallTimeout)
|
||||
bound := CallTimeout
|
||||
if within > 0 {
|
||||
bound = within
|
||||
}
|
||||
res, err := asking("tools/call", callParams(full, caller, call, args), bound)
|
||||
if err != nil {
|
||||
if errors.Is(err, errNoAnswer) {
|
||||
// Its caller is gone by now and nobody else would hear of it (issue 1966).
|
||||
@@ -841,6 +859,9 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
b, _ := json.Marshal(text)
|
||||
return b, nil
|
||||
}}
|
||||
tool.Run = func(caller string, args json.RawMessage) (json.RawMessage, error) {
|
||||
return tool.RunWithin(caller, args, 0)
|
||||
}
|
||||
if _, seen := groups[under]; !seen {
|
||||
order = append(order, under)
|
||||
}
|
||||
|
||||
@@ -82,7 +82,8 @@ while IFS= read -r line; do
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) echo started >&2; printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[{"name":"t","description":"d"}]}}\n' "$id" ;;
|
||||
*'"method":"tools/call"'*) [ "$MODE" = long ] && printf '%4096s\n' x ;;
|
||||
*'"method":"tools/call"'*) [ "$MODE" = long ] && printf '%4096s\n' x
|
||||
[ "$MODE" = late ] && { sleep 0.3; printf '{"jsonrpc":"2.0","id":%s,"result":{"content":[{"type":"text","text":"{}"}]}}\n' "$id"; } ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
@@ -150,3 +151,28 @@ func TestAnOverlongLineAndAnUnansweredCallAreSaid(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §760: a verb served from a bundle waits the `timeout` its definition names, not the runner's own bound: a bundle
|
||||
// that answers after CallTimeout is answered through Within and fails through Run.
|
||||
func TestAToolWithinWaitsTheBoundItsVerbNames(t *testing.T) {
|
||||
oldCall := CallTimeout
|
||||
CallTimeout = 100 * time.Millisecond
|
||||
t.Cleanup(func() { CallTimeout = oldCall })
|
||||
dir := t.TempDir()
|
||||
entry := filepath.Join(dir, "bundle")
|
||||
if err := os.WriteFile(entry, []byte(callingBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l, err := Start("caller", entry, append(os.Environ(), "MODE=late"), &refusingState{}, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
tool := l.Registrations[0].Tools[0]
|
||||
if _, err := tool.Run("caller-1", nil); err == nil || !strings.Contains(err.Error(), "did not answer") {
|
||||
t.Fatalf("a call that waits the runner's own bound of 100ms: %v, want it unanswered", err)
|
||||
}
|
||||
if got, err := tool.Within(5*time.Second).Run("caller-1", nil); err != nil || string(got) != "{}" {
|
||||
t.Fatalf("a call that waits the verb's 5s: %s, %v, want its answer", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,7 +303,7 @@ func (l *Logs) All() string { return strings.Join(l.lines, "\n") }
|
||||
// redelivery in seconds rather than the mesh's minutes.
|
||||
func (m *Mesh) Consumer(t *testing.T, node, module string, filters []string, ackWait time.Duration) {
|
||||
t.Helper()
|
||||
cfg := &nats.ConsumerConfig{Durable: node + "_" + module, AckPolicy: nats.AckExplicitPolicy,
|
||||
cfg := &nats.ConsumerConfig{Durable: node + "_" + module + "_tools", AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: ackWait, MaxDeliver: 10, DeliverPolicy: nats.DeliverNewPolicy}
|
||||
if len(filters) == 1 {
|
||||
cfg.FilterSubject = filters[0]
|
||||
@@ -340,7 +340,7 @@ func (m *Mesh) Publish(t *testing.T, subject string, body any) {
|
||||
// delivered.
|
||||
func (m *Mesh) Pending(t *testing.T, node, module string) (ackPending, notDelivered uint64) {
|
||||
t.Helper()
|
||||
info, err := m.js.ConsumerInfo("EVENTS", node+"_"+module)
|
||||
info, err := m.js.ConsumerInfo("EVENTS", node+"_"+module+"_tools")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -430,7 +430,7 @@ func (m *Mesh) ModuleStream(t *testing.T, module, name string) {
|
||||
// ConsumerOn makes a module's durable consumer on a machine on any stream, as Consumer does on EVENTS.
|
||||
func (m *Mesh) ConsumerOn(t *testing.T, stream, node, module string, filters []string, ackWait time.Duration) {
|
||||
t.Helper()
|
||||
cfg := &nats.ConsumerConfig{Durable: node + "_" + module, AckPolicy: nats.AckExplicitPolicy,
|
||||
cfg := &nats.ConsumerConfig{Durable: node + "_" + module + "_tools", AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: ackWait, MaxDeliver: 10, DeliverPolicy: nats.DeliverNewPolicy}
|
||||
if len(filters) == 1 {
|
||||
cfg.FilterSubject = filters[0]
|
||||
@@ -445,7 +445,7 @@ func (m *Mesh) ConsumerOn(t *testing.T, stream, node, module string, filters []s
|
||||
// PendingOn is what a module's consumer on a stream still holds, as Pending is on EVENTS.
|
||||
func (m *Mesh) PendingOn(t *testing.T, stream, node, module string) (ackPending, notDelivered uint64) {
|
||||
t.Helper()
|
||||
info, err := m.js.ConsumerInfo(stream, node+"_"+module)
|
||||
info, err := m.js.ConsumerInfo(stream, node+"_"+module+"_tools")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -466,7 +466,7 @@ func (m *Mesh) Messages(t *testing.T, stream string) uint64 {
|
||||
// it cannot narrow is withdrawn (novox/hq ADR 0363 §6).
|
||||
func (m *Mesh) RemoveConsumer(t *testing.T, stream, node, module string) {
|
||||
t.Helper()
|
||||
if err := m.js.DeleteConsumer(stream, node+"_"+module); err != nil {
|
||||
if err := m.js.DeleteConsumer(stream, node+"_"+module+"_tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
@@ -177,15 +178,18 @@ func (c *consumers) objectsGet(module string, r buscalls.ObjectsNameRequest) (an
|
||||
if st == nil {
|
||||
return nil, fmt.Errorf("the runner has no bus to read %s of %s from", r.Name, module)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), bus.StateTimeout)
|
||||
defer cancel()
|
||||
ctx, cancel, opened := openingContext()
|
||||
info, rd, found, err := st.ObjectsOpen(ctx, module, r)
|
||||
opened()
|
||||
if err != nil || !found {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
if info.Size > buscalls.ChunkBytes {
|
||||
return c.openGet(module, info, rd, sessionDigest(info.Digest), false)
|
||||
// the reader reads on ctx after this call returns: it ends with the session, not with this call
|
||||
return c.openGet(module, info, cancelOnClose{rd, cancel}, sessionDigest(info.Digest), false)
|
||||
}
|
||||
defer cancel()
|
||||
defer rd.Close()
|
||||
data, err := io.ReadAll(rd)
|
||||
if err != nil {
|
||||
@@ -208,11 +212,32 @@ func (c *consumers) sentGet(module string, r buscalls.ObjectsSentGetRequest) (an
|
||||
if st == nil {
|
||||
return nil, fmt.Errorf("the runner has no bus to read %s from", r.Object)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), bus.StateTimeout)
|
||||
defer cancel()
|
||||
ctx, cancel, opened := openingContext()
|
||||
info, rd, err := st.SentOpen(ctx, r)
|
||||
opened()
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
return c.openGet(module, info, rd, sessionDigest(r.Digest), true)
|
||||
return c.openGet(module, info, cancelOnClose{rd, cancel}, sessionDigest(r.Digest), true)
|
||||
}
|
||||
|
||||
// openingContext is the context an object is opened on: it is bound at bus.StateTimeout while opening, and once opened (the
|
||||
// returned function) is bound no more: a reader of an object that outlives the call reads on it until its session ends.
|
||||
func openingContext() (context.Context, context.CancelFunc, func()) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t := time.AfterFunc(bus.StateTimeout, cancel)
|
||||
return ctx, cancel, func() { t.Stop() }
|
||||
}
|
||||
|
||||
// cancelOnClose ends the context an object's reader reads on when the reader is closed: at the end of its get session.
|
||||
type cancelOnClose struct {
|
||||
io.ReadCloser
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (c cancelOnClose) Close() error {
|
||||
err := c.ReadCloser.Close()
|
||||
c.cancel()
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2,16 +2,18 @@ package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/approval"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
@@ -66,15 +68,14 @@ func TestACallOnTheCallerNamedSubjectReachesTheModuleWithItsCaller(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
// deskDefined is the seat list's definition of the fixture seat node-desk, its verb who taking its class's default.
|
||||
// deskDefined is the seat list's definition of the fixture seat node-desk, its verb who.
|
||||
var deskDefined = WithSeatDefinitions(func() map[string]seatdef.Definition {
|
||||
return map[string]seatdef.Definition{"node-desk": {Seat: "node-desk", Verbs: []seatdef.Verb{{Name: "who"}}}}
|
||||
})
|
||||
|
||||
// The subject that names no caller is still served for a module's tool, for one release (retired by hq issue 464), and says
|
||||
// so: the module is named no caller, whatever the body claims. A seat's verb refuses it before the bundle runs (design 47 G
|
||||
// §610; lead answers A-165, A-179), and refuses every caller while the seat's definition is unread, names no such verb or
|
||||
// opens it to none.
|
||||
// so: the module is named no caller, whatever the body claims. A seat's verb answers every caller, an unproven one too (design 47
|
||||
// H §752).
|
||||
func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", true, map[string][]string{"node-desk": {"who"}}))
|
||||
@@ -102,9 +103,10 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) {
|
||||
}
|
||||
return err
|
||||
})
|
||||
// Every caller is answered, an unproven one too (design 47 H §752).
|
||||
if strings.HasPrefix(subject, "mesh.seat.") {
|
||||
if !strings.Contains(reply.Error, "node-desk.who takes no unproven caller") || reply.Result.Call != "" {
|
||||
t.Errorf("%s: an unproven caller of a seat's verb was answered %+v (error %q), want refused before the bundle", subject, reply.Result, reply.Error)
|
||||
if reply.Error != "" {
|
||||
t.Errorf("%s: an unproven caller of a seat's verb was refused: %q, want answered", subject, reply.Error)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -112,21 +114,6 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) {
|
||||
t.Errorf("%s: the module was named caller %q (error %q), want none", subject, reply.Result.Caller, reply.Error)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
defs map[string]seatdef.Definition
|
||||
says string
|
||||
}{
|
||||
{nil, "is not yet read"},
|
||||
{map[string]seatdef.Definition{"node-desk": {Seat: "node-desk"}}, "names no verb who"},
|
||||
{map[string]seatdef.Definition{"node-desk": {Seat: "node-desk", Verbs: []seatdef.Verb{{Name: "who", Callers: &seatdef.Callers{List: []string{}}}}}}, "open to no caller"},
|
||||
{map[string]seatdef.Definition{"node-desk": {Seat: "node-desk", Verbs: []seatdef.Verb{{Name: "who", Class: seatdef.ClassAct}}}}, ""},
|
||||
} {
|
||||
defs := c.defs
|
||||
err := (&consumers{definitions: func() map[string]seatdef.Definition { return defs }}).admitted("node-desk", "who", "workstation.node-tools")
|
||||
if c.says == "" && err != nil || c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)) {
|
||||
t.Errorf("a proven caller with the definitions %v: %v, want %q", c.defs, err, c.says)
|
||||
}
|
||||
}
|
||||
// Design 47 G A-249: a bundle's seat call is received by the runner in chunks only for a verb whose definition says `chunks`,
|
||||
// of any machine: a verb a node holds is asked of the node that ran the ask (A-254).
|
||||
t.Run("OnlyAChunkedSeatVerbIsAskedInChunks", func(t *testing.T) {
|
||||
@@ -134,12 +121,12 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) {
|
||||
return map[string]seatdef.Definition{"mesh-catalog": {Verbs: []seatdef.Verb{{Name: "input", Chunks: true}, {Name: "entry"}}}}
|
||||
}}
|
||||
for key, want := range map[string]bool{
|
||||
"seat:mesh-catalog.input": true,
|
||||
"seat:mesh-catalog.entry": false,
|
||||
"seat:mesh-catalog.input@home-server": true,
|
||||
"seat:mesh-catalog.entry@home-server": false,
|
||||
"seat:mesh-unread.input": false,
|
||||
"mesh-delivery-indexer.indexer": false,
|
||||
"seat:mesh-catalog.input": true,
|
||||
"seat:mesh-catalog.entry": false,
|
||||
"seat:mesh-catalog.input@home-server": true,
|
||||
"seat:mesh-catalog.entry@home-server": false,
|
||||
"seat:mesh-unread.input": false,
|
||||
"mesh-delivery-indexer.indexer": false,
|
||||
} {
|
||||
if got := c.chunked(key); got != want {
|
||||
t.Errorf("chunked(%s) = %v, want %v", key, got, want)
|
||||
@@ -149,6 +136,108 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) {
|
||||
t.Error("a runner that holds no seat definitions asks no chunks")
|
||||
}
|
||||
})
|
||||
// Design 47 H §760: a bundle's ask waits the `timeout` its verb's definition names, whole seconds; a verb that names none, a
|
||||
// seat the definitions do not hold, a module's tool and a runner that holds no definitions wait the runner's own bound (zero).
|
||||
t.Run("AVerbsTimeoutIsTheBoundItsAskWaits", func(t *testing.T) {
|
||||
forty, six := 40, 600
|
||||
defs := map[string]seatdef.Definition{"mesh-broker": {Verbs: []seatdef.Verb{{Name: "rates", Timeout: &forty}, {Name: "slow", Chunks: true, Timeout: &six}, {Name: "plain"}}}}
|
||||
for _, row := range []struct {
|
||||
key string
|
||||
want time.Duration
|
||||
way string
|
||||
}{
|
||||
{"seat:mesh-broker.rates", 40 * time.Second, "plain"},
|
||||
{"seat:mesh-broker.rates@home-server", 40 * time.Second, "plain"},
|
||||
{"seat:mesh-broker.slow", 600 * time.Second, "chunked"},
|
||||
{"seat:mesh-broker.plain", 0, "plain"},
|
||||
{"seat:mesh-unread.rates", 0, "plain"},
|
||||
{"mesh-delivery-indexer.indexer", 0, "plain"},
|
||||
} {
|
||||
var got time.Duration
|
||||
var way string
|
||||
c := &consumers{definitions: func() map[string]seatdef.Definition { return defs },
|
||||
askPlain: func(_, _ string, _ any, within time.Duration) (bus.Answered, error) {
|
||||
got, way = within, "plain"
|
||||
return bus.Answered{}, nil
|
||||
},
|
||||
askChunked: func(_, _ string, _ any, within time.Duration) (bus.Answered, error) {
|
||||
got, way = within, "chunked"
|
||||
return bus.Answered{}, nil
|
||||
}}
|
||||
if _, err := (&moduleBus{all: c, module: "m"}).Ask(json.RawMessage(`{"key":"` + row.key + `","body":{}}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != row.want || way != row.way {
|
||||
t.Errorf("Ask %s waited %s (%s), want %s (%s)", row.key, got, way, row.want, row.way)
|
||||
}
|
||||
}
|
||||
if got := (&consumers{}).withinOf("seat:mesh-broker.rates"); got != 0 {
|
||||
t.Errorf("a runner that holds no definitions waits %s, want its own bound", got)
|
||||
}
|
||||
})
|
||||
// Design 47 G A-490m: a verb of a seat the list defines whose definition does not name it is read again once and never asked
|
||||
// without its call; a seat the list does not define, a module's tool and a bundle asking through moduleBus.Ask behave as before.
|
||||
t.Run("AVerbTheDefinitionDoesNotNameIsReadAgainAndNeverAskedWithoutItsCall", func(t *testing.T) {
|
||||
stale := map[string]seatdef.Definition{"mesh-wants": {Verbs: []seatdef.Verb{{Name: "revision"}}}}
|
||||
fresh := map[string]seatdef.Definition{"mesh-wants": {Verbs: []seatdef.Verb{{Name: "revision"}, {Name: "input", Chunks: true}}}}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
key string
|
||||
again func(held *map[string]seatdef.Definition) error
|
||||
wantWay string // "plain", "chunked" or "" for a refusal
|
||||
wantErr string
|
||||
wantReads int
|
||||
}{
|
||||
{"a defined seat's verb the read names", "seat:mesh-wants.input", func(h *map[string]seatdef.Definition) error { *h = fresh; return nil }, "chunked", "", 1},
|
||||
{"a read that fails", "seat:mesh-wants.input", func(h *map[string]seatdef.Definition) error { return errors.New("the seat does not answer") }, "", "does not answer", 1},
|
||||
{"a read that still does not name it", "seat:mesh-wants.input@node-a", func(h *map[string]seatdef.Definition) error { return nil }, "", "names no verb input", 1},
|
||||
{"a seat the list does not define", "seat:issue-tracker.list", func(h *map[string]seatdef.Definition) error { return nil }, "plain", "", 0},
|
||||
{"a module's tool", "mesh-delivery-indexer.indexer", func(h *map[string]seatdef.Definition) error { return nil }, "plain", "", 0},
|
||||
{"a verb the list names without chunks", "seat:mesh-wants.revision", func(h *map[string]seatdef.Definition) error { return nil }, "plain", "", 0},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
held, reads, way := stale, 0, ""
|
||||
c := &consumers{definitions: func() map[string]seatdef.Definition { return held }, reread: func() error { reads++; return row.again(&held) },
|
||||
askPlain: func(_, _ string, _ any, _ time.Duration) (bus.Answered, error) {
|
||||
way = "plain"
|
||||
return bus.Answered{}, nil
|
||||
},
|
||||
askChunked: func(_, _ string, _ any, _ time.Duration) (bus.Answered, error) {
|
||||
way = "chunked"
|
||||
return bus.Answered{}, nil
|
||||
}}
|
||||
_, err := (&moduleBus{all: c, module: "m"}).Ask(json.RawMessage(`{"key":"` + row.key + `","body":{}}`))
|
||||
if way != row.wantWay || reads != row.wantReads || (row.wantErr == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), row.wantErr)) {
|
||||
t.Errorf("Ask %s: asked %q, %v after %d reads, want %q, %q after %d", row.key, way, err, reads, row.wantWay, row.wantErr, row.wantReads)
|
||||
}
|
||||
})
|
||||
}
|
||||
// the asks that wait on a read share it
|
||||
release, reads := make(chan struct{}), 0
|
||||
held := stale
|
||||
c := &consumers{definitions: func() map[string]seatdef.Definition { return held }, reread: func() error { reads++; <-release; held = fresh; return nil }}
|
||||
done := make(chan bool, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
go func() { got, err := c.chunkedAsked("seat:mesh-wants.input"); done <- got && err == nil }()
|
||||
}
|
||||
for deadline := time.Now().Add(5 * time.Second); c.rereadWaiting() < 2; runtime.Gosched() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("two asks never joined the read running")
|
||||
}
|
||||
}
|
||||
close(release)
|
||||
for i := 0; i < 3; i++ {
|
||||
if !<-done {
|
||||
t.Error("an ask waiting on the shared read was not answered chunked")
|
||||
}
|
||||
}
|
||||
if reads != 1 {
|
||||
t.Errorf("%d reads of the seat list for three waiting asks, want 1", reads)
|
||||
}
|
||||
if got, err := (&consumers{}).chunkedAsked("seat:mesh-wants.input"); got || err != nil {
|
||||
t.Errorf("a runner that holds no definitions asks no chunks: %v, %v", got, err)
|
||||
}
|
||||
})
|
||||
// **The tool runner answers a held seat's marked verb by caller and serves the seat's `approvals`** (design 47 G §677; lead
|
||||
// answers A-265, A-271), with zsh's node-login-shell `execute` as the case: the marked verb reaches the gate with the caller
|
||||
// its subject names, and `approvals` is answered from the runner's own record of the calls its gate holds, with the
|
||||
|
||||
@@ -193,3 +193,44 @@ func TestAHeartbeatHandlerSetAgainWhileWordsAreHandedOnIsNotARace(t *testing.T)
|
||||
t.Error("a bundle gave a handler and no heartbeat is asked")
|
||||
}
|
||||
}
|
||||
|
||||
// In a module's own runtime the module's consumer is bound once: one event on it reaches the bundle and then the runtime's
|
||||
// hearer (the branch follower), and an event the bundle did not take is offered again before the hearer hears it (design 47
|
||||
// H §752, H-A-132).
|
||||
func TestOneEventOnAModulesConsumerReachesTheBundleAndTheHearer(t *testing.T) {
|
||||
var order []string
|
||||
all := &consumers{logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
WithHearer("mod", func(env bus.Envelope) error { order = append(order, "hearer "+env.Key); return nil })(all)
|
||||
m := &moduleEvents{}
|
||||
all.of["mod"] = m
|
||||
bundle := &deliverSlot{}
|
||||
refuse := false
|
||||
bundle.set(func(raw json.RawMessage) error {
|
||||
if refuse {
|
||||
return errorf("not now")
|
||||
}
|
||||
var env bus.Envelope
|
||||
if err := json.Unmarshal(raw, &env); err != nil {
|
||||
return err
|
||||
}
|
||||
order = append(order, "bundle "+env.Key)
|
||||
return nil
|
||||
})
|
||||
m.delivers = []*deliverSlot{bundle}
|
||||
if err := all.handOut("mod", m, bus.Envelope{Key: "mesh-catalog.catalog-revision.main", Body: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(order, ", "); got != "bundle mesh-catalog.catalog-revision.main, hearer mesh-catalog.catalog-revision.main" {
|
||||
t.Fatalf("the event reached %q, want the bundle and then the hearer", got)
|
||||
}
|
||||
order, refuse = nil, true
|
||||
if err := all.handOut("mod", m, bus.Envelope{Key: "a.b", Body: json.RawMessage(`{}`)}); err == nil || len(order) != 0 {
|
||||
t.Fatalf("an event the bundle did not take: err %v, reached %q; want it offered again, heard by none", err, order)
|
||||
}
|
||||
// Another module's events are handed to its bundles alone.
|
||||
other := &moduleEvents{delivers: []*deliverSlot{{f: func(json.RawMessage) error { order = append(order, "other"); return nil }}}}
|
||||
all.of["other"] = other
|
||||
if err := all.handOut("other", other, bus.Envelope{Key: "a.b", Body: json.RawMessage(`{}`)}); err != nil || strings.Join(order, ",") != "other" {
|
||||
t.Fatalf("another module's event: err %v, reached %q", err, order)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package runtime
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/approval"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
@@ -48,40 +49,12 @@ func WithGate(g Gate) Option { return func(c *consumers) { c.gate = g } }
|
||||
|
||||
var _ Gate = (*approval.Gate)(nil)
|
||||
|
||||
// WithSeatDefinitions refuses, where a seat's verb is answered, a caller outside the verb's effective callers, before the
|
||||
// bundle runs (design 47 G §610; lead answers A-165, A-179): defs are the seat definitions the runner's seat list holds.
|
||||
// Without it a verb is answered for every caller, as before.
|
||||
// WithSeatDefinitions gives the runner the seat definitions its seat list holds: a verb's `chunks` and `timeout` are read from them
|
||||
// (design 47 H §760). No definition names who may call: every caller is answered (design 47 H §752).
|
||||
func WithSeatDefinitions(defs func() map[string]seatdef.Definition) Option {
|
||||
return func(c *consumers) { c.definitions = defs }
|
||||
}
|
||||
|
||||
// admitted says why a seat's verb is refused to a caller, nil when it is answered: an unproven caller is refused, and so is
|
||||
// every caller while the seat's definition is not read, names no such verb, or the verb's effective callers are empty; a
|
||||
// proven caller is among them, since the grants part composes a verb's call subject only for its effective callers (A-179).
|
||||
func (c *consumers) admitted(seat, verb, caller string) error {
|
||||
if c.definitions == nil {
|
||||
return nil
|
||||
}
|
||||
if caller == "" {
|
||||
return fmt.Errorf("Not run: %s.%s takes no unproven caller: call it on its subject that names its caller, "+
|
||||
"mesh.seat.%s.call.%s.<caller>", seat, verb, seat, verb)
|
||||
}
|
||||
def, ok := c.definitions()[seat]
|
||||
if !ok {
|
||||
return fmt.Errorf("Not run: the definition of the seat %s is not yet read by this tool runner's seat list, so no caller "+
|
||||
"of %s.%s is admitted", seat, seat, verb)
|
||||
}
|
||||
for _, v := range def.Verbs {
|
||||
if v.Name == verb {
|
||||
if len(v.EffectiveCallers()) == 0 {
|
||||
return fmt.Errorf("Not run: %s.%s is open to no caller", seat, verb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("Not run: the seat %s's definition names no verb %s, so no caller is admitted", seat, verb)
|
||||
}
|
||||
|
||||
// unservedMarks is, of the module's marked tools, those its bundle's registrations under its own name do not register.
|
||||
func (r *runner) unservedMarks(module string, regs []launch.Registration) []string {
|
||||
if r.events.gate == nil {
|
||||
@@ -98,6 +71,91 @@ func (r *runner) unservedMarks(module string, regs []launch.Registration) []stri
|
||||
return r.events.gate.Unserved(module, r.fingerprintOf(module), registered)
|
||||
}
|
||||
|
||||
// WithSeatDefinitionsAgain gives the runner the way to read the seat list again (the seat list reader's Reload): a seat verb a bundle asks
|
||||
// that the definitions held do not name is looked up again after it, never taken for a verb that sends no chunks.
|
||||
func WithSeatDefinitionsAgain(again func() error) Option {
|
||||
return func(c *consumers) { c.reread = again }
|
||||
}
|
||||
|
||||
// chunkedAsked is chunked for a verb a bundle asks, where the answer decides whether the runner chooses the `call` the holder
|
||||
// requires (AskAsChunked) (design 47 G A-490m). A verb of a seat the definitions held DEFINE, whose definition does not name it,
|
||||
// is not taken for one that sends no chunks, since the holder then refuses an ask that carries no call: the seat list is read
|
||||
// again once (one read shared by the asks that wait on it) and a verb still unnamed is an error saying so. An ask of a seat the
|
||||
// definitions do not define at all, a module's tool and a runner that holds no definitions (nil) are asked as before.
|
||||
func (c *consumers) chunkedAsked(key string) (bool, error) {
|
||||
if c.definitions == nil || !strings.HasPrefix(key, "seat:") {
|
||||
return false, nil
|
||||
}
|
||||
name, _, _ := strings.Cut(key, "@")
|
||||
seat, verb, ok := strings.Cut(strings.TrimPrefix(name, "seat:"), ".")
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
find := func() (chunks, defined, named bool) {
|
||||
d, defined := c.definitions()[seat]
|
||||
for _, v := range d.Verbs {
|
||||
if v.Name == verb {
|
||||
return v.Chunks, true, true
|
||||
}
|
||||
}
|
||||
return false, defined, false
|
||||
}
|
||||
chunks, defined, named := find()
|
||||
if named || !defined {
|
||||
return chunks, nil
|
||||
}
|
||||
var why string
|
||||
if c.reread != nil {
|
||||
if err := c.rereadShared(); err != nil {
|
||||
why = " (reading the seat list again failed: " + err.Error() + ")"
|
||||
}
|
||||
chunks, defined, named = find()
|
||||
if named {
|
||||
return chunks, nil
|
||||
}
|
||||
if !defined {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return false, fmt.Errorf("the definition of the seat %s names no verb %s, so the runner cannot tell whether it sends its answer in chunks and does not ask it%s", seat, verb, why)
|
||||
}
|
||||
|
||||
// rereadShared reads the seat list again; asks that arrive while a read runs wait for it and share its result.
|
||||
func (c *consumers) rereadShared() error {
|
||||
c.rereadMu.Lock()
|
||||
if f := c.rereading; f != nil {
|
||||
f.waiters++
|
||||
c.rereadMu.Unlock()
|
||||
<-f.done
|
||||
return f.err
|
||||
}
|
||||
f := &rereadFlight{done: make(chan struct{})}
|
||||
c.rereading = f
|
||||
c.rereadMu.Unlock()
|
||||
f.err = c.reread()
|
||||
c.rereadMu.Lock()
|
||||
c.rereading = nil
|
||||
c.rereadMu.Unlock()
|
||||
close(f.done)
|
||||
return f.err
|
||||
}
|
||||
|
||||
type rereadFlight struct {
|
||||
done chan struct{}
|
||||
err error
|
||||
waiters int // the asks that joined the read (under rereadMu)
|
||||
}
|
||||
|
||||
// rereadWaiting is how many asks have joined the read now running; a test waits on it instead of sleeping.
|
||||
func (c *consumers) rereadWaiting() int {
|
||||
c.rereadMu.Lock()
|
||||
defer c.rereadMu.Unlock()
|
||||
if c.rereading == nil {
|
||||
return 0
|
||||
}
|
||||
return c.rereading.waiters
|
||||
}
|
||||
|
||||
// chunked says whether key names a seat's verb whose definition says `chunks` (design 47 G A-249): its answer is received in
|
||||
// chunks by the runner, whose call hands the bundle the whole.
|
||||
func (c *consumers) chunked(key string) bool {
|
||||
@@ -116,3 +174,23 @@ func (c *consumers) chunked(key string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// withinOf is how long a call of key waits for its answer: the `timeout` the verb's definition names, in whole seconds (design 47
|
||||
// H §760, H-A-43); zero, which the bus reads as the tool runner's own bound, for a key that is no seat's verb, a seat the
|
||||
// definitions do not hold, a verb they do not name and a verb that names none.
|
||||
func (c *consumers) withinOf(key string) time.Duration {
|
||||
if c.definitions == nil || !strings.HasPrefix(key, "seat:") {
|
||||
return 0
|
||||
}
|
||||
name, _, _ := strings.Cut(key, "@")
|
||||
seat, verb, ok := strings.Cut(strings.TrimPrefix(name, "seat:"), ".")
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
for _, v := range c.definitions()[seat].Verbs {
|
||||
if v.Name == verb && v.Timeout != nil {
|
||||
return time.Duration(*v.Timeout) * time.Second
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// The call a module's tool reaches the gate by carries the module's composed environment. The gate no longer tells a first-view tool
|
||||
// by an environment word: it knows one by its name forms <module>_first_views and <module>_first_view_again (H-A-69), and the runner
|
||||
// sets no view-callers word.
|
||||
func TestAModuleToolsCallCarriesItsBundlesComposedEnvironment(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
envs map[string]map[string]string
|
||||
want bool
|
||||
}{
|
||||
"a module with an environment": {map[string]map[string]string{"ledger": {"LEDGER_WORD": "x"}}, true},
|
||||
"a module with none": {nil, false},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", true, map[string][]string{"node-desk": {"who"}}))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
gate := &gateHolding{}
|
||||
stop, err := Run(nodeTools, []Served{{"ledger", []string{mt.Fixture("caller-echo.py")}}}, tc.envs, func(string, ...any) {}, deskDefined, WithGate(gate))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
asker := connect(t, "node-tools", "anchor")
|
||||
mt.Until(t, func() error { _, err := asker.Ask("ledger.who@anchor", map[string]any{}, ""); return err })
|
||||
var found bool
|
||||
for _, c := range gate.calls {
|
||||
if c.Seat == "" && c.Name == "who" {
|
||||
_, found = c.Env["LEDGER_WORD"]
|
||||
}
|
||||
}
|
||||
if found != tc.want {
|
||||
t.Errorf("the gate saw the composed word: %v, want %v (calls %+v)", found, tc.want, gate.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,7 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) {
|
||||
defer stop()
|
||||
asker := connect(t, "console", "workstation")
|
||||
listed := func(module string) (string, error) {
|
||||
answered, err := asker.Ask(module+"."+ToolsVerb, map[string]any{}, "")
|
||||
answered, err := asker.AskWithin(module+"."+ToolsVerb, map[string]any{}, "", callWithin)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -67,6 +67,18 @@ func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) {
|
||||
})
|
||||
}
|
||||
t.Run("a ledger bound later brings them", func(t *testing.T) {
|
||||
// Its own tool answers the first call after each binding comes or goes: the repairs' refresh keeps the
|
||||
// subscriptions it still names (design 47 H, H-A-160).
|
||||
for round := 1; round <= 10; round++ {
|
||||
plain.Bindings = nil
|
||||
if round%2 == 1 {
|
||||
plain.Bindings = owner.Bindings
|
||||
}
|
||||
mesh.Issue(t, plain)
|
||||
if _, err := asker.AskWithin("plain.who@anchor", map[string]any{}, "", callWithin); err != nil {
|
||||
t.Fatalf("binding change %d: the first call of plain.who failed: %v — dropped while its tools were served again", round, err)
|
||||
}
|
||||
}
|
||||
plain.Bindings = owner.Bindings
|
||||
mesh.Issue(t, plain)
|
||||
mt.Until(t, func() error {
|
||||
|
||||
@@ -2,6 +2,7 @@ package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -10,6 +11,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
@@ -154,13 +157,17 @@ func newReloadBed(t *testing.T) *reloadBed {
|
||||
}
|
||||
}
|
||||
|
||||
func (b *reloadBed) run(t *testing.T) {
|
||||
// run starts the runtime, stopped at the test's end or once by what it returns.
|
||||
func (b *reloadBed) run(t *testing.T) func() {
|
||||
t.Helper()
|
||||
stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
var once sync.Once
|
||||
stopOnce := func() { once.Do(stop) }
|
||||
t.Cleanup(stopOnce)
|
||||
return stopOnce
|
||||
}
|
||||
|
||||
func (b *reloadBed) answers(t *testing.T, tool string) {
|
||||
@@ -263,23 +270,66 @@ func TestAReplacedBundleIsReloaded(t *testing.T) {
|
||||
|
||||
// A replaced bundle's old tool stays gone when the module's membership is issued again: a stopped tool
|
||||
// leaves no membership handler behind to bind it once more, and a call on it starts no unmanaged second
|
||||
// copy of the bundle (issue 1945, review round 2).
|
||||
// copy of the bundle (issue 1945, review round 2). And the tool still served answers the first call asked
|
||||
// after each membership, within the test's own bound: the runner keeps the subscriptions the membership
|
||||
// still names, so no call is dropped while it follows one (design 47 H, H-A-158).
|
||||
func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
stop := b.run(t)
|
||||
b.answers(t, "first")
|
||||
place(t, b.entry, "second")
|
||||
b.answers(t, "second")
|
||||
b.silent(t, "first")
|
||||
|
||||
b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
b.answers(t, "second")
|
||||
for round := 1; round <= 5; round++ {
|
||||
b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
if got, err := call(t, b.asker, "ledger.second@anchor", map[string]any{}); err != nil || !strings.Contains(got, `"tool":"second"`) {
|
||||
t.Fatalf("membership %d issued again: the first call of ledger.second answered %s, %v — dropped while the runner "+
|
||||
"followed the membership\n%s", round, got, err, b.logs.All())
|
||||
}
|
||||
}
|
||||
// Long enough for the reissued membership to have been heard, before the old tool is asked.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil {
|
||||
t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All())
|
||||
}
|
||||
|
||||
// A ledger bound while the bundle is loaded again: its repairs are served once, by the module loaded, and none is
|
||||
// left behind on the stand-in the reload served nothing on (design 47 H, H-A-160). The bundle placed holds its
|
||||
// start until the test lets it go, so the membership is heard inside the reload.
|
||||
held, release := filepath.Join(t.TempDir(), "held"), filepath.Join(t.TempDir(), "release")
|
||||
placeScript(t, b.entry, strings.Replace(bundleServing("third"), "import json, os, sys\n",
|
||||
"import json, os, sys, time\nopen("+fmt.Sprintf("%q", held)+", \"w\").close()\nwhile not os.path.exists("+
|
||||
fmt.Sprintf("%q", release)+"):\n time.sleep(0.01)\n", 1))
|
||||
mt.Until(t, func() error { _, err := os.Stat(held); return err })
|
||||
bound := mt.MembershipOf("ledger", "anchor", false, nil)
|
||||
bound.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger,
|
||||
History: "LEDGER_ledger_records", Subject: "mesh.ledger.ledger.records.history", State: "ledger_records",
|
||||
Snapshot: "mesh.ledger.ledger.records.snapshot"}}
|
||||
b.mesh.Issue(t, bound)
|
||||
mt.Until(t, func() error {
|
||||
if m := b.nodeTools.Membership("ledger"); m == nil || len(m.Bindings) == 0 {
|
||||
return errors.New("the ledger's binding is not heard yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err := os.WriteFile(release, nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.answers(t, "third")
|
||||
mt.Until(t, func() error {
|
||||
_, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{})
|
||||
if errors.Is(err, bus.ErrNoResponders) || (err != nil && err.Error() == "timeout") {
|
||||
return fmt.Errorf("the ledger's repairs are not served after its reload: %v", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
stop()
|
||||
if _, err := call(t, b.asker, "ledger."+RepairLag+"@anchor", map[string]any{}); !errors.Is(err, bus.ErrNoResponders) {
|
||||
t.Fatalf("the runtime stopped, and ledger.%s still answered (%v): served on the reload's stand-in and never stopped\n%s",
|
||||
RepairLag, err, b.logs.All())
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: a bundle removed while served is no longer served, and its module says why.
|
||||
|
||||
@@ -177,6 +177,11 @@ func (r *runner) refreshRepairs() {
|
||||
changed := false
|
||||
for _, m := range r.modules {
|
||||
l := r.of[m]
|
||||
// A module being loaded again is left to its reload, which serves its repairs as its binding is then: served
|
||||
// here on the stand-in, they would be served twice and never stopped (design 47 H, H-A-160).
|
||||
if l.reloading {
|
||||
continue
|
||||
}
|
||||
served := false
|
||||
for _, reg := range l.own {
|
||||
served = served || reg.repairs
|
||||
@@ -185,10 +190,6 @@ func (r *runner) refreshRepairs() {
|
||||
if served == (repairs != nil) {
|
||||
continue
|
||||
}
|
||||
for i := len(l.serving) - 1; i >= 0; i-- {
|
||||
l.serving[i]()
|
||||
}
|
||||
l.serving, l.names = nil, nil
|
||||
l.own, l.regs = withRepairs(l.own, repairs), withRepairs(l.regs, repairs)
|
||||
r.serve(l)
|
||||
changed = true
|
||||
@@ -207,14 +208,16 @@ type loadedModule struct {
|
||||
regs []registration // every registration its bundles made
|
||||
own []registration // those under the name of a module served here
|
||||
failed string
|
||||
names []string // the tools it serves, as module.tool
|
||||
serving []func() // its tools' subscriptions
|
||||
names []string // the tools it serves, as module.tool
|
||||
serving map[string]func() // its tools' subscriptions, by module.tool; serve alone changes it
|
||||
// reloading marks the stand-in a reload puts in place while it loads the module again: it serves nothing.
|
||||
reloading bool
|
||||
}
|
||||
|
||||
// unload stops serving the module's tools and stops its bundles.
|
||||
func (l *loadedModule) unload() {
|
||||
for i := len(l.serving) - 1; i >= 0; i-- {
|
||||
l.serving[i]()
|
||||
for _, stop := range l.serving {
|
||||
stop()
|
||||
}
|
||||
l.serving, l.names = nil, nil
|
||||
for i := len(l.children) - 1; i >= 0; i-- {
|
||||
@@ -241,6 +244,9 @@ type runner struct {
|
||||
// §424, §618), never the runner's own.
|
||||
fingerprintOf func(module string) string
|
||||
|
||||
// envOf is a module's composed environment, the words the gate tells a first-view tool by (lead answers A-377).
|
||||
envOf func(module string) map[string]string
|
||||
|
||||
seatsAgain func()
|
||||
stops []func() // the watch, the announcement and the seats, in the order they began
|
||||
}
|
||||
@@ -253,6 +259,19 @@ func WithLedgerWriters(writer func(module string) (*bus.Conn, error)) Option {
|
||||
return func(c *consumers) { c.ledgers.writer = writer }
|
||||
}
|
||||
|
||||
// WithHearer hands every event of module's consumer also to hear, in process, after the module's bundles took it: the
|
||||
// consumer is bound once, for the bundles, and no second pull loop reads the same durable, so in a module's own runtime the
|
||||
// branch follower and the bundle each hear every event (design 47 H §752, H-A-113, H-A-132). An error of hear offers the
|
||||
// event again, as a bundle's does.
|
||||
func WithHearer(module string, hear func(bus.Envelope) error) Option {
|
||||
return func(c *consumers) {
|
||||
if c.hearers == nil {
|
||||
c.hearers = map[string][]func(bus.Envelope) error{}
|
||||
}
|
||||
c.hearers[module] = append(c.hearers[module], hear)
|
||||
}
|
||||
}
|
||||
|
||||
// Run launches, binds and serves, and serves each module's bundle again when it appears, changes or
|
||||
// goes, with no restart. It answers a stop function.
|
||||
func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, logf func(string, ...any), opts ...Option) (func(), error) {
|
||||
@@ -275,6 +294,7 @@ func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, log
|
||||
base := os.Environ()
|
||||
r.envFor = func(module string) []string { return BundleEnv(base, envs[module], module, node) }
|
||||
r.fingerprintOf = func(module string) string { return envs[module][FingerprintWord] }
|
||||
r.envOf = func(module string) map[string]string { return envs[module] }
|
||||
|
||||
// The module's events, for every child of it that subscribes (ADR 0198): one consumer per module,
|
||||
// bound the first time any of its children subscribes, each event handed to every child that did.
|
||||
@@ -310,7 +330,7 @@ func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, log
|
||||
}
|
||||
logf("%s", line)
|
||||
conn.OnMembership(func(bus.Membership) { go func() { r.refreshRepairs(); conn.Flush() }() })
|
||||
stopSeats, again := serveSeats(conn, r.modules, r.registrations, r.events.gated, r.events.admitted, r.events.chunked, r.approvalsOf(),
|
||||
stopSeats, again := serveSeats(conn, r.modules, r.registrations, r.events.gated, r.events.chunked, r.events.withinOf, r.approvalsOf(),
|
||||
r.fingerprintOf, logf)
|
||||
r.seatsAgain = again
|
||||
r.stops = append(r.stops, stopSeats)
|
||||
@@ -395,6 +415,7 @@ func (r *runner) claimed() map[string]bool {
|
||||
func (r *runner) load(s Served) *loadedModule {
|
||||
l := &loadedModule{served: s, seen: map[string]string{}}
|
||||
claimed := r.claimed()
|
||||
var all []launch.Registration // every bundle's registrations, the module's marks checked against them together (A-398)
|
||||
for _, entry := range s.Entrypoints {
|
||||
path, _ := filepath.Abs(entry)
|
||||
module := s.Module
|
||||
@@ -425,13 +446,7 @@ func (r *runner) load(s Served) *loadedModule {
|
||||
fail(clashed)
|
||||
continue
|
||||
}
|
||||
// A bundle whose marks name a tool it does not register is not loaded: no tool of its module is served here, rather
|
||||
// than one served unmarked that the manifest meant to mark (design 46 7-approval-mark §4).
|
||||
if unserved := r.unservedMarks(module, child.Registrations); len(unserved) > 0 {
|
||||
child.Stop()
|
||||
fail(fmt.Sprintf("its marks name %s, which it does not register (design 46 7-approval-mark §4)", strings.Join(unserved, ", ")))
|
||||
continue
|
||||
}
|
||||
all = append(all, child.Registrations...)
|
||||
l.children = append(l.children, child.Stop)
|
||||
for _, reg := range child.Registrations {
|
||||
got := registration{module: reg.Module, owner: module, tools: reg.Tools}
|
||||
@@ -445,6 +460,18 @@ func (r *runner) load(s Served) *loadedModule {
|
||||
}
|
||||
}
|
||||
}
|
||||
// A module whose marks name a tool none of its bundles registers is not loaded: no tool of it is served here, rather
|
||||
// than one served unmarked that the manifest meant to mark (design 46 7-approval-mark §4). Checked once, against every
|
||||
// bundle of the module together (lead answer A-398: gitea's eight bundles each overwrote the gate's set).
|
||||
if unserved := r.unservedMarks(s.Module, all); len(unserved) > 0 {
|
||||
for _, stop := range l.children {
|
||||
stop()
|
||||
}
|
||||
l.children, l.regs, l.own = nil, nil, nil
|
||||
l.failed = fmt.Sprintf("its marks name %s, which none of its bundles registers (design 46 7-approval-mark §4)", strings.Join(unserved, ", "))
|
||||
r.logf("[mesh-tools] %s is not loaded: %s; its tools are not served here", s.Module, l.failed)
|
||||
return l
|
||||
}
|
||||
if repairs := r.repairsOf(s.Module); repairs != nil {
|
||||
l.regs, l.own = append(l.regs, *repairs), append(l.own, *repairs)
|
||||
}
|
||||
@@ -469,24 +496,47 @@ func refusal(own []registration) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// serve binds a loaded module's tools. r.mu is held.
|
||||
// serve binds a loaded module's tools as its registrations name them now, and is the one routine that changes its
|
||||
// subscriptions (design 47 H, H-A-158, H-A-160): a tool still named keeps its subscription, a newly named one is
|
||||
// subscribed, and only then is one no longer named stopped — no tool is served twice, and none is dropped while the
|
||||
// module's tools are served again. r.mu is held.
|
||||
func (r *runner) serve(l *loadedModule) {
|
||||
if l.serving == nil {
|
||||
l.serving = map[string]func(){}
|
||||
}
|
||||
named := map[string]bool{}
|
||||
l.names = nil
|
||||
for _, reg := range l.own {
|
||||
for _, t := range reg.tools {
|
||||
t := t
|
||||
// Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365).
|
||||
// Through the approval gate, before the bundle or the runner's own tool (design 46 7-approval-mark §20).
|
||||
module, own := reg.module, reg.repairs
|
||||
stop, err := r.conn.HandleCalled(reg.module+"."+t.Name, func(caller string, body json.RawMessage) (any, error) {
|
||||
return r.events.gated(approval.Call{Address: module + "." + t.Name, Module: module, Name: t.Name, Own: own,
|
||||
Fingerprint: r.fingerprintOf(module), Caller: caller, Args: argsOf(body), Description: t.Description}, t, caller)
|
||||
})
|
||||
if err != nil {
|
||||
r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err)
|
||||
address := reg.module + "." + t.Name
|
||||
if named[address] {
|
||||
continue
|
||||
}
|
||||
l.names = append(l.names, reg.module+"."+t.Name)
|
||||
l.serving = append(l.serving, stop)
|
||||
named[address] = true
|
||||
if _, kept := l.serving[address]; !kept {
|
||||
// Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365).
|
||||
// Through the approval gate, before the bundle or the runner's own tool (design 46 7-approval-mark §20).
|
||||
module, own := reg.module, reg.repairs
|
||||
stop, err := r.conn.HandleCalled(address, func(caller string, body json.RawMessage) (any, error) {
|
||||
return r.events.gated(approval.Call{Address: module + "." + t.Name, Module: module, Name: t.Name, Own: own,
|
||||
Fingerprint: r.fingerprintOf(module), Env: r.envOf(module), Caller: caller, Args: argsOf(body),
|
||||
Description: t.Description}, t, caller)
|
||||
})
|
||||
if err != nil {
|
||||
r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err)
|
||||
delete(named, address)
|
||||
continue
|
||||
}
|
||||
l.serving[address] = stop
|
||||
}
|
||||
l.names = append(l.names, address)
|
||||
}
|
||||
}
|
||||
for address, stop := range l.serving {
|
||||
if !named[address] {
|
||||
stop()
|
||||
delete(l.serving, address)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -588,7 +638,7 @@ func (r *runner) reconcile() {
|
||||
func (r *runner) reload(module, what string) {
|
||||
r.mu.Lock()
|
||||
old := r.of[module]
|
||||
r.of[module] = &loadedModule{served: old.served, seen: old.seen}
|
||||
r.of[module] = &loadedModule{served: old.served, seen: old.seen, reloading: true}
|
||||
r.mu.Unlock()
|
||||
// Its events are refused first, so one heard while the old bundle stops is refused at once.
|
||||
r.events.dropModule(module)
|
||||
@@ -603,6 +653,11 @@ func (r *runner) reload(module, what string) {
|
||||
r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
// Its repairs as its binding is now, not as it was when the load began: the refresh left the stand-in alone.
|
||||
if fresh.failed == "" {
|
||||
repairs := r.repairsOf(module)
|
||||
fresh.own, fresh.regs = withRepairs(fresh.own, repairs), withRepairs(fresh.regs, repairs)
|
||||
}
|
||||
r.of[module] = fresh
|
||||
r.serve(fresh)
|
||||
r.bindVerbs()
|
||||
@@ -721,7 +776,7 @@ func subjectsOf(conn *bus.Conn, module, tool string) []string {
|
||||
// whenever a membership changes, and when again is called because a bundle was loaded again. Whether
|
||||
// this machine holds the seat is the bus's to decide. It answers its stop and again.
|
||||
func serveSeats(conn *bus.Conn, modules []string, current func() []registration, gated gatedRun,
|
||||
admitted func(seat, verb, caller string) error, chunked func(key string) bool, approvals func(caller string) (any, error),
|
||||
chunked func(key string) bool, within func(key string) time.Duration, approvals func(caller string) (any, error),
|
||||
fingerprintOf func(string) string, logf func(string, ...any)) (func(), func()) {
|
||||
var mu sync.Mutex
|
||||
var stops []func()
|
||||
@@ -761,9 +816,6 @@ func serveSeats(conn *bus.Conn, modules []string, current func() []registration,
|
||||
if v.Verb == seatdef.ApprovalsVerb && approvals != nil {
|
||||
seat := v.Seat
|
||||
stop, err := conn.HandleSubjectCalled(v.Subject, func(caller string, _ json.RawMessage) (any, error) {
|
||||
if err := admitted(seat, seatdef.ApprovalsVerb, caller); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return approvals(caller)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -779,27 +831,37 @@ func serveSeats(conn *bus.Conn, modules []string, current func() []registration,
|
||||
logf("[mesh-tools] %s claims %s and implements no %s, which that seat promises; not served", module, v.Seat, v.Verb)
|
||||
continue
|
||||
}
|
||||
// Its callers checked, then through the approval gate, before the bundle (design 47 G §610, A-179; design 46
|
||||
// Through the approval gate, for any caller, before the bundle (design 47 H §752; design 46
|
||||
// 7-approval-mark §20).
|
||||
module, v := module, v
|
||||
stop, err := conn.HandleSubjectCalled(v.Subject, func(caller string, body json.RawMessage) (any, error) {
|
||||
if err := admitted(v.Seat, v.Verb, caller); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
handle := func(caller string, body json.RawMessage) (any, error) {
|
||||
answer, err := gated(approval.Call{Address: v.Seat + "." + v.Verb, Module: module, Seat: v.Seat, Name: v.Verb,
|
||||
Fingerprint: fingerprintOf(module), Caller: caller, Args: argsOf(body), Description: t.Description}, t, caller)
|
||||
Fingerprint: fingerprintOf(module), Caller: caller, Args: argsOf(body), Description: t.Description}, t.Within(within("seat:"+v.Seat+"."+v.Verb)), caller)
|
||||
if err != nil || !chunked("seat:"+v.Seat+"."+v.Verb) {
|
||||
return answer, err
|
||||
}
|
||||
// A verb marked chunks: the bundle answers the whole file, the runner sends it in chunks (design 47 G A-257).
|
||||
return conn.AnswerChunked(v.Seat, v.Verb, body, answer)
|
||||
})
|
||||
}
|
||||
stop, err := conn.HandleSubjectCalled(v.Subject, handle)
|
||||
if err != nil {
|
||||
logf("[mesh-tools] cannot serve %s's %s on %s: %v", v.Seat, v.Verb, v.Subject, err)
|
||||
continue
|
||||
}
|
||||
stops = append(stops, stop)
|
||||
logf("[mesh-tools] serving %s's %s on %s, admitted where %s holds the seat", v.Seat, v.Verb, v.Subject, module)
|
||||
logf("[mesh-tools] serving %s's %s on %s, where %s holds the seat", v.Seat, v.Verb, v.Subject, module)
|
||||
// A verb of a node-scoped seat is also answered on the subject that names no machine, by any one holder, in
|
||||
// one queue group per seat and verb (lead answer A-393).
|
||||
if open, ok := anyHolderSubject(v, m.Node); ok && !have[open] {
|
||||
have[open] = true
|
||||
openStop, err := conn.HandleSubjectCalledQueue(open, "seat."+v.Seat+"."+v.Verb, handle)
|
||||
if err != nil {
|
||||
logf("[mesh-tools] cannot serve %s's %s on %s: %v", v.Seat, v.Verb, open, err)
|
||||
continue
|
||||
}
|
||||
stops = append(stops, openStop)
|
||||
logf("[mesh-tools] serving %s's %s on %s, in a queue group, for a call that names no machine", v.Seat, v.Verb, open)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -816,6 +878,15 @@ func serveSeats(conn *bus.Conn, modules []string, current func() []registration,
|
||||
}, serve
|
||||
}
|
||||
|
||||
// anyHolderSubject is a node-scoped seat verb's subject without its machine, the one asked with no machine named; false
|
||||
// where the verb's subject does not end in the machine (a seat held once).
|
||||
func anyHolderSubject(v bus.SeatVerb, node string) (string, bool) {
|
||||
if node == "" || !strings.HasSuffix(v.Subject, "."+v.Verb+"."+node) {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSuffix(v.Subject, "."+node), true
|
||||
}
|
||||
|
||||
// consumers holds, per module the runtime serves, the one durable consumer its events arrive on and
|
||||
// the children its events are handed to (novox/hq ADR 0198).
|
||||
type consumers struct {
|
||||
@@ -835,8 +906,16 @@ type consumers struct {
|
||||
sessions sessionTable
|
||||
// gate is the approval gate every call is served through (gate.go); nil serves every call as before.
|
||||
gate Gate
|
||||
// definitions are the seat definitions a seat verb's callers are read from (gate.go); nil admits every caller.
|
||||
// definitions are the seat definitions a seat verb's `chunks` and `timeout` are read from (gate.go); nil reads none.
|
||||
definitions func() map[string]seatdef.Definition
|
||||
// reread reads the seat list again (gate.go, chunkedAsked); nil reads nothing.
|
||||
reread func() error
|
||||
rereadMu sync.Mutex
|
||||
rereading *rereadFlight
|
||||
// hearers hear every event of a module's consumer in process, after the module's bundles took it (WithHearer).
|
||||
hearers map[string][]func(bus.Envelope) error
|
||||
// askPlain and askChunked are how a bundle's ask is sent (the bus connection's AskAs and AskAsChunked); a test gives its own.
|
||||
askPlain, askChunked func(module, key string, body any, within time.Duration) (bus.Answered, error)
|
||||
}
|
||||
|
||||
type moduleEvents struct {
|
||||
@@ -965,11 +1044,21 @@ func (b *moduleBus) Ask(params json.RawMessage) (json.RawMessage, error) {
|
||||
if len(asked.Body) == 0 {
|
||||
body = map[string]any{}
|
||||
}
|
||||
ask := b.all.conn.AskAs
|
||||
if b.all.chunked(asked.Key) {
|
||||
ask = b.all.conn.AskAsChunked // the runner receives the chunks of the answer; the bundle receives none (design 47 G A-249)
|
||||
ask := b.all.askPlain
|
||||
if ask == nil {
|
||||
ask = b.all.conn.AskAsWithin
|
||||
}
|
||||
answered, err := ask(b.module, asked.Key, body)
|
||||
chunks, err := b.all.chunkedAsked(asked.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if chunks {
|
||||
if ask = b.all.askChunked; ask == nil {
|
||||
ask = b.all.conn.AskAsChunkedWithin
|
||||
}
|
||||
// the runner receives the chunks of the answer; the bundle receives none (design 47 G A-249)
|
||||
}
|
||||
answered, err := ask(b.module, asked.Key, body, b.all.withinOf(asked.Key))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1029,7 +1118,7 @@ func (b *moduleBus) Subscribe(deliver func(json.RawMessage) error, beat func(jso
|
||||
return err
|
||||
}
|
||||
m.stop = stop
|
||||
c.logf("[mesh-tools] %s's events arrive on its consumer %s", module, bus.ConsumerOf(c.conn.Node(), module))
|
||||
c.logf("[mesh-tools] %s's events arrive on its consumer %s", module, bus.ToolsConsumerOf(c.conn.Node(), module))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1067,6 +1156,7 @@ func (c *consumers) handOut(module string, m *moduleEvents, env bus.Envelope) er
|
||||
return errModuleReloading
|
||||
}
|
||||
targets := append([]*deliverSlot(nil), m.delivers...)
|
||||
hearers := append([]func(bus.Envelope) error(nil), c.hearers[module]...)
|
||||
c.mu.Unlock()
|
||||
for _, d := range targets {
|
||||
if err := d.get()(raw); err != nil {
|
||||
@@ -1075,6 +1165,14 @@ func (c *consumers) handOut(module string, m *moduleEvents, env bus.Envelope) er
|
||||
return err
|
||||
}
|
||||
}
|
||||
// And the runtime's own hearers of the module's events, on this same consumer (WithHearer).
|
||||
for _, hear := range hearers {
|
||||
if err := hear(env); err != nil {
|
||||
c.logf("[mesh-tools] the tool runner did not take %s%s of %s's consumer: %v; offered again in %s", env.Key,
|
||||
eventRef(env), module, err, bus.NakDelay)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -26,9 +26,13 @@ func connect(t *testing.T, module, node string) *bus.Conn {
|
||||
return c
|
||||
}
|
||||
|
||||
// callWithin is the tests' own bound on a call (principle 11, design 47 H, H-A-158): a call that is never answered
|
||||
// fails the try it is part of within it, never after the tool runner's own 30 s.
|
||||
const callWithin = 2 * time.Second
|
||||
|
||||
func call(t *testing.T, asker *bus.Conn, key string, args any) (string, error) {
|
||||
t.Helper()
|
||||
got, err := asker.Ask(key, args, "")
|
||||
got, err := asker.AskWithin(key, args, "", callWithin)
|
||||
return string(got.Result), err
|
||||
}
|
||||
|
||||
@@ -226,7 +230,7 @@ func TestAnEntrypointThatIsNotExecutableIsRefused(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !logs.Has("marked's bundle", "failed to load: its marks name wipe, which it does not register (design 46 7-approval-mark §4)") {
|
||||
if !logs.Has("marked is not loaded: its marks name wipe, which none of its bundles registers (design 46 7-approval-mark §4)") {
|
||||
t.Errorf("a bundle not serving its marked tool was not refused at the load:\n%s", logs.All())
|
||||
}
|
||||
if _, err := call(t, asker, "served.who@anchor", map[string]any{}); err != nil {
|
||||
@@ -312,3 +316,46 @@ func TestAnUntakenEventSaysWhoFailed(t *testing.T) {
|
||||
t.Errorf("the event is not named by its id: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A node-scoped seat's verb asked with no machine is answered by a holder, by the tool form the SDK asks
|
||||
// (lead answer A-393).
|
||||
func TestANodeScopedSeatVerbIsAlsoAnsweredWithNoMachine(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("theta", "anchor", false, map[string][]string{"node-shelf": {"list"}}))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
asker := connect(t, "console", "workstation")
|
||||
regs := func() []registration {
|
||||
return []registration{{module: "node-shelf", owner: "theta", tools: []launch.Tool{{Name: "list",
|
||||
Run: func(string, json.RawMessage) (json.RawMessage, error) { return json.RawMessage(`{"shelf":["x"]}`), nil }}}}}
|
||||
}
|
||||
gated := func(_ approval.Call, tool launch.Tool, caller string) (any, error) {
|
||||
return tool.Run(caller, nil)
|
||||
}
|
||||
nodeTools.Follow("theta")
|
||||
stop, _ := serveSeats(nodeTools, []string{"theta"}, regs, gated, func(string) bool { return false }, func(string) time.Duration { return 0 }, nil, func(string) string { return "" }, (&mt.Logs{}).Logf)
|
||||
defer stop()
|
||||
for _, key := range []string{"seat:node-shelf.list@anchor", "seat:node-shelf.list"} {
|
||||
t.Run(key, func(t *testing.T) {
|
||||
mt.Until(t, func() error {
|
||||
got, err := call(t, asker, key, map[string]any{})
|
||||
if err == nil {
|
||||
same(t, got, `{"shelf":["x"]}`)
|
||||
}
|
||||
return err
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnyHolderSubject(t *testing.T) {
|
||||
for _, c := range []struct{ subject, node, want string }{
|
||||
{"mesh.seat.node-build-go.tool.steps.shanks", "shanks", "mesh.seat.node-build-go.tool.steps"},
|
||||
{"mesh.seat.mesh-delivery.tool.show", "shanks", ""},
|
||||
{"mesh.seat.node-build-go.tool.steps.shanks", "", ""},
|
||||
} {
|
||||
got, ok := anyHolderSubject(bus.SeatVerb{Seat: "s", Verb: "steps", Subject: c.subject}, c.node)
|
||||
if got != c.want || ok != (c.want != "") {
|
||||
t.Errorf("%s on %q: %q %v", c.subject, c.node, got, ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,12 +37,26 @@ func (f *fakeObjects) ObjectsPutStream(ctx context.Context, _ string, r buscalls
|
||||
return buscalls.ObjectInfo{Name: r.Name, Size: uint64(len(data)), Description: description}, nil
|
||||
}
|
||||
|
||||
func (f *fakeObjects) ObjectsOpen(context.Context, string, buscalls.ObjectsNameRequest) (buscalls.ObjectInfo, io.ReadCloser, bool, error) {
|
||||
return buscalls.ObjectInfo{Name: "o", Size: uint64(len(f.read))}, io.NopCloser(bytes.NewReader(f.read)), true, nil
|
||||
// ctxReader reads as the bus's object reader does: on the context it was opened on, failing once that is cancelled.
|
||||
type ctxReader struct {
|
||||
ctx context.Context
|
||||
r io.Reader
|
||||
}
|
||||
|
||||
func (f *fakeObjects) SentOpen(context.Context, buscalls.ObjectsSentGetRequest) (buscalls.ObjectInfo, io.ReadCloser, error) {
|
||||
return buscalls.ObjectInfo{Size: uint64(len(f.read))}, io.NopCloser(bytes.NewReader(f.read)), nil
|
||||
func (c ctxReader) Read(p []byte) (int, error) {
|
||||
if err := c.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.r.Read(p)
|
||||
}
|
||||
func (c ctxReader) Close() error { return nil }
|
||||
|
||||
func (f *fakeObjects) ObjectsOpen(ctx context.Context, _ string, _ buscalls.ObjectsNameRequest) (buscalls.ObjectInfo, io.ReadCloser, bool, error) {
|
||||
return buscalls.ObjectInfo{Name: "o", Size: uint64(len(f.read))}, ctxReader{ctx, bytes.NewReader(f.read)}, true, nil
|
||||
}
|
||||
|
||||
func (f *fakeObjects) SentOpen(ctx context.Context, _ buscalls.ObjectsSentGetRequest) (buscalls.ObjectInfo, io.ReadCloser, error) {
|
||||
return buscalls.ObjectInfo{Size: uint64(len(f.read))}, ctxReader{ctx, bytes.NewReader(f.read)}, nil
|
||||
}
|
||||
|
||||
func aSession(t *testing.T, f *fakeObjects) *moduleBus {
|
||||
|
||||
@@ -78,7 +78,10 @@ func increment(b launch.Bus, key string, attempts int) (n uint64, lost int, err
|
||||
|
||||
// Two writers incrementing one counter a thousand times each, against a real bus, end at two thousand:
|
||||
// every number handed out once and none lost, because the runtime writes only at the revision read
|
||||
// (novox/hq research 042: an issue's number is handed out once).
|
||||
// (novox/hq research 042: an issue's number is handed out once). A write is lost only to the other writer's
|
||||
// write, so one increment loses at most as many times as the other writer increments: that, not a guess,
|
||||
// is the bound on its tries, and a machine whose scheduler lets one writer win every round still passes
|
||||
// (one processor starved a writer of a thousand tries; principle 11, design 47 H §774).
|
||||
func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) {
|
||||
b := counterBus(t)
|
||||
const each = 1000
|
||||
@@ -91,7 +94,7 @@ func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) {
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < each; i++ {
|
||||
n, l, err := increment(b, "issue", 1000)
|
||||
n, l, err := increment(b, "issue", each+1)
|
||||
lost[w] += l
|
||||
if err != nil {
|
||||
errs[w] = err
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
# A tools bundle for the tool runner's whole test (design 47 H §774): MCP over stdio, no SDK. A tool `who`, the seat node-desk's verbs
|
||||
# `who` (answers at once) and `slow` (answers after three seconds), and a tool `reach` that asks the seat verb node-desk.fresh as the
|
||||
# bundle does, through the runtime's `mesh/ask`, and says what came back.
|
||||
import json, sys, time
|
||||
|
||||
def say(m):
|
||||
m["jsonrpc"] = "2.0"; sys.stdout.write(json.dumps(m) + "\n"); sys.stdout.flush()
|
||||
|
||||
def text(rid, value):
|
||||
say({"id": rid, "result": {"content": [{"type": "text", "text": json.dumps(value)}]}})
|
||||
|
||||
TOOLS = [
|
||||
{"name": "who", "description": "the module's own tool", "inputSchema": {"type": "object", "properties": {}}},
|
||||
{"name": "reach", "description": "asks the seat verb node-desk.fresh", "inputSchema": {"type": "object", "properties": {}}},
|
||||
{"name": "node-desk.who", "description": "the seat's verb", "inputSchema": {"type": "object", "properties": {}}},
|
||||
{"name": "node-desk.slow", "description": "the seat's verb that answers late", "inputSchema": {"type": "object", "properties": {}}},
|
||||
]
|
||||
for line in sys.stdin:
|
||||
req = json.loads(line); rid = req.get("id"); m = req.get("method"); p = req.get("params") or {}
|
||||
if m == "initialize":
|
||||
say({"id": rid, "result": {"protocolVersion": "2025-03-26", "capabilities": {"tools": {}}, "serverInfo": {"name": "ledger", "version": "1"}}})
|
||||
elif m == "tools/list":
|
||||
say({"id": rid, "result": {"tools": TOOLS}})
|
||||
elif m == "tools/call":
|
||||
name = p.get("name")
|
||||
if name == "node-desk.slow":
|
||||
time.sleep(3)
|
||||
if name == "reach":
|
||||
say({"id": "q1", "method": "mesh/ask", "params": {"key": "seat:node-desk.fresh", "body": {}}})
|
||||
for answer in sys.stdin:
|
||||
a = json.loads(answer)
|
||||
if a.get("id") == "q1":
|
||||
text(rid, a.get("result") if "error" not in a else {"error": a["error"]})
|
||||
break
|
||||
else:
|
||||
text(rid, {"tool": name})
|
||||
elif rid is not None:
|
||||
say({"id": rid, "result": {}})
|
||||
Reference in New Issue
Block a user