diff --git a/node-tools/cmd/node-tools/main.go b/node-tools/cmd/node-tools/main.go index 1621e83..1241d36 100644 --- a/node-tools/cmd/node-tools/main.go +++ b/node-tools/cmd/node-tools/main.go @@ -53,6 +53,17 @@ func main() { fmt.Fprintf(os.Stderr, "node-tools: %q is not a mode; this runtime serves (novox/hq ADR 0193)\n", os.Args[1]) os.Exit(2) } + // Stopped by the service manager's signal; the runtime itself is serve, which a whole test starts by its entry (design 47 H §774). + ctx, stopSignals := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stopSignals() + if err := serve(ctx); err != nil { + log.Fatalf("node-tools: %v", err) + } +} + +// serve is the tool runner: it reads its credential and serves until ctx ends, then shuts down in order. It is the entry a whole test +// starts, beside the process's own (design 47 H §774, H-A-61). +func serve(ctx context.Context) error { until := time.Now().Add(startWait) cred := credential(until) conn := connectPatiently(cred, until) @@ -61,11 +72,11 @@ func main() { served, err := runtime.ServedModulesFrom(os.Getenv(runtime.ToolModules), cred.Module) if err != nil { - log.Fatalf("node-tools: %v", err) + return err } envs, err := runtime.TakeToolEnvs() if err != nil { - log.Fatalf("node-tools: %v", err) + return err } // Its own secret ledgers: each module's ledger connection, the only one its history is published on (novox/hq ADR // 0597 §9). Followed, so a credential minted or retired is taken with no restart. @@ -90,7 +101,8 @@ func main() { Build: os.Getenv(runtime.FingerprintWord), Hear: hearOne(conn)}) // A failed read or write of the state approvals is made again at the next membership change the runner hears (A-157). conn.OnMembership(func(bus.Membership) { go gate.StateAgain() }) - opts = append(opts, runtime.WithGate(gate), runtime.WithSeatDefinitions(definitionsOf(seatList))) + opts = append(opts, runtime.WithGate(gate), runtime.WithSeatDefinitions(definitionsOf(seatList)), + runtime.WithSeatDefinitionsAgain(rereadOf(seatList))) // A run still unrecorded is asked again at each start and when the hand-act seat's holder is heard again (lead answer A-157). if stopBounds, err := gate.WatchBounds(); err != nil { log.Printf("[mesh-tools] the gate's bound keys are not watched (%v): an unanswered hand-act is said in the log only", err) @@ -101,7 +113,7 @@ func main() { go gate.Recover() stop, err := runtime.Run(conn, served, envs, log.Printf, opts...) if err != nil { - log.Fatalf("node-tools: %v", err) + return err } // And says it is there (novox/hq to-be 45 S11): the machine's runtime, on its machine's name. A // runtime on another credential — a module's own, in a test — is not the machine's. @@ -126,17 +138,16 @@ func main() { } who := node + "." + cred.Module surface := console.NewSurface(conn, source, who) + surface.Definitions = definitionsOf(seatList) surface.Approvals = gateTools{Gate: gate, state: approvalsState{conn: conn, module: cred.Module}} up, err = console.Serve(surface, listen) if err != nil { - log.Fatalf("node-tools: %v", err) + return err } log.Printf("mesh console listening on http://%s/mcp as %s", up.Address, who) } - signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT) - <-signals + <-ctx.Done() // First of all: the service manager signalled every bundle with this process, and none is started again // (issue 1126). launch.ShutDown() @@ -147,6 +158,7 @@ func main() { _ = up.Close() } conn.Close() + return nil } // startWait is how long a runtime waits, from its start, for its broker credential to be on the machine and for the bus diff --git a/node-tools/cmd/node-tools/seatlist.go b/node-tools/cmd/node-tools/seatlist.go index a545710..db08cd0 100644 --- a/node-tools/cmd/node-tools/seatlist.go +++ b/node-tools/cmd/node-tools/seatlist.go @@ -63,11 +63,25 @@ func startSeatListOn(b seatlist.Bus, bound <-chan time.Time, logf func(string, . return nil } catalogue.UseSeats(reader.Seats()) + // a first read that ended without an answer is no failure of the reader, which holds the bootstrap and waits for the seat's next + // event: so it is said here, where nothing else would say it, whether it ends before the bound or after it (a read the bus does + // not answer is asked again for about 30 s, so it ends after the bound); the runner then reads again where a bundle asks a verb of + // a seat the list defines that its definition does not name (design 47 G A-490m) + sayUnanswered := func() { + if st := reader.Status(); !st.Heard { + logf("node-tools: the first read of the seat list ended without an answer (%s); the runner serves with the bootstrapped seats and reads again when a bundle asks a verb of a seat the list defines that its definition does not name", st.LastError) + } + } select { case <-reader.Ready(): catalogue.UseSeats(reader.Seats()) + sayUnanswered() case <-bound: logf("node-tools: the seat list is not read yet; the runner serves with the bootstrapped seats and reads on") + go func() { + <-reader.Ready() + sayUnanswered() + }() } return reader } @@ -82,3 +96,14 @@ func definitionsOf(reader *seatlist.Reader) func() map[string]seatdef.Definition return reader.Definitions() } } + +// rereadOf is how the runner reads the seat list again where a verb is not named by the definitions it holds; it reads nothing +// while the reader did not start. +func rereadOf(reader *seatlist.Reader) func() error { + return func() error { + if reader == nil { + return fmt.Errorf("the seat list reader did not start") + } + return reader.Reload() + } +} diff --git a/node-tools/cmd/node-tools/seatlist_test.go b/node-tools/cmd/node-tools/seatlist_test.go index fda2420..2f5f941 100644 --- a/node-tools/cmd/node-tools/seatlist_test.go +++ b/node-tools/cmd/node-tools/seatlist_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "strings" + "sync" "sync/atomic" "testing" "time" @@ -32,19 +33,23 @@ func TestTheRunnerWaitsForTheFirstReadOfTheSeatListOnlyToItsBound(t *testing.T) wantSaid string wantReady bool }{ - {"the first read ends before the bound", true, "", true}, + {"the first read ends before the bound, unanswered", true, "the first read of the seat list ended without an answer", true}, {"the bound passes with the seat silent", false, "the seat list is not read yet", false}, + {"the bound passes, then the read ends unanswered", false, "the first read of the seat list ended without an answer", false}, } { t.Run(row.name, func(t *testing.T) { b := &busOfAStandIn{release: make(chan struct{})} bound := make(chan time.Time, 1) + var mu sync.Mutex var said []string + saidAgain := make(chan struct{}, 4) + say := func(f string, a ...any) { mu.Lock(); said = append(said, f); mu.Unlock(); saidAgain <- struct{}{} } if row.release { close(b.release) } else { bound <- time.Time{} } - r := startSeatListOn(b, bound, func(f string, a ...any) { said = append(said, f) }) + r := startSeatListOn(b, bound, say) if r == nil { t.Fatal("no reader") } @@ -57,12 +62,25 @@ func TestTheRunnerWaitsForTheFirstReadOfTheSeatListOnlyToItsBound(t *testing.T) if ended != row.wantReady { t.Errorf("first read ended = %v, want %v", ended, row.wantReady) } - if got := strings.Join(said, ";"); !strings.Contains(got, row.wantSaid) || (row.wantSaid == "" && got != "") { - t.Errorf("said %q, want %q", got, row.wantSaid) - } if !row.release { close(b.release) <-r.Ready() + // the read that ends after the bound is said by a goroutine of its own: wait for it, to a deadline that only a fault reaches + if strings.Contains(row.wantSaid, "first read") { + for i := 0; i < 2; i++ { + select { + case <-saidAgain: + case <-time.After(5 * time.Second): + t.Fatal("the unanswered end of the first read was never said") + } + } + } + } + mu.Lock() + got := strings.Join(said, ";") + mu.Unlock() + if !strings.Contains(got, row.wantSaid) { + t.Errorf("said %q, want %q", got, row.wantSaid) } }) } diff --git a/node-tools/cmd/node-tools/whole_integration_test.go b/node-tools/cmd/node-tools/whole_integration_test.go new file mode 100644 index 0000000..e68bb2e --- /dev/null +++ b/node-tools/cmd/node-tools/whole_integration_test.go @@ -0,0 +1,164 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/seatdef" + "git.novox.be/novox/mesh-sdk/go/seatlist" + + "github.com/novox/mesh-tools/node-tools/internal/bus" + mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" +) + +// catalogStandIn is the seat mesh-catalog's verb `seat-definitions`, as the seat list's reader asks it: the seat node-desk whose +// first listing names the verbs who and slow (slow with a timeout of one second) and, from the second listing on, also the verb +// fresh, which sends its answer in chunks. +type catalogStandIn struct { + mu sync.Mutex + listings int +} + +func (c *catalogStandIn) definition(withFresh bool) json.RawMessage { + one := 1 + verbs := []seatdef.Verb{{Name: "who"}, {Name: "slow", Timeout: &one}} + if withFresh { + verbs = append(verbs, seatdef.Verb{Name: "fresh", Chunks: true}) + } + raw, _ := json.Marshal(seatdef.Definition{Format: seatdef.Format, Seat: "node-desk", Scope: seatdef.ScopeNode, Grain: seatdef.SeatList, + Decision: "ADR 0001", Domain: seatdef.Domains[0], Verbs: verbs}) + return raw +} + +func (c *catalogStandIn) handle(body json.RawMessage) (any, error) { + var asked struct { + Seat string `json:"seat"` + } + _ = json.Unmarshal(body, &asked) + c.mu.Lock() + defer c.mu.Unlock() + if asked.Seat == "" { + c.listings++ + return seatlist.Listing{Revision: uint64(c.listings), Seats: []seatlist.Entry{{Seat: "node-desk", Source: seatlist.SourceTrunk, + Digest: map[bool]string{false: "d1", true: "d2"}[c.listings > 1]}}}, nil + } + withFresh := c.listings > 1 + return seatlist.Answer{Seat: asked.Seat, Found: true, Revision: uint64(c.listings), Entry: &seatlist.Entry{Seat: asked.Seat, + Source: seatlist.SourceTrunk, Digest: map[bool]string{false: "d1", true: "d2"}[withFresh], Definition: c.definition(withFresh)}}, nil +} + +// **The tool runner, started whole by its entry** (design 47 H §774, H-A-16, H-A-61): the process's own `serve`, on an embedded bus with +// JetStream, with a credential file and the environment the node-engine writes, serving one bundle's tool and seat verbs, over the +// seat list read from a stand-in of the seat mesh-catalog. +// - the bundle's tool and its seat verb are answered on the bus (§734's way of serving a verb from a bundle); +// - a verb whose definition names a timeout of one second is waited one second by the serving runner, not the runner's own +// thirty (design 47 H §760, H-A-43); +// - a bundle's ask of a seat verb its first seat list does not name is asked after one reread of the list, with its call token +// for a verb that sends chunks (design 47 G A-490m). +func TestTheToolRunnerStartedByItsEntryServesABundleOverTheSeatListItReads(t *testing.T) { + mesh := mt.New(t) + mesh.Issue(t, mt.MembershipOf("ledger", "anchor", true, map[string][]string{"node-desk": {"who", "slow"}})) + dir := t.TempDir() + credential, _ := json.Marshal(bus.Credential{URL: mt.URL(t), Module: runtimeModule, Node: "anchor"}) + file := filepath.Join(dir, "broker") + if err := os.WriteFile(file, credential, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("MESH_BROKER_FILE", file) + t.Setenv("MESH_TOOL_MODULES", "ledger="+mt.Fixture("whole-bundle.py")) + t.Setenv("MESH_CONSOLE_LISTEN", "127.0.0.1:0") + + open := func(module, node string) *bus.Conn { + c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: module, Node: node, User: node + "." + module, Password: "unused"}) + if err != nil { + t.Fatal(err) + } + c.Logf = func(string, ...any) {} + t.Cleanup(c.Close) + return c + } + catalog := &catalogStandIn{} + holder := open("mesh-catalog", "anchor") + if _, err := holder.HandleSubject("mesh.seat.mesh-catalog.tool.seat-definitions", func(body json.RawMessage) (any, error) { return catalog.handle(body) }); err != nil { + t.Fatal(err) + } + var mu sync.Mutex + var fresh []map[string]any + if _, err := holder.HandleSubject("mesh.seat.node-desk.tool.fresh", func(body json.RawMessage) (any, error) { + var got map[string]any + _ = json.Unmarshal(body, &got) + mu.Lock() + fresh = append(fresh, got) + mu.Unlock() + return map[string]any{"ok": true}, nil + }); err != nil { + t.Fatal(err) + } + holder.Flush() + + ctx, stop := context.WithCancel(context.Background()) + ended := make(chan error, 1) + go func() { ended <- serve(ctx) }() + t.Cleanup(func() { + stop() + select { + case err := <-ended: + if err != nil { + t.Errorf("serve ended with %v", err) + } + case <-time.After(30 * time.Second): + t.Error("the tool runner did not stop within 30s") + } + }) + // a module's account, which the approval gate runs unmarked while the catalog half is unread (the stand-in serves no catalog half) + asker := open("shop", "workstation") + ask := func(key string) (string, error) { + answered, err := asker.Ask(key, map[string]any{}, "") + return string(answered.Result), err + } + + // the bundle's tool and its seat verb are served + for _, key := range []string{"ledger.who", "seat:node-desk.who@anchor"} { + mt.Until(t, func() error { + got, err := ask(key) + if err == nil && !strings.Contains(got, `"tool":`) { + t.Errorf("%s answered %s, want the bundle's", key, got) + } + return err + }) + } + + // a verb whose definition names a timeout of one second is waited one second + began := time.Now() + if _, err := ask("seat:node-desk.slow@anchor"); err == nil { + t.Error("the verb that answers after three seconds was answered within its timeout of one second") + } + if took := time.Since(began); took < time.Second || took > 2500*time.Millisecond { + t.Errorf("the verb of a timeout of one second was waited %s, want about one second, not the runner's own %s", took, bus.RequestTimeout) + } + + // the bundle's ask of a verb its first list does not name is asked after one reread, with its call token + mt.Until(t, func() error { + got, err := ask("ledger.reach") + if err == nil && strings.Contains(got, "error") { + t.Errorf("the bundle's ask of node-desk.fresh failed: %s", got) + } + return err + }) + catalog.mu.Lock() + listings := catalog.listings + catalog.mu.Unlock() + mu.Lock() + defer mu.Unlock() + if listings != 2 || len(fresh) != 1 || fresh[0]["call"] == nil || fresh[0]["call"] == "" { + t.Errorf("the list was read %d times (want 2) and the holder of node-desk.fresh was asked %v (want once, with a call token)", listings, fresh) + } +} diff --git a/node-tools/internal/approval/approval_test.go b/node-tools/internal/approval/approval_test.go index 97c209f..be5a973 100644 --- a/node-tools/internal/approval/approval_test.go +++ b/node-tools/internal/approval/approval_test.go @@ -218,16 +218,6 @@ func TestTheRefusalsAndAnswers(t *testing.T) { t.Fatalf("asked %d, %v, runs %d", len(w.asked), err, w.runs) } }}, - {"§17: an ask the router holds back as the fourth open one still waits", func(t *testing.T, w *world) { - _, _ = w.do(call("wipe", `{}`)) - a := w.asked[0] - rec := w.records["shell."+a.ID] - rec.State, rec.HeldSince = "held-back", operatorchannelclient.FormatTime(w.now) - w.records["shell."+a.ID] = rec - if _, err := w.do(call("wipe", `{}`)); err == nil || !strings.Contains(err.Error(), "still waiting") || len(w.asked) != 1 { - t.Fatalf("%v, asked %d", err, len(w.asked)) - } - }}, {"§14, §15: approved, the same call runs once; made again it is already run and asks anew", func(t *testing.T, w *world) { _, _ = w.do(call("wipe", `{"dir":"scratch"}`)) w.decide(gate.OptionApprove, w.now.Add(time.Minute)) @@ -321,9 +311,9 @@ func TestTheRefusalsAndAnswers(t *testing.T) { _, _ = w.do(call("wipe", `{}`)) a := w.asked[0] rec := w.records["shell."+a.ID] - rec.State, rec.Answer = "refused", "the asker has too many open asks" + rec.State, rec.Answer = "refused", "the asker is not known to the router" w.records["shell."+a.ID] = rec - if _, err := w.do(call("wipe", `{}`)); err == nil || !strings.Contains(err.Error(), "the asker has too many open asks") { + if _, err := w.do(call("wipe", `{}`)); err == nil || !strings.Contains(err.Error(), "the asker is not known to the router") { t.Fatalf("%v", err) } }}, diff --git a/node-tools/internal/bus/bus.go b/node-tools/internal/bus/bus.go index 8e843af..9997853 100644 --- a/node-tools/internal/bus/bus.go +++ b/node-tools/internal/bus/bus.go @@ -32,9 +32,18 @@ import ( "github.com/novox/mesh-tools/node-tools/internal/wire" ) -// RequestTimeout is how long a call waits for its answer — what modules already expect. +// RequestTimeout is how long a call waits for its answer when its verb's definition names no `timeout`: the tool runner's own +// bound, what modules already expect (design 47 H §760, H-A-43). const RequestTimeout = 30 * time.Second +// waitOf is the bound a call waits: within where the verb's definition names one, else the tool runner's own. +func waitOf(within time.Duration) time.Duration { + if within > 0 { + return within + } + return RequestTimeout +} + // Credential is a broker credential as the mesh delivers it (novox/hq ADR 0120). type Credential struct { URL string `json:"url"` @@ -793,6 +802,12 @@ func (c *Conn) reachedAt(key string) (string, error) { // Ask calls a tool by key — or on a subject the mesh listed for it — and learns which machine // answered. Core request/reply: a tool call is never persisted (design 25 §3). func (c *Conn) Ask(key string, body any, on string) (Answered, error) { + return c.AskWithin(key, body, on, 0) +} + +// AskWithin is Ask that waits within for its answer, the `timeout` its verb's definition names (design 47 H §760, H-A-43); +// zero waits the tool runner's own bound, RequestTimeout. +func (c *Conn) AskWithin(key string, body any, on string, within time.Duration) (Answered, error) { subject := on if subject == "" { s, err := c.reachedAt(key) @@ -809,12 +824,12 @@ func (c *Conn) Ask(key string, body any, on string) (Answered, error) { // account publish with its own name alone; asked again on the subject that names no caller only when the // first reached nobody — nothing serves it yet, or the bus does not grant it yet — so no call is done twice. if called := CalledSubject(subject, c.cred.User); called != "" { - answered, fallBack, err := c.askCalled(called, data) + answered, fallBack, err := c.askCalled(called, data, within) if !fallBack { return answered, err } } - return c.askOnce(subject, data) + return c.askOnce(subject, data, within) } // ErrNoResponders is the bus's own refusal: nothing subscribes to the subject asked. It is told apart @@ -822,8 +837,8 @@ func (c *Conn) Ask(key string, body any, on string) (Answered, error) { var ErrNoResponders = errors.New("503 no responders") // askOnce asks once. -func (c *Conn) askOnce(subject string, data []byte) (Answered, error) { - msg, err := c.nc.Request(subject, data, RequestTimeout) +func (c *Conn) askOnce(subject string, data []byte, within time.Duration) (Answered, error) { + msg, err := c.nc.Request(subject, data, waitOf(within)) if err != nil { if errors.Is(err, nats.ErrNoResponders) { return Answered{}, ErrNoResponders @@ -1036,6 +1051,11 @@ func SeatToolSubject(seat, verb, scope, node string) string { // `.` another's, `seat:.[@]` a role's — resolved through what the // mesh issued that module to reach, as its own runtime resolved it, else the derived shape. func (c *Conn) AskAs(module, key string, body any) (Answered, error) { + return c.AskAsWithin(module, key, body, 0) +} + +// AskAsWithin is AskAs that waits within for its answer (design 47 H §760); zero waits RequestTimeout. +func (c *Conn) AskAsWithin(module, key string, body any, within time.Duration) (Answered, error) { name, wanted, _ := strings.Cut(key, "@") subject := "" if m := c.Membership(module); m != nil { @@ -1058,7 +1078,7 @@ func (c *Conn) AskAs(module, key string, body any) (Answered, error) { } subject = s } - return c.Ask(key, body, subject) + return c.AskWithin(key, body, subject, within) } // EventsStream is where every module's events land, and ConsumerOf the durable consumer the diff --git a/node-tools/internal/bus/bus_test.go b/node-tools/internal/bus/bus_test.go index a44a3e7..9e3f8bd 100644 --- a/node-tools/internal/bus/bus_test.go +++ b/node-tools/internal/bus/bus_test.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "testing" + "time" ) func TestSubjectsAreTheOnesTheTypeScriptRuntimeUses(t *testing.T) { @@ -60,3 +61,12 @@ func TestAnEventDeliveredAgainHasItsOwnKey(t *testing.T) { } } } + +// Design 47 H §760: a call waits the bound its verb's definition names, else the tool runner's own. +func TestWaitOfIsTheVerbsBoundElseTheRunnersOwn(t *testing.T) { + for within, want := range map[time.Duration]time.Duration{0: RequestTimeout, 40 * time.Second: 40 * time.Second, 600 * time.Second: 600 * time.Second} { + if got := waitOf(within); got != want { + t.Errorf("waitOf(%s) = %s, want %s", within, got, want) + } + } +} diff --git a/node-tools/internal/bus/caller.go b/node-tools/internal/bus/caller.go index b9fa39e..24e07fe 100644 --- a/node-tools/internal/bus/caller.go +++ b/node-tools/internal/bus/caller.go @@ -7,6 +7,7 @@ import ( "regexp" "strings" "sync" + "time" "github.com/nats-io/nats.go" ) @@ -150,9 +151,9 @@ func (c *Conn) publishRefused(err error) bool { // askCalled asks once on a caller-named subject. fallBack says the call reached nobody — nothing serves that // subject yet, or the bus refused this user its publish — so it may be asked on the subject that names no // caller without being done twice. -func (c *Conn) askCalled(subject string, data []byte) (answered Answered, fallBack bool, err error) { +func (c *Conn) askCalled(subject string, data []byte, within time.Duration) (answered Answered, fallBack bool, err error) { ctx, cancel := context.WithCancelCause(context.Background()) - ctx, stop := context.WithTimeout(ctx, RequestTimeout) + ctx, stop := context.WithTimeout(ctx, waitOf(within)) defer stop() c.calls.mu.Lock() if c.calls.pending == nil { diff --git a/node-tools/internal/bus/caller_bus_test.go b/node-tools/internal/bus/caller_bus_test.go index c46049d..6a6f051 100644 --- a/node-tools/internal/bus/caller_bus_test.go +++ b/node-tools/internal/bus/caller_bus_test.go @@ -216,3 +216,30 @@ func TestAFallBackForWantOfAResponderIsSaidOncePerSubject(t *testing.T) { t.Errorf("the fall-back was said %d times, want once: %v", n, said) } } + +// Design 47 H §760: a call waits the bound its verb's definition names, and ends there with the timeout; the caller of a verb whose +// definition names none waits the tool runner's own bound, RequestTimeout. +func TestACallWaitsTheBoundItsVerbNamesAndTimesOutThere(t *testing.T) { + url := grantedBus(t, map[string][]string{"anchor.node-tools": {"_INBOX.>"}, "shanks.node-tools": {"mesh.mod.*.tool.>"}}) + server := dial(t, url, "node-tools", "anchor", "anchor.node-tools") + release := make(chan struct{}) + stop, err := server.HandleSubjectCalled("mesh.mod.ledger.tool.slow", func(string, json.RawMessage) (any, error) { + <-release + return map[string]any{}, nil + }) + if err != nil { + t.Fatal(err) + } + defer stop() + defer close(release) + server.Flush() + asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools") + began := time.Now() + _, err = asker.AskWithin("ledger.slow", map[string]any{}, "", 200*time.Millisecond) + if err == nil || err.Error() != "timeout" { + t.Fatalf("a call whose verb names a bound the holder outlasts: %v, want the timeout", err) + } + if took := time.Since(began); took > bus.RequestTimeout/2 { + t.Errorf("the call waited %s, want its own bound of 200ms, not the tool runner's %s", took, bus.RequestTimeout) + } +} diff --git a/node-tools/internal/bus/chunked.go b/node-tools/internal/bus/chunked.go index 7b9257b..3d7f846 100644 --- a/node-tools/internal/bus/chunked.go +++ b/node-tools/internal/bus/chunked.go @@ -25,6 +25,11 @@ const FileField = "file" // counts with buscalls.SeatChunks, each acknowledged before the next, and answers the verb's own answer with the whole // file added as FileField. The asking bundle receives no chunk itself. func (c *Conn) AskAsChunked(module, key string, body any) (Answered, error) { + return c.AskAsChunkedWithin(module, key, body, 0) +} + +// AskAsChunkedWithin is AskAsChunked that waits within for the verb's answer (design 47 H §760); zero waits RequestTimeout. +func (c *Conn) AskAsChunkedWithin(module, key string, body any, within time.Duration) (Answered, error) { // The verb may be asked of the machine that ran the ask (`@`, A-254): the chunk subjects name the seat, the verb and // the call alone. name, _, _ := strings.Cut(key, "@") @@ -52,7 +57,7 @@ func (c *Conn) AskAsChunked(module, key string, body any) (Answered, error) { return Answered{}, fmt.Errorf("the chunks of %s cannot be heard: %w", key, err) } defer func() { _ = sub.Unsubscribe() }() - answered, err := c.AskAs(module, key, args) + answered, err := c.AskAsWithin(module, key, args, within) if err != nil { return answered, err } diff --git a/node-tools/internal/bus/state_watch_test.go b/node-tools/internal/bus/state_watch_test.go index 15280aa..2780b72 100644 --- a/node-tools/internal/bus/state_watch_test.go +++ b/node-tools/internal/bus/state_watch_test.go @@ -16,7 +16,7 @@ import ( ) // **A watch whose current values take longer than the bound to hand over still holds** (novox/hq issue -// 1726). The gitea module watches the checker's owed: one key per pull request head, each a round trip +// 1726). The gitea module watches another module's state (the assigner's pins stand for it): one key per pull request head, each a round trip // to the bundle. With a few hundred keys the hand-over outlasted StateTimeout at every attempt, the // watch was refused, and the newest keys, handed over last, never reached mesh/check: a group's passing // verdict was never written on its members' heads. The bound is on progress: a watch fails only when @@ -45,7 +45,7 @@ func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) { var mu sync.Mutex var current, live []string // Each value takes 40 ms to be taken: 33 values take about 1.3 s, three times the bound. - stop, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(ch bus.StateChange) error { + stop, err := conn.StateWatch("gitea", "mesh-delivery-assigner.pins", "", func(ch bus.StateChange) error { if ch.Current { time.Sleep(40 * time.Millisecond) } @@ -127,12 +127,12 @@ func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) { } var stop func() if tc.refused { - if _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err == nil { + if _, err := conn.StateWatch("gitea", "mesh-delivery-assigner.pins", "", deliver); err == nil { t.Fatal("the watch whose bundle took nothing was not refused") } } else { var err error - if stop, err = conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err != nil { + if stop, err = conn.StateWatch("gitea", "mesh-delivery-assigner.pins", "", deliver); err != nil { t.Fatal(err) } for i := 0; i < 4; i++ { @@ -173,7 +173,7 @@ func TestAWatchIsRefusedWhenNoCurrentValueIsTakenWithinTheBound(t *testing.T) { release := make(chan struct{}) t.Cleanup(func() { close(release) }) began := time.Now() - _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(bus.StateChange) error { + _, err := conn.StateWatch("gitea", "mesh-delivery-assigner.pins", "", func(bus.StateChange) error { <-release return nil }) @@ -229,16 +229,16 @@ func TestAMembershipTheGrantsPartWroteWrappedIsServed(t *testing.T) { } } -// owedWatcher is the gitea module's view of the checker's owed: the bucket, and a runtime connection -// that serves gitea with the owed issued for reading. +// owedWatcher is the gitea module's view of another module's state: the bucket, and a runtime connection +// that serves gitea with that state issued for reading. func owedWatcher(t *testing.T) (*bus.Conn, interface { Put(string, []byte) (uint64, error) }) { t.Helper() mesh := mt.New(t) - kv := mesh.Bucket(t, "mesh-delivery-checker_owed") + kv := mesh.Bucket(t, "mesh-delivery-assigner_pins") gitea := mt.MembershipOf("gitea", "anchor", false, nil) - gitea.State = []bus.StateIssued{{Name: "mesh-delivery-checker.owed", Bucket: "mesh-delivery-checker_owed"}} + gitea.State = []bus.StateIssued{{Name: "mesh-delivery-assigner.pins", Bucket: "mesh-delivery-assigner_pins"}} mesh.Issue(t, gitea) conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"}) if err != nil { diff --git a/node-tools/internal/console/address.go b/node-tools/internal/console/address.go index 706f38c..dcf7ee3 100644 --- a/node-tools/internal/console/address.go +++ b/node-tools/internal/console/address.go @@ -927,10 +927,23 @@ func schemaWithoutNode(raw json.RawMessage) map[string]any { } // answerText is a discovery verb's answer as MCP content: JSON, indented. +// withinOf is how long a call of the target waits: the `timeout` its seat verb's definition names (design 47 H §760), else the +// runner's own bound, RequestTimeout. +func (s *Surface) withinOf(t target) time.Duration { + if t.Seat && s.Definitions != nil { + for _, v := range s.Definitions()[t.Tool.Module].Verbs { + if v.Name == t.Tool.Name && v.Timeout != nil { + return time.Duration(*v.Timeout) * time.Second + } + } + } + return bus.RequestTimeout +} + // callFailed is why a call failed, as mesh_call says it: a marked tool or verb that reaches no responder names `withheld` // and the grants part's read (design 46 7-approval-mark §6), and a target known only from an earlier round says so. func (s *Surface) callFailed(t target, err error) string { - why := whyItFailed(t.Address, err) + why := whyItFailedWithin(t.Address, err, s.withinOf(t)) if ok, _ := s.markedTool(t); ok && errors.Is(err, bus.ErrNoResponders) { // A marked subject withheld from a machine whose tool runner does not honour the mark (§6). why += "; it needs the operator's approval, so it may be withheld where the machine's tool runner does not state " + @@ -1147,7 +1160,7 @@ func (s *Surface) discover(ctx context.Context, name string, args map[string]any } var got bus.Answered if t.Seat { - got, err = s.conn.Ask(t.Key, callArgs, "") + got, err = s.conn.AskWithin(t.Key, callArgs, "", s.withinOf(t)) } else { got, err = callTool(s.conn, t.Key, callArgs, nil, x.Listing) } diff --git a/node-tools/internal/console/client.go b/node-tools/internal/console/client.go index dc1ef15..69d25df 100644 --- a/node-tools/internal/console/client.go +++ b/node-tools/internal/console/client.go @@ -6,6 +6,7 @@ import ( "fmt" "regexp" "strings" + "time" "github.com/nats-io/nats.go" @@ -120,6 +121,11 @@ var ( // whyItFailed says why a call failed, so the remedy is in the words. func whyItFailed(key string, err error) string { + return whyItFailedWithin(key, err, bus.RequestTimeout) +} + +// whyItFailedWithin is whyItFailed for a call that waited within, the bound its verb's definition names or the runner's own. +func whyItFailedWithin(key string, err error, within time.Duration) string { if err == nil { err = errors.New("failed") } @@ -139,7 +145,7 @@ func whyItFailed(key string, err error) string { case timedOut.MatchString(msg): // **A timeout is not a failure** (novox/hq issue 265). Something took the call, so it may // still be running and may already have done what was asked: a push said this while it pushed. - said := key + " did not answer within " + bus.RequestTimeout.String() + ". That is not a failure: " + + said := key + " did not answer within " + within.String() + ". That is not a failure: " + "something is serving it, so it may still be running and may already have done what was " + "asked — check before repeating a call that changes the mesh." return said diff --git a/node-tools/internal/console/mcp.go b/node-tools/internal/console/mcp.go index 57e4e8b..99f2a27 100644 --- a/node-tools/internal/console/mcp.go +++ b/node-tools/internal/console/mcp.go @@ -8,6 +8,7 @@ import ( "encoding/json" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/seatdef" "os" "strings" "sync" @@ -65,6 +66,9 @@ type Surface struct { Record func(args map[string]any) error // Approvals is this machine's approval gate, read by mesh_approvals and mesh_describe; nil when the runner has none. Approvals Approvals + // Definitions is the seat definitions this runner holds, by seat, which name a verb's `timeout` (design 47 H §760); nil when it + // holds none, and a call then waits the runner's own bound. + Definitions func() map[string]seatdef.Definition } // MethodSeats is the console's JSON-RPC method beside MCP's that answers every seat's verbs and holders, in the shape diff --git a/node-tools/internal/console/refusal_test.go b/node-tools/internal/console/refusal_test.go index 5d2111f..6316836 100644 --- a/node-tools/internal/console/refusal_test.go +++ b/node-tools/internal/console/refusal_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/seatdef" "git.novox.be/novox/mesh-sdk/go/stateobjects" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -74,6 +75,27 @@ func TestARefusalIsSaidOnlyForTheBussOwn(t *testing.T) { // the grammar's example, the handshake's instructions and a timeout's remedy named the controller's verbs. func TestATimeoutIsNotAFailure(t *testing.T) { got := whyItFailed("seat:mesh-delivery.deliveries", errors.New("timeout")) + // Design 47 H §760: mesh_call waits the `timeout` the verb's definition names and a timeout says that bound; a verb naming none, + // a module's tool and a surface holding no definitions wait and say the runner's own 30s. + forty := 40 + s := &Surface{Definitions: func() map[string]seatdef.Definition { + return map[string]seatdef.Definition{"mesh-broker": {Verbs: []seatdef.Verb{{Name: "rates", Timeout: &forty}, {Name: "plain"}}}} + }} + for _, row := range []struct { + t target + want string + }{ + {target{Address: "mesh-broker.rates", Seat: true, Tool: Tool{Module: "mesh-broker", Name: "rates"}}, "within 40s"}, + {target{Address: "mesh-broker.plain", Seat: true, Tool: Tool{Module: "mesh-broker", Name: "plain"}}, "within 30s"}, + {target{Address: "mesh-broker.rates", Tool: Tool{Module: "mesh-broker", Name: "rates"}}, "within 30s"}, + } { + if said := s.callFailed(row.t, errors.New("timeout")); !strings.Contains(said, row.want) { + t.Errorf("%s: a timeout says %q, want it to say %q", row.t.Address, said, row.want) + } + } + if said := (&Surface{}).callFailed(target{Seat: true, Tool: Tool{Module: "mesh-broker", Name: "rates"}}, errors.New("timeout")); !strings.Contains(said, "within 30s") { + t.Errorf("a surface that holds no definitions says %q, want 30s", said) + } for _, want := range []string{"not a failure", "may already have done"} { if !strings.Contains(got, want) { t.Errorf("a seat's timeout does not say %q: %s", want, got) diff --git a/node-tools/internal/launch/launch.go b/node-tools/internal/launch/launch.go index 203b28e..652c7f2 100644 --- a/node-tools/internal/launch/launch.go +++ b/node-tools/internal/launch/launch.go @@ -46,6 +46,20 @@ type Tool struct { // Run calls the tool for caller: the bus user the call's subject named, or "" when it named none. A // caller is handed to the bundle in the call's `_meta` (MetaCaller), never among its arguments. Run func(caller string, args json.RawMessage) (json.RawMessage, error) + // RunWithin is Run that waits within for the bundle's answer, the `timeout` the verb's seat definition names (design 47 H §760, + // H-A-43); zero waits CallTimeout. A tool the runtime serves itself has none: Run is its call. + RunWithin func(caller string, args json.RawMessage, within time.Duration) (json.RawMessage, error) +} + +// Within is the tool as called for a verb whose definition names a timeout: its Run waits within for the bundle's answer. A zero +// within, and a tool that has no RunWithin, are the tool as it is. +func (t Tool) Within(within time.Duration) Tool { + if within <= 0 || t.RunWithin == nil { + return t + } + run := t.RunWithin + t.Run = func(caller string, args json.RawMessage) (json.RawMessage, error) { return run(caller, args, within) } + return t } // MetaCaller is where in a `tools/call`'s `_meta` the runtime names the caller it took from the subject the @@ -799,13 +813,17 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an input = json.RawMessage("{}") } tool := Tool{Name: name, Description: t.Description, Input: input, - Run: func(caller string, args json.RawMessage) (json.RawMessage, error) { + Run: nil, RunWithin: func(caller string, args json.RawMessage, within time.Duration) (json.RawMessage, error) { if len(args) == 0 || string(args) == "null" { args = json.RawMessage("{}") } call, end := mesh.Begin(caller) defer end() - res, err := asking("tools/call", callParams(full, caller, call, args), CallTimeout) + bound := CallTimeout + if within > 0 { + bound = within + } + res, err := asking("tools/call", callParams(full, caller, call, args), bound) if err != nil { if errors.Is(err, errNoAnswer) { // Its caller is gone by now and nobody else would hear of it (issue 1966). @@ -841,6 +859,9 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an b, _ := json.Marshal(text) return b, nil }} + tool.Run = func(caller string, args json.RawMessage) (json.RawMessage, error) { + return tool.RunWithin(caller, args, 0) + } if _, seen := groups[under]; !seen { order = append(order, under) } diff --git a/node-tools/internal/launch/state_failed_test.go b/node-tools/internal/launch/state_failed_test.go index 8742e0b..c002136 100644 --- a/node-tools/internal/launch/state_failed_test.go +++ b/node-tools/internal/launch/state_failed_test.go @@ -82,7 +82,8 @@ while IFS= read -r line; do case "$line" in *'"method":"initialize"'*) echo started >&2; printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;; *'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[{"name":"t","description":"d"}]}}\n' "$id" ;; - *'"method":"tools/call"'*) [ "$MODE" = long ] && printf '%4096s\n' x ;; + *'"method":"tools/call"'*) [ "$MODE" = long ] && printf '%4096s\n' x + [ "$MODE" = late ] && { sleep 0.3; printf '{"jsonrpc":"2.0","id":%s,"result":{"content":[{"type":"text","text":"{}"}]}}\n' "$id"; } ;; esac done ` @@ -150,3 +151,28 @@ func TestAnOverlongLineAndAnUnansweredCallAreSaid(t *testing.T) { }) } } + +// Design 47 H §760: a verb served from a bundle waits the `timeout` its definition names, not the runner's own bound: a bundle +// that answers after CallTimeout is answered through Within and fails through Run. +func TestAToolWithinWaitsTheBoundItsVerbNames(t *testing.T) { + oldCall := CallTimeout + CallTimeout = 100 * time.Millisecond + t.Cleanup(func() { CallTimeout = oldCall }) + dir := t.TempDir() + entry := filepath.Join(dir, "bundle") + if err := os.WriteFile(entry, []byte(callingBundle), 0o755); err != nil { + t.Fatal(err) + } + l, err := Start("caller", entry, append(os.Environ(), "MODE=late"), &refusingState{}, func(string, ...any) {}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(l.Stop) + tool := l.Registrations[0].Tools[0] + if _, err := tool.Run("caller-1", nil); err == nil || !strings.Contains(err.Error(), "did not answer") { + t.Fatalf("a call that waits the runner's own bound of 100ms: %v, want it unanswered", err) + } + if got, err := tool.Within(5*time.Second).Run("caller-1", nil); err != nil || string(got) != "{}" { + t.Fatalf("a call that waits the verb's 5s: %s, %v, want its answer", got, err) + } +} diff --git a/node-tools/internal/runtime/caller_test.go b/node-tools/internal/runtime/caller_test.go index 611e7fe..86c4a0f 100644 --- a/node-tools/internal/runtime/caller_test.go +++ b/node-tools/internal/runtime/caller_test.go @@ -2,16 +2,18 @@ package runtime import ( "encoding/json" + "errors" + "runtime" "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/approval" "git.novox.be/novox/mesh-sdk/go/seatdef" + "github.com/novox/mesh-tools/node-tools/internal/bus" "github.com/nats-io/nats.go" - "github.com/novox/mesh-tools/node-tools/internal/bus" mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" ) @@ -66,15 +68,14 @@ func TestACallOnTheCallerNamedSubjectReachesTheModuleWithItsCaller(t *testing.T) } } -// deskDefined is the seat list's definition of the fixture seat node-desk, its verb who taking its class's default. +// deskDefined is the seat list's definition of the fixture seat node-desk, its verb who. var deskDefined = WithSeatDefinitions(func() map[string]seatdef.Definition { return map[string]seatdef.Definition{"node-desk": {Seat: "node-desk", Verbs: []seatdef.Verb{{Name: "who"}}}} }) // The subject that names no caller is still served for a module's tool, for one release (retired by hq issue 464), and says -// so: the module is named no caller, whatever the body claims. A seat's verb refuses it before the bundle runs (design 47 G -// §610; lead answers A-165, A-179), and refuses every caller while the seat's definition is unread, names no such verb or -// opens it to none. +// so: the module is named no caller, whatever the body claims. A seat's verb answers every caller, an unproven one too (design 47 +// H §752). func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) { mesh := mt.New(t) mesh.Issue(t, mt.MembershipOf("ledger", "anchor", true, map[string][]string{"node-desk": {"who"}})) @@ -102,7 +103,7 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) { } return err }) - // Every caller is admitted, an unproven one too (the operator, 2026-11-15 23:25 CET). + // Every caller is answered, an unproven one too (design 47 H §752). if strings.HasPrefix(subject, "mesh.seat.") { if reply.Error != "" { t.Errorf("%s: an unproven caller of a seat's verb was refused: %q, want answered", subject, reply.Error) @@ -113,21 +114,6 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) { t.Errorf("%s: the module was named caller %q (error %q), want none", subject, reply.Result.Caller, reply.Error) } } - for _, c := range []struct { - defs map[string]seatdef.Definition - says string - }{ - {nil, ""}, - {map[string]seatdef.Definition{"node-desk": {Seat: "node-desk"}}, ""}, - {map[string]seatdef.Definition{"node-desk": {Seat: "node-desk", Verbs: []seatdef.Verb{{Name: "who", Callers: &seatdef.Callers{List: []string{}}}}}}, ""}, - {map[string]seatdef.Definition{"node-desk": {Seat: "node-desk", Verbs: []seatdef.Verb{{Name: "who", Class: seatdef.ClassAct}}}}, ""}, - } { - defs := c.defs - err := (&consumers{definitions: func() map[string]seatdef.Definition { return defs }}).admitted("node-desk", "who", "workstation.node-tools") - if c.says == "" && err != nil || c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)) { - t.Errorf("a proven caller with the definitions %v: %v, want %q", c.defs, err, c.says) - } - } // Design 47 G A-249: a bundle's seat call is received by the runner in chunks only for a verb whose definition says `chunks`, // of any machine: a verb a node holds is asked of the node that ran the ask (A-254). t.Run("OnlyAChunkedSeatVerbIsAskedInChunks", func(t *testing.T) { @@ -150,6 +136,108 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) { t.Error("a runner that holds no seat definitions asks no chunks") } }) + // Design 47 H §760: a bundle's ask waits the `timeout` its verb's definition names, whole seconds; a verb that names none, a + // seat the definitions do not hold, a module's tool and a runner that holds no definitions wait the runner's own bound (zero). + t.Run("AVerbsTimeoutIsTheBoundItsAskWaits", func(t *testing.T) { + forty, six := 40, 600 + defs := map[string]seatdef.Definition{"mesh-broker": {Verbs: []seatdef.Verb{{Name: "rates", Timeout: &forty}, {Name: "slow", Chunks: true, Timeout: &six}, {Name: "plain"}}}} + for _, row := range []struct { + key string + want time.Duration + way string + }{ + {"seat:mesh-broker.rates", 40 * time.Second, "plain"}, + {"seat:mesh-broker.rates@home-server", 40 * time.Second, "plain"}, + {"seat:mesh-broker.slow", 600 * time.Second, "chunked"}, + {"seat:mesh-broker.plain", 0, "plain"}, + {"seat:mesh-unread.rates", 0, "plain"}, + {"mesh-delivery-indexer.indexer", 0, "plain"}, + } { + var got time.Duration + var way string + c := &consumers{definitions: func() map[string]seatdef.Definition { return defs }, + askPlain: func(_, _ string, _ any, within time.Duration) (bus.Answered, error) { + got, way = within, "plain" + return bus.Answered{}, nil + }, + askChunked: func(_, _ string, _ any, within time.Duration) (bus.Answered, error) { + got, way = within, "chunked" + return bus.Answered{}, nil + }} + if _, err := (&moduleBus{all: c, module: "m"}).Ask(json.RawMessage(`{"key":"` + row.key + `","body":{}}`)); err != nil { + t.Fatal(err) + } + if got != row.want || way != row.way { + t.Errorf("Ask %s waited %s (%s), want %s (%s)", row.key, got, way, row.want, row.way) + } + } + if got := (&consumers{}).withinOf("seat:mesh-broker.rates"); got != 0 { + t.Errorf("a runner that holds no definitions waits %s, want its own bound", got) + } + }) + // Design 47 G A-490m: a verb of a seat the list defines whose definition does not name it is read again once and never asked + // without its call; a seat the list does not define, a module's tool and a bundle asking through moduleBus.Ask behave as before. + t.Run("AVerbTheDefinitionDoesNotNameIsReadAgainAndNeverAskedWithoutItsCall", func(t *testing.T) { + stale := map[string]seatdef.Definition{"mesh-wants": {Verbs: []seatdef.Verb{{Name: "revision"}}}} + fresh := map[string]seatdef.Definition{"mesh-wants": {Verbs: []seatdef.Verb{{Name: "revision"}, {Name: "input", Chunks: true}}}} + for _, row := range []struct { + name string + key string + again func(held *map[string]seatdef.Definition) error + wantWay string // "plain", "chunked" or "" for a refusal + wantErr string + wantReads int + }{ + {"a defined seat's verb the read names", "seat:mesh-wants.input", func(h *map[string]seatdef.Definition) error { *h = fresh; return nil }, "chunked", "", 1}, + {"a read that fails", "seat:mesh-wants.input", func(h *map[string]seatdef.Definition) error { return errors.New("the seat does not answer") }, "", "does not answer", 1}, + {"a read that still does not name it", "seat:mesh-wants.input@node-a", func(h *map[string]seatdef.Definition) error { return nil }, "", "names no verb input", 1}, + {"a seat the list does not define", "seat:issue-tracker.list", func(h *map[string]seatdef.Definition) error { return nil }, "plain", "", 0}, + {"a module's tool", "mesh-delivery-indexer.indexer", func(h *map[string]seatdef.Definition) error { return nil }, "plain", "", 0}, + {"a verb the list names without chunks", "seat:mesh-wants.revision", func(h *map[string]seatdef.Definition) error { return nil }, "plain", "", 0}, + } { + t.Run(row.name, func(t *testing.T) { + held, reads, way := stale, 0, "" + c := &consumers{definitions: func() map[string]seatdef.Definition { return held }, reread: func() error { reads++; return row.again(&held) }, + askPlain: func(_, _ string, _ any, _ time.Duration) (bus.Answered, error) { + way = "plain" + return bus.Answered{}, nil + }, + askChunked: func(_, _ string, _ any, _ time.Duration) (bus.Answered, error) { + way = "chunked" + return bus.Answered{}, nil + }} + _, err := (&moduleBus{all: c, module: "m"}).Ask(json.RawMessage(`{"key":"` + row.key + `","body":{}}`)) + if way != row.wantWay || reads != row.wantReads || (row.wantErr == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), row.wantErr)) { + t.Errorf("Ask %s: asked %q, %v after %d reads, want %q, %q after %d", row.key, way, err, reads, row.wantWay, row.wantErr, row.wantReads) + } + }) + } + // the asks that wait on a read share it + release, reads := make(chan struct{}), 0 + held := stale + c := &consumers{definitions: func() map[string]seatdef.Definition { return held }, reread: func() error { reads++; <-release; held = fresh; return nil }} + done := make(chan bool, 3) + for i := 0; i < 3; i++ { + go func() { got, err := c.chunkedAsked("seat:mesh-wants.input"); done <- got && err == nil }() + } + for deadline := time.Now().Add(5 * time.Second); c.rereadWaiting() < 2; runtime.Gosched() { + if time.Now().After(deadline) { + t.Fatal("two asks never joined the read running") + } + } + close(release) + for i := 0; i < 3; i++ { + if !<-done { + t.Error("an ask waiting on the shared read was not answered chunked") + } + } + if reads != 1 { + t.Errorf("%d reads of the seat list for three waiting asks, want 1", reads) + } + if got, err := (&consumers{}).chunkedAsked("seat:mesh-wants.input"); got || err != nil { + t.Errorf("a runner that holds no definitions asks no chunks: %v, %v", got, err) + } + }) // **The tool runner answers a held seat's marked verb by caller and serves the seat's `approvals`** (design 47 G §677; lead // answers A-265, A-271), with zsh's node-login-shell `execute` as the case: the marked verb reaches the gate with the caller // its subject names, and `approvals` is answered from the runner's own record of the calls its gate holds, with the diff --git a/node-tools/internal/runtime/gate.go b/node-tools/internal/runtime/gate.go index 70e2feb..499aef4 100644 --- a/node-tools/internal/runtime/gate.go +++ b/node-tools/internal/runtime/gate.go @@ -1,7 +1,9 @@ package runtime import ( + "fmt" "strings" + "time" "git.novox.be/novox/mesh-sdk/go/approval" "git.novox.be/novox/mesh-sdk/go/seatdef" @@ -47,18 +49,12 @@ func WithGate(g Gate) Option { return func(c *consumers) { c.gate = g } } var _ Gate = (*approval.Gate)(nil) -// WithSeatDefinitions refuses, where a seat's verb is answered, a caller outside the verb's effective callers, before the -// bundle runs (design 47 G §610; lead answers A-165, A-179): defs are the seat definitions the runner's seat list holds. -// Without it a verb is answered for every caller, as before. +// WithSeatDefinitions gives the runner the seat definitions its seat list holds: a verb's `chunks` and `timeout` are read from them +// (design 47 H §760). No definition names who may call: every caller is answered (design 47 H §752). func WithSeatDefinitions(defs func() map[string]seatdef.Definition) Option { return func(c *consumers) { c.definitions = defs } } -// admitted says why a seat's verb is refused to a caller: never. Every caller is admitted, whether or not the seat's definition -// is read and whatever callers it names (the operator, 2026-11-15 23:25 CET: every module, agent and operator may call anything; -// approval stays as the mark on a verb, held by the gate). -func (c *consumers) admitted(seat, verb, caller string) error { return nil } - // unservedMarks is, of the module's marked tools, those its bundle's registrations under its own name do not register. func (r *runner) unservedMarks(module string, regs []launch.Registration) []string { if r.events.gate == nil { @@ -75,6 +71,91 @@ func (r *runner) unservedMarks(module string, regs []launch.Registration) []stri return r.events.gate.Unserved(module, r.fingerprintOf(module), registered) } +// WithSeatDefinitionsAgain gives the runner the way to read the seat list again (the seat list reader's Reload): a seat verb a bundle asks +// that the definitions held do not name is looked up again after it, never taken for a verb that sends no chunks. +func WithSeatDefinitionsAgain(again func() error) Option { + return func(c *consumers) { c.reread = again } +} + +// chunkedAsked is chunked for a verb a bundle asks, where the answer decides whether the runner chooses the `call` the holder +// requires (AskAsChunked) (design 47 G A-490m). A verb of a seat the definitions held DEFINE, whose definition does not name it, +// is not taken for one that sends no chunks, since the holder then refuses an ask that carries no call: the seat list is read +// again once (one read shared by the asks that wait on it) and a verb still unnamed is an error saying so. An ask of a seat the +// definitions do not define at all, a module's tool and a runner that holds no definitions (nil) are asked as before. +func (c *consumers) chunkedAsked(key string) (bool, error) { + if c.definitions == nil || !strings.HasPrefix(key, "seat:") { + return false, nil + } + name, _, _ := strings.Cut(key, "@") + seat, verb, ok := strings.Cut(strings.TrimPrefix(name, "seat:"), ".") + if !ok { + return false, nil + } + find := func() (chunks, defined, named bool) { + d, defined := c.definitions()[seat] + for _, v := range d.Verbs { + if v.Name == verb { + return v.Chunks, true, true + } + } + return false, defined, false + } + chunks, defined, named := find() + if named || !defined { + return chunks, nil + } + var why string + if c.reread != nil { + if err := c.rereadShared(); err != nil { + why = " (reading the seat list again failed: " + err.Error() + ")" + } + chunks, defined, named = find() + if named { + return chunks, nil + } + if !defined { + return false, nil + } + } + return false, fmt.Errorf("the definition of the seat %s names no verb %s, so the runner cannot tell whether it sends its answer in chunks and does not ask it%s", seat, verb, why) +} + +// rereadShared reads the seat list again; asks that arrive while a read runs wait for it and share its result. +func (c *consumers) rereadShared() error { + c.rereadMu.Lock() + if f := c.rereading; f != nil { + f.waiters++ + c.rereadMu.Unlock() + <-f.done + return f.err + } + f := &rereadFlight{done: make(chan struct{})} + c.rereading = f + c.rereadMu.Unlock() + f.err = c.reread() + c.rereadMu.Lock() + c.rereading = nil + c.rereadMu.Unlock() + close(f.done) + return f.err +} + +type rereadFlight struct { + done chan struct{} + err error + waiters int // the asks that joined the read (under rereadMu) +} + +// rereadWaiting is how many asks have joined the read now running; a test waits on it instead of sleeping. +func (c *consumers) rereadWaiting() int { + c.rereadMu.Lock() + defer c.rereadMu.Unlock() + if c.rereading == nil { + return 0 + } + return c.rereading.waiters +} + // chunked says whether key names a seat's verb whose definition says `chunks` (design 47 G A-249): its answer is received in // chunks by the runner, whose call hands the bundle the whole. func (c *consumers) chunked(key string) bool { @@ -93,3 +174,23 @@ func (c *consumers) chunked(key string) bool { } return false } + +// withinOf is how long a call of key waits for its answer: the `timeout` the verb's definition names, in whole seconds (design 47 +// H §760, H-A-43); zero, which the bus reads as the tool runner's own bound, for a key that is no seat's verb, a seat the +// definitions do not hold, a verb they do not name and a verb that names none. +func (c *consumers) withinOf(key string) time.Duration { + if c.definitions == nil || !strings.HasPrefix(key, "seat:") { + return 0 + } + name, _, _ := strings.Cut(key, "@") + seat, verb, ok := strings.Cut(strings.TrimPrefix(name, "seat:"), ".") + if !ok { + return 0 + } + for _, v := range c.definitions()[seat].Verbs { + if v.Name == verb && v.Timeout != nil { + return time.Duration(*v.Timeout) * time.Second + } + } + return 0 +} diff --git a/node-tools/internal/runtime/gate_env_test.go b/node-tools/internal/runtime/gate_env_test.go index abf3bc9..90c99ac 100644 --- a/node-tools/internal/runtime/gate_env_test.go +++ b/node-tools/internal/runtime/gate_env_test.go @@ -6,16 +6,16 @@ import ( mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" ) -// Design 47 G lead answers A-377: the call a module's tool reaches the gate by carries the module's composed -// environment, so the gate tells a first-view tool (the one whose MESH_VIEW_CALLERS_ the bundle holds) from an -// ordinary one while its catalog half is unread. +// The call a module's tool reaches the gate by carries the module's composed environment. The gate no longer tells a first-view tool +// by an environment word: it knows one by its name forms _first_views and _first_view_again (H-A-69), and the runner +// sets no view-callers word. func TestAModuleToolsCallCarriesItsBundlesComposedEnvironment(t *testing.T) { for name, tc := range map[string]struct { envs map[string]map[string]string want bool }{ - "a first-view tool": {map[string]map[string]string{"ledger": {"MESH_VIEW_CALLERS_WHO": "grants"}}, true}, - "an ordinary tool": {nil, false}, + "a module with an environment": {map[string]map[string]string{"ledger": {"LEDGER_WORD": "x"}}, true}, + "a module with none": {nil, false}, } { t.Run(name, func(t *testing.T) { mesh := mt.New(t) @@ -32,11 +32,11 @@ func TestAModuleToolsCallCarriesItsBundlesComposedEnvironment(t *testing.T) { var found bool for _, c := range gate.calls { if c.Seat == "" && c.Name == "who" { - _, found = c.Env["MESH_VIEW_CALLERS_WHO"] + _, found = c.Env["LEDGER_WORD"] } } if found != tc.want { - t.Errorf("the gate saw the first-view word: %v, want %v (calls %+v)", found, tc.want, gate.calls) + t.Errorf("the gate saw the composed word: %v, want %v (calls %+v)", found, tc.want, gate.calls) } }) } diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index f248a02..144459f 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -314,7 +314,7 @@ func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, log } logf("%s", line) conn.OnMembership(func(bus.Membership) { go func() { r.refreshRepairs(); conn.Flush() }() }) - stopSeats, again := serveSeats(conn, r.modules, r.registrations, r.events.gated, r.events.admitted, r.events.chunked, r.approvalsOf(), + stopSeats, again := serveSeats(conn, r.modules, r.registrations, r.events.gated, r.events.chunked, r.events.withinOf, r.approvalsOf(), r.fingerprintOf, logf) r.seatsAgain = again r.stops = append(r.stops, stopSeats) @@ -733,7 +733,7 @@ func subjectsOf(conn *bus.Conn, module, tool string) []string { // whenever a membership changes, and when again is called because a bundle was loaded again. Whether // this machine holds the seat is the bus's to decide. It answers its stop and again. func serveSeats(conn *bus.Conn, modules []string, current func() []registration, gated gatedRun, - admitted func(seat, verb, caller string) error, chunked func(key string) bool, approvals func(caller string) (any, error), + chunked func(key string) bool, within func(key string) time.Duration, approvals func(caller string) (any, error), fingerprintOf func(string) string, logf func(string, ...any)) (func(), func()) { var mu sync.Mutex var stops []func() @@ -773,9 +773,6 @@ func serveSeats(conn *bus.Conn, modules []string, current func() []registration, if v.Verb == seatdef.ApprovalsVerb && approvals != nil { seat := v.Seat stop, err := conn.HandleSubjectCalled(v.Subject, func(caller string, _ json.RawMessage) (any, error) { - if err := admitted(seat, seatdef.ApprovalsVerb, caller); err != nil { - return nil, err - } return approvals(caller) }) if err != nil { @@ -791,15 +788,12 @@ func serveSeats(conn *bus.Conn, modules []string, current func() []registration, logf("[mesh-tools] %s claims %s and implements no %s, which that seat promises; not served", module, v.Seat, v.Verb) continue } - // Its callers checked, then through the approval gate, before the bundle (design 47 G §610, A-179; design 46 + // Through the approval gate, for any caller, before the bundle (design 47 H §752; design 46 // 7-approval-mark §20). module, v := module, v handle := func(caller string, body json.RawMessage) (any, error) { - if err := admitted(v.Seat, v.Verb, caller); err != nil { - return nil, err - } answer, err := gated(approval.Call{Address: v.Seat + "." + v.Verb, Module: module, Seat: v.Seat, Name: v.Verb, - Fingerprint: fingerprintOf(module), Caller: caller, Args: argsOf(body), Description: t.Description}, t, caller) + Fingerprint: fingerprintOf(module), Caller: caller, Args: argsOf(body), Description: t.Description}, t.Within(within("seat:"+v.Seat+"."+v.Verb)), caller) if err != nil || !chunked("seat:"+v.Seat+"."+v.Verb) { return answer, err } @@ -812,7 +806,7 @@ func serveSeats(conn *bus.Conn, modules []string, current func() []registration, continue } stops = append(stops, stop) - logf("[mesh-tools] serving %s's %s on %s, admitted where %s holds the seat", v.Seat, v.Verb, v.Subject, module) + logf("[mesh-tools] serving %s's %s on %s, where %s holds the seat", v.Seat, v.Verb, v.Subject, module) // A verb of a node-scoped seat is also answered on the subject that names no machine, by any one holder, in // one queue group per seat and verb (lead answer A-393). if open, ok := anyHolderSubject(v, m.Node); ok && !have[open] { @@ -869,8 +863,14 @@ type consumers struct { sessions sessionTable // gate is the approval gate every call is served through (gate.go); nil serves every call as before. gate Gate - // definitions are the seat definitions a seat verb's callers are read from (gate.go); nil admits every caller. + // definitions are the seat definitions a seat verb's `chunks` and `timeout` are read from (gate.go); nil reads none. definitions func() map[string]seatdef.Definition + // reread reads the seat list again (gate.go, chunkedAsked); nil reads nothing. + reread func() error + rereadMu sync.Mutex + rereading *rereadFlight + // askPlain and askChunked are how a bundle's ask is sent (the bus connection's AskAs and AskAsChunked); a test gives its own. + askPlain, askChunked func(module, key string, body any, within time.Duration) (bus.Answered, error) } type moduleEvents struct { @@ -999,11 +999,21 @@ func (b *moduleBus) Ask(params json.RawMessage) (json.RawMessage, error) { if len(asked.Body) == 0 { body = map[string]any{} } - ask := b.all.conn.AskAs - if b.all.chunked(asked.Key) { - ask = b.all.conn.AskAsChunked // the runner receives the chunks of the answer; the bundle receives none (design 47 G A-249) + ask := b.all.askPlain + if ask == nil { + ask = b.all.conn.AskAsWithin } - answered, err := ask(b.module, asked.Key, body) + chunks, err := b.all.chunkedAsked(asked.Key) + if err != nil { + return nil, err + } + if chunks { + if ask = b.all.askChunked; ask == nil { + ask = b.all.conn.AskAsChunkedWithin + } + // the runner receives the chunks of the answer; the bundle receives none (design 47 G A-249) + } + answered, err := ask(b.module, asked.Key, body, b.all.withinOf(asked.Key)) if err != nil { return nil, err } diff --git a/node-tools/internal/runtime/runtime_test.go b/node-tools/internal/runtime/runtime_test.go index 1c48105..2d07444 100644 --- a/node-tools/internal/runtime/runtime_test.go +++ b/node-tools/internal/runtime/runtime_test.go @@ -328,8 +328,7 @@ func TestANodeScopedSeatVerbIsAlsoAnsweredWithNoMachine(t *testing.T) { return tool.Run(caller, nil) } nodeTools.Follow("theta") - stop, _ := serveSeats(nodeTools, []string{"theta"}, regs, gated, func(_, _, _ string) error { return nil }, - func(string) bool { return false }, nil, func(string) string { return "" }, (&mt.Logs{}).Logf) + stop, _ := serveSeats(nodeTools, []string{"theta"}, regs, gated, func(string) bool { return false }, func(string) time.Duration { return 0 }, nil, func(string) string { return "" }, (&mt.Logs{}).Logf) defer stop() for _, key := range []string{"seat:node-shelf.list@anchor", "seat:node-shelf.list"} { t.Run(key, func(t *testing.T) { diff --git a/node-tools/test/fixtures/whole-bundle.py b/node-tools/test/fixtures/whole-bundle.py new file mode 100755 index 0000000..5d8863b --- /dev/null +++ b/node-tools/test/fixtures/whole-bundle.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# A tools bundle for the tool runner's whole test (design 47 H §774): MCP over stdio, no SDK. A tool `who`, the seat node-desk's verbs +# `who` (answers at once) and `slow` (answers after three seconds), and a tool `reach` that asks the seat verb node-desk.fresh as the +# bundle does, through the runtime's `mesh/ask`, and says what came back. +import json, sys, time + +def say(m): + m["jsonrpc"] = "2.0"; sys.stdout.write(json.dumps(m) + "\n"); sys.stdout.flush() + +def text(rid, value): + say({"id": rid, "result": {"content": [{"type": "text", "text": json.dumps(value)}]}}) + +TOOLS = [ + {"name": "who", "description": "the module's own tool", "inputSchema": {"type": "object", "properties": {}}}, + {"name": "reach", "description": "asks the seat verb node-desk.fresh", "inputSchema": {"type": "object", "properties": {}}}, + {"name": "node-desk.who", "description": "the seat's verb", "inputSchema": {"type": "object", "properties": {}}}, + {"name": "node-desk.slow", "description": "the seat's verb that answers late", "inputSchema": {"type": "object", "properties": {}}}, +] +for line in sys.stdin: + req = json.loads(line); rid = req.get("id"); m = req.get("method"); p = req.get("params") or {} + if m == "initialize": + say({"id": rid, "result": {"protocolVersion": "2025-03-26", "capabilities": {"tools": {}}, "serverInfo": {"name": "ledger", "version": "1"}}}) + elif m == "tools/list": + say({"id": rid, "result": {"tools": TOOLS}}) + elif m == "tools/call": + name = p.get("name") + if name == "node-desk.slow": + time.sleep(3) + if name == "reach": + say({"id": "q1", "method": "mesh/ask", "params": {"key": "seat:node-desk.fresh", "body": {}}}) + for answer in sys.stdin: + a = json.loads(answer) + if a.get("id") == "q1": + text(rid, a.get("result") if "error" not in a else {"error": a["error"]}) + break + else: + text(rid, {"tool": name}) + elif rid is not None: + say({"id": rid, "result": {}})