diff --git a/node-tools/cmd/node-tools/main.go b/node-tools/cmd/node-tools/main.go index 1621e83..154455c 100644 --- a/node-tools/cmd/node-tools/main.go +++ b/node-tools/cmd/node-tools/main.go @@ -90,7 +90,8 @@ func main() { Build: os.Getenv(runtime.FingerprintWord), Hear: hearOne(conn)}) // A failed read or write of the state approvals is made again at the next membership change the runner hears (A-157). conn.OnMembership(func(bus.Membership) { go gate.StateAgain() }) - opts = append(opts, runtime.WithGate(gate), runtime.WithSeatDefinitions(definitionsOf(seatList))) + opts = append(opts, runtime.WithGate(gate), runtime.WithSeatDefinitions(definitionsOf(seatList)), + runtime.WithSeatDefinitionsAgain(rereadOf(seatList))) // A run still unrecorded is asked again at each start and when the hand-act seat's holder is heard again (lead answer A-157). if stopBounds, err := gate.WatchBounds(); err != nil { log.Printf("[mesh-tools] the gate's bound keys are not watched (%v): an unanswered hand-act is said in the log only", err) diff --git a/node-tools/cmd/node-tools/seatlist.go b/node-tools/cmd/node-tools/seatlist.go index a545710..db08cd0 100644 --- a/node-tools/cmd/node-tools/seatlist.go +++ b/node-tools/cmd/node-tools/seatlist.go @@ -63,11 +63,25 @@ func startSeatListOn(b seatlist.Bus, bound <-chan time.Time, logf func(string, . return nil } catalogue.UseSeats(reader.Seats()) + // a first read that ended without an answer is no failure of the reader, which holds the bootstrap and waits for the seat's next + // event: so it is said here, where nothing else would say it, whether it ends before the bound or after it (a read the bus does + // not answer is asked again for about 30 s, so it ends after the bound); the runner then reads again where a bundle asks a verb of + // a seat the list defines that its definition does not name (design 47 G A-490m) + sayUnanswered := func() { + if st := reader.Status(); !st.Heard { + logf("node-tools: the first read of the seat list ended without an answer (%s); the runner serves with the bootstrapped seats and reads again when a bundle asks a verb of a seat the list defines that its definition does not name", st.LastError) + } + } select { case <-reader.Ready(): catalogue.UseSeats(reader.Seats()) + sayUnanswered() case <-bound: logf("node-tools: the seat list is not read yet; the runner serves with the bootstrapped seats and reads on") + go func() { + <-reader.Ready() + sayUnanswered() + }() } return reader } @@ -82,3 +96,14 @@ func definitionsOf(reader *seatlist.Reader) func() map[string]seatdef.Definition return reader.Definitions() } } + +// rereadOf is how the runner reads the seat list again where a verb is not named by the definitions it holds; it reads nothing +// while the reader did not start. +func rereadOf(reader *seatlist.Reader) func() error { + return func() error { + if reader == nil { + return fmt.Errorf("the seat list reader did not start") + } + return reader.Reload() + } +} diff --git a/node-tools/cmd/node-tools/seatlist_test.go b/node-tools/cmd/node-tools/seatlist_test.go index fda2420..2f5f941 100644 --- a/node-tools/cmd/node-tools/seatlist_test.go +++ b/node-tools/cmd/node-tools/seatlist_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "strings" + "sync" "sync/atomic" "testing" "time" @@ -32,19 +33,23 @@ func TestTheRunnerWaitsForTheFirstReadOfTheSeatListOnlyToItsBound(t *testing.T) wantSaid string wantReady bool }{ - {"the first read ends before the bound", true, "", true}, + {"the first read ends before the bound, unanswered", true, "the first read of the seat list ended without an answer", true}, {"the bound passes with the seat silent", false, "the seat list is not read yet", false}, + {"the bound passes, then the read ends unanswered", false, "the first read of the seat list ended without an answer", false}, } { t.Run(row.name, func(t *testing.T) { b := &busOfAStandIn{release: make(chan struct{})} bound := make(chan time.Time, 1) + var mu sync.Mutex var said []string + saidAgain := make(chan struct{}, 4) + say := func(f string, a ...any) { mu.Lock(); said = append(said, f); mu.Unlock(); saidAgain <- struct{}{} } if row.release { close(b.release) } else { bound <- time.Time{} } - r := startSeatListOn(b, bound, func(f string, a ...any) { said = append(said, f) }) + r := startSeatListOn(b, bound, say) if r == nil { t.Fatal("no reader") } @@ -57,12 +62,25 @@ func TestTheRunnerWaitsForTheFirstReadOfTheSeatListOnlyToItsBound(t *testing.T) if ended != row.wantReady { t.Errorf("first read ended = %v, want %v", ended, row.wantReady) } - if got := strings.Join(said, ";"); !strings.Contains(got, row.wantSaid) || (row.wantSaid == "" && got != "") { - t.Errorf("said %q, want %q", got, row.wantSaid) - } if !row.release { close(b.release) <-r.Ready() + // the read that ends after the bound is said by a goroutine of its own: wait for it, to a deadline that only a fault reaches + if strings.Contains(row.wantSaid, "first read") { + for i := 0; i < 2; i++ { + select { + case <-saidAgain: + case <-time.After(5 * time.Second): + t.Fatal("the unanswered end of the first read was never said") + } + } + } + } + mu.Lock() + got := strings.Join(said, ";") + mu.Unlock() + if !strings.Contains(got, row.wantSaid) { + t.Errorf("said %q, want %q", got, row.wantSaid) } }) } diff --git a/node-tools/internal/runtime/caller_test.go b/node-tools/internal/runtime/caller_test.go index 611e7fe..40fb1ff 100644 --- a/node-tools/internal/runtime/caller_test.go +++ b/node-tools/internal/runtime/caller_test.go @@ -2,16 +2,18 @@ package runtime import ( "encoding/json" + "errors" + "runtime" "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/approval" "git.novox.be/novox/mesh-sdk/go/seatdef" + "github.com/novox/mesh-tools/node-tools/internal/bus" "github.com/nats-io/nats.go" - "github.com/novox/mesh-tools/node-tools/internal/bus" mt "github.com/novox/mesh-tools/node-tools/internal/meshtest" ) @@ -150,6 +152,63 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) { t.Error("a runner that holds no seat definitions asks no chunks") } }) + // Design 47 G A-490m: a verb of a seat the list defines whose definition does not name it is read again once and never asked + // without its call; a seat the list does not define, a module's tool and a bundle asking through moduleBus.Ask behave as before. + t.Run("AVerbTheDefinitionDoesNotNameIsReadAgainAndNeverAskedWithoutItsCall", func(t *testing.T) { + stale := map[string]seatdef.Definition{"mesh-wants": {Verbs: []seatdef.Verb{{Name: "revision"}}}} + fresh := map[string]seatdef.Definition{"mesh-wants": {Verbs: []seatdef.Verb{{Name: "revision"}, {Name: "input", Chunks: true}}}} + for _, row := range []struct { + name string + key string + again func(held *map[string]seatdef.Definition) error + wantWay string // "plain", "chunked" or "" for a refusal + wantErr string + wantReads int + }{ + {"a defined seat's verb the read names", "seat:mesh-wants.input", func(h *map[string]seatdef.Definition) error { *h = fresh; return nil }, "chunked", "", 1}, + {"a read that fails", "seat:mesh-wants.input", func(h *map[string]seatdef.Definition) error { return errors.New("the seat does not answer") }, "", "does not answer", 1}, + {"a read that still does not name it", "seat:mesh-wants.input@node-a", func(h *map[string]seatdef.Definition) error { return nil }, "", "names no verb input", 1}, + {"a seat the list does not define", "seat:issue-tracker.list", func(h *map[string]seatdef.Definition) error { return nil }, "plain", "", 0}, + {"a module's tool", "mesh-delivery-indexer.indexer", func(h *map[string]seatdef.Definition) error { return nil }, "plain", "", 0}, + {"a verb the list names without chunks", "seat:mesh-wants.revision", func(h *map[string]seatdef.Definition) error { return nil }, "plain", "", 0}, + } { + t.Run(row.name, func(t *testing.T) { + held, reads, way := stale, 0, "" + c := &consumers{definitions: func() map[string]seatdef.Definition { return held }, reread: func() error { reads++; return row.again(&held) }, + askPlain: func(_, _ string, _ any) (bus.Answered, error) { way = "plain"; return bus.Answered{}, nil }, + askChunked: func(_, _ string, _ any) (bus.Answered, error) { way = "chunked"; return bus.Answered{}, nil }} + _, err := (&moduleBus{all: c, module: "m"}).Ask(json.RawMessage(`{"key":"` + row.key + `","body":{}}`)) + if way != row.wantWay || reads != row.wantReads || (row.wantErr == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), row.wantErr)) { + t.Errorf("Ask %s: asked %q, %v after %d reads, want %q, %q after %d", row.key, way, err, reads, row.wantWay, row.wantErr, row.wantReads) + } + }) + } + // the asks that wait on a read share it + release, reads := make(chan struct{}), 0 + held := stale + c := &consumers{definitions: func() map[string]seatdef.Definition { return held }, reread: func() error { reads++; <-release; held = fresh; return nil }} + done := make(chan bool, 3) + for i := 0; i < 3; i++ { + go func() { got, err := c.chunkedAsked("seat:mesh-wants.input"); done <- got && err == nil }() + } + for deadline := time.Now().Add(5 * time.Second); c.rereadWaiting() < 2; runtime.Gosched() { + if time.Now().After(deadline) { + t.Fatal("two asks never joined the read running") + } + } + close(release) + for i := 0; i < 3; i++ { + if !<-done { + t.Error("an ask waiting on the shared read was not answered chunked") + } + } + if reads != 1 { + t.Errorf("%d reads of the seat list for three waiting asks, want 1", reads) + } + if got, err := (&consumers{}).chunkedAsked("seat:mesh-wants.input"); got || err != nil { + t.Errorf("a runner that holds no definitions asks no chunks: %v, %v", got, err) + } + }) // **The tool runner answers a held seat's marked verb by caller and serves the seat's `approvals`** (design 47 G ยง677; lead // answers A-265, A-271), with zsh's node-login-shell `execute` as the case: the marked verb reaches the gate with the caller // its subject names, and `approvals` is answered from the runner's own record of the calls its gate holds, with the diff --git a/node-tools/internal/runtime/gate.go b/node-tools/internal/runtime/gate.go index 70e2feb..8777b3d 100644 --- a/node-tools/internal/runtime/gate.go +++ b/node-tools/internal/runtime/gate.go @@ -1,6 +1,7 @@ package runtime import ( + "fmt" "strings" "git.novox.be/novox/mesh-sdk/go/approval" @@ -75,6 +76,91 @@ func (r *runner) unservedMarks(module string, regs []launch.Registration) []stri return r.events.gate.Unserved(module, r.fingerprintOf(module), registered) } +// WithSeatDefinitionsAgain gives the runner the way to read the seat list again (the seat list reader's Reload): a seat verb a bundle asks +// that the definitions held do not name is looked up again after it, never taken for a verb that sends no chunks. +func WithSeatDefinitionsAgain(again func() error) Option { + return func(c *consumers) { c.reread = again } +} + +// chunkedAsked is chunked for a verb a bundle asks, where the answer decides whether the runner chooses the `call` the holder +// requires (AskAsChunked) (design 47 G A-490m). A verb of a seat the definitions held DEFINE, whose definition does not name it, +// is not taken for one that sends no chunks, since the holder then refuses an ask that carries no call: the seat list is read +// again once (one read shared by the asks that wait on it) and a verb still unnamed is an error saying so. An ask of a seat the +// definitions do not define at all, a module's tool and a runner that holds no definitions (nil) are asked as before. +func (c *consumers) chunkedAsked(key string) (bool, error) { + if c.definitions == nil || !strings.HasPrefix(key, "seat:") { + return false, nil + } + name, _, _ := strings.Cut(key, "@") + seat, verb, ok := strings.Cut(strings.TrimPrefix(name, "seat:"), ".") + if !ok { + return false, nil + } + find := func() (chunks, defined, named bool) { + d, defined := c.definitions()[seat] + for _, v := range d.Verbs { + if v.Name == verb { + return v.Chunks, true, true + } + } + return false, defined, false + } + chunks, defined, named := find() + if named || !defined { + return chunks, nil + } + var why string + if c.reread != nil { + if err := c.rereadShared(); err != nil { + why = " (reading the seat list again failed: " + err.Error() + ")" + } + chunks, defined, named = find() + if named { + return chunks, nil + } + if !defined { + return false, nil + } + } + return false, fmt.Errorf("the definition of the seat %s names no verb %s, so the runner cannot tell whether it sends its answer in chunks and does not ask it%s", seat, verb, why) +} + +// rereadShared reads the seat list again; asks that arrive while a read runs wait for it and share its result. +func (c *consumers) rereadShared() error { + c.rereadMu.Lock() + if f := c.rereading; f != nil { + f.waiters++ + c.rereadMu.Unlock() + <-f.done + return f.err + } + f := &rereadFlight{done: make(chan struct{})} + c.rereading = f + c.rereadMu.Unlock() + f.err = c.reread() + c.rereadMu.Lock() + c.rereading = nil + c.rereadMu.Unlock() + close(f.done) + return f.err +} + +type rereadFlight struct { + done chan struct{} + err error + waiters int // the asks that joined the read (under rereadMu) +} + +// rereadWaiting is how many asks have joined the read now running; a test waits on it instead of sleeping. +func (c *consumers) rereadWaiting() int { + c.rereadMu.Lock() + defer c.rereadMu.Unlock() + if c.rereading == nil { + return 0 + } + return c.rereading.waiters +} + // chunked says whether key names a seat's verb whose definition says `chunks` (design 47 G A-249): its answer is received in // chunks by the runner, whose call hands the bundle the whole. func (c *consumers) chunked(key string) bool { diff --git a/node-tools/internal/runtime/runtime.go b/node-tools/internal/runtime/runtime.go index f248a02..5d8d1d3 100644 --- a/node-tools/internal/runtime/runtime.go +++ b/node-tools/internal/runtime/runtime.go @@ -871,6 +871,12 @@ type consumers struct { gate Gate // definitions are the seat definitions a seat verb's callers are read from (gate.go); nil admits every caller. definitions func() map[string]seatdef.Definition + // reread reads the seat list again (gate.go, chunkedAsked); nil reads nothing. + reread func() error + rereadMu sync.Mutex + rereading *rereadFlight + // askPlain and askChunked are how a bundle's ask is sent (the bus connection's AskAs and AskAsChunked); a test gives its own. + askPlain, askChunked func(module, key string, body any) (bus.Answered, error) } type moduleEvents struct { @@ -999,9 +1005,19 @@ func (b *moduleBus) Ask(params json.RawMessage) (json.RawMessage, error) { if len(asked.Body) == 0 { body = map[string]any{} } - ask := b.all.conn.AskAs - if b.all.chunked(asked.Key) { - ask = b.all.conn.AskAsChunked // the runner receives the chunks of the answer; the bundle receives none (design 47 G A-249) + ask := b.all.askPlain + if ask == nil { + ask = b.all.conn.AskAs + } + chunks, err := b.all.chunkedAsked(asked.Key) + if err != nil { + return nil, err + } + if chunks { + if ask = b.all.askChunked; ask == nil { + ask = b.all.conn.AskAsChunked + } + // the runner receives the chunks of the answer; the bundle receives none (design 47 G A-249) } answered, err := ask(b.module, asked.Key, body) if err != nil {