The node's runtime is its modules' bus: it binds each module's consumer and hands its events to the bundle (hq ADR 0198)

A launched bundle's mesh/subscribe binds the module's own durable consumer — EVENTS, <node>_<module>,
by name as the module's own runtime bound it, so nothing is lost or replayed in the move — and every
event goes to each child of the module that subscribed as mesh/event, acknowledged only when all
answered, negatively acknowledged after a short delay when one failed or died, terminated when it is
not an event. mesh/ask calls a tool as the module. Every launched bundle is started again when it
exits, with backoff, since long-running code waits for no call. Requires SDK 0.1.6.

node-tools/internal/launch/launch.go

@@ -46,8 +46,26 @@ type Registration struct {
 	Tools  []Tool
 }
 
-// Publisher publishes an event a bundle asked the runtime to emit, as the bundle's module.
-type Publisher func(params json.RawMessage) error
+// Bus is what a launched bundle reaches the mesh through (novox/hq ADR 0193, ADR 0198): the runtime
+// publishes, asks and subscribes on the module's behalf. Subscribe is asked each time the module's
+// code subscribes; the runtime binds the module's consumer once and calls deliver for every event,
+// acknowledging it on the bus when deliver returns nil.
+type Bus interface {
+	Publish(params json.RawMessage) error
+	Ask(params json.RawMessage) (json.RawMessage, error)
+	Subscribe(deliver func(envelope json.RawMessage) error) error
+}
+
+// EventTimeout bounds how long a bundle has to handle one event before it is offered again.
+var EventTimeout = 2 * time.Minute
+
+// Restart backoff: a bundle that exits is started again at once, then after growing pauses while it
+// keeps exiting, back to at once once it has run a while.
+var (
+	RestartFirst  = 500 * time.Millisecond
+	RestartMost   = 30 * time.Second
+	RestartSettle = time.Minute
+)
 
 // Launched is a running bundle: what it registered, and how to stop it.
 type Launched struct {
@@ -103,10 +121,26 @@ var stackLine = regexp.MustCompile(`^\s+at\s`)
 
 // Start launches one bundle and learns its tools. It fails when the child cannot be started or does
 // not complete the handshake. A child that exits later is started again on its next call.
-func Start(module, entry string, env []string, publish Publisher, logf func(string, ...any)) (*Launched, error) {
+func Start(module, entry string, env []string, mesh Bus, logf func(string, ...any)) (*Launched, error) {
 	var mu sync.Mutex
 	var current *child
+	// The child that last subscribed is the one events are handed to: a restarted child subscribes
+	// again as its code is imported, and from then on the module's events are its.
+	var subscriber *child
 	stopped := false
+	deliver := func(envelope json.RawMessage) error {
+		mu.Lock()
+		c := subscriber
+		mu.Unlock()
+		if c == nil {
+			return errors.New(module + "'s bundle is not running to take its events")
+		}
+		_, err := c.ask(module, "mesh/event", map[string]any{"envelope": envelope}, EventTimeout)
+		return err
+	}
+	var restart func(after time.Duration)
+	var startedAt time.Time
+	pause := RestartFirst
 
 	start := func() (*child, error) {
 		cmd := exec.Command(entry)
@@ -163,18 +197,32 @@ func Start(module, entry string, env []string, publish Publisher, logf func(stri
 					logf("[mesh-tools] %s's bundle said something that is not a reply: %s", module, line)
 					continue
 				}
-				// The bundle asks the runtime to emit (ADR 0193): published as this module, answered
-				// once the bus has accepted it. Nothing else a bundle may ask.
+				// The bundle asks the runtime (ADR 0193, ADR 0198): to emit, to call a tool, to hand it the
+				// module's events. Each on the module's behalf, answered when done; nothing else.
 				if m.Method != "" {
 					go func(m message) {
-						if m.Method != "mesh/publish" {
+						var result any = map[string]any{}
+						var err error
+						switch m.Method {
+						case "mesh/publish":
+							err = mesh.Publish(m.Params)
+						case "mesh/ask":
+							var asked json.RawMessage
+							if asked, err = mesh.Ask(m.Params); err == nil {
+								result = asked
+							}
+						case "mesh/subscribe":
+							mu.Lock()
+							subscriber = c
+							mu.Unlock()
+							err = mesh.Subscribe(deliver)
+						default:
 							if len(m.ID) > 0 {
 								_ = c.write(map[string]any{"jsonrpc": "2.0", "id": m.ID,
 									"error": map[string]any{"code": -32601, "message": "the runtime answers no " + m.Method + " from a bundle"}})
 							}
 							return
 						}
-						err := publish(m.Params)
 						if len(m.ID) == 0 {
 							return
 						}
@@ -183,7 +231,7 @@ func Start(module, entry string, env []string, publish Publisher, logf func(stri
 								"error": map[string]any{"code": -32000, "message": err.Error()}})
 							return
 						}
-						_ = c.write(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": map[string]any{}})
+						_ = c.write(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": result})
 					}(m)
 					continue
 				}
@@ -220,13 +268,32 @@ func Start(module, entry string, env []string, publish Publisher, logf func(stri
 			c.mu.Unlock()
 			close(c.dead)
 			mu.Lock()
+			// Only a child that was serving is brought back; one that died in its own handshake was
+			// never accepted, and whoever started it was told why.
+			wasServing := current == c
 			if current == c {
 				current = nil
 			}
+			if subscriber == c {
+				subscriber = nil
+			}
 			wasStopped := stopped
+			// Started again at once if it ran a while; after a growing pause while it keeps exiting.
+			wait := RestartFirst
+			if time.Since(startedAt) < RestartSettle {
+				wait = pause
+				if pause *= 2; pause > RestartMost {
+					pause = RestartMost
+				}
+			} else {
+				pause = RestartFirst
+			}
 			mu.Unlock()
-			if !wasStopped {
-				logf("[mesh-tools] %s; started again on its next call", why)
+			if !wasStopped && wasServing {
+				// Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner —
+				// is not waiting for a call to bring it back, and a tool bundle back early costs nothing.
+				logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond))
+				restart(wait)
 			}
 		}()
 		if _, err := c.ask(module, "initialize", map[string]any{"protocolVersion": Protocol, "capabilities": map[string]any{},
@@ -248,19 +315,62 @@ func Start(module, entry string, env []string, publish Publisher, logf func(stri
 				return nil, err
 			}
 			mu.Lock()
-			current = fresh
+			if current == nil {
+				current = fresh
+				startedAt = time.Now()
+			} else {
+				_ = fresh.cmd.Process.Signal(syscall.SIGTERM)
+				fresh = current
+			}
 			c = fresh
 			mu.Unlock()
 		}
 		return c.ask(module, method, params, timeout)
 	}
 
+	restart = func(after time.Duration) {
+		go func() {
+			time.Sleep(after)
+			mu.Lock()
+			if stopped || current != nil {
+				mu.Unlock()
+				return
+			}
+			mu.Unlock()
+			fresh, err := start()
+			mu.Lock()
+			defer mu.Unlock()
+			if err != nil {
+				if !stopped {
+					wait := pause
+					if pause *= 2; pause > RestartMost {
+						pause = RestartMost
+					}
+					logf("[mesh-tools] %s's bundle did not start again: %v; trying in %s", module, err, wait)
+					go restart(wait)
+				}
+				return
+			}
+			if stopped {
+				_ = fresh.cmd.Process.Signal(syscall.SIGTERM)
+				return
+			}
+			if current == nil {
+				current = fresh
+				startedAt = time.Now()
+			} else {
+				_ = fresh.cmd.Process.Signal(syscall.SIGTERM)
+			}
+		}()
+	}
+
 	first, err := start()
 	if err != nil {
 		return nil, err
 	}
 	mu.Lock()
 	current = first
+	startedAt = time.Now()
 	mu.Unlock()
 
 	raw, err := asking("tools/list", map[string]any{}, HandshakeTimeout)