Serve a tools bundle as soon as it is in place, changed or gone, with no restart; say why a bundle cannot start (issue 1945)

node-tools loaded every bundle once at start, so a bundle delivered after
it started (mesh-files on shanks, 2026-10-25 12:08) went unserved until a
hand restart, and a missing file was said as "not executable". The
runtime now watches the entrypoints' directories with inotify, loads a
module again once a burst of changes settles, and discovery answers with
what is served now. A bundle that cannot start says it does not exist, is
not executable with its mode, or the exec error.

Tests: 1945/AC19-AC22 in internal/runtime/reload_test.go.
This commit is contained in:
2026-10-25 15:09:49 +01:00
parent 77b05089c8
commit b0ded57294
5 changed files with 858 additions and 130 deletions
+367 -129
View File
@@ -135,7 +135,51 @@ type registration struct {
tools []launch.Tool
}
// Run launches, binds and serves. It answers a stop function.
// loadedModule is one served module as its bundles were last loaded: what they registered, what of it
// is served, and each entrypoint's signature at that load, which tells the watch that it changed.
type loadedModule struct {
served Served
seen map[string]string // entrypoint → its signature when it was loaded
children []func()
regs []registration // every registration its bundles made
own []registration // those under the name of a module served here
failed string
names []string // the tools it serves, as module.tool
serving []func() // its tools' subscriptions
}
// unload stops serving the module's tools and stops its bundles.
func (l *loadedModule) unload() {
for i := len(l.serving) - 1; i >= 0; i-- {
l.serving[i]()
}
l.serving, l.names = nil, nil
for i := len(l.children) - 1; i >= 0; i-- {
l.children[i]()
}
l.children = nil
}
// runner is the runtime at work: every served module as last loaded, loaded again when its bundle
// appears, changes or goes (issue 1945, note 23).
type runner struct {
conn *bus.Conn
logf func(string, ...any)
envFor func(module string) []string
events *consumers
modules []string
isServed map[string]bool
mu sync.Mutex
of map[string]*loadedModule
verbs map[string]func() // each module's `tools` verb, where it is answered
seatsAgain func()
stops []func() // the watch, the announcement and the seats, in the order they began
}
// Run launches, binds and serves, and serves each module's bundle again when it appears, changes or
// goes, with no restart. It answers a stop function.
func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, logf func(string, ...any)) (func(), error) {
if account := os.Getenv(OperatorAccount); account != "" {
home := ""
@@ -144,153 +188,323 @@ func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, log
}
logf("[mesh-tools] the operator's account here is %s%s", account, home)
}
modules := make([]string, 0, len(served))
isServed := map[string]bool{}
r := &runner{conn: conn, logf: logf, isServed: map[string]bool{}, of: map[string]*loadedModule{},
verbs: map[string]func(){}}
for _, s := range served {
modules = append(modules, s.Module)
isServed[s.Module] = true
r.modules = append(r.modules, s.Module)
r.isServed[s.Module] = true
conn.Follow(s.Module)
}
node := conn.Node()
base := os.Environ()
envFor := func(module string) []string { return BundleEnv(base, envs[module], module, node) }
r.envFor = func(module string) []string { return BundleEnv(base, envs[module], module, node) }
// The module's events, for every child of it that subscribes (ADR 0198): one consumer per module,
// bound the first time any of its children subscribes, each event handed to every child that did.
events := &consumers{conn: conn, logf: logf, of: map[string]*moduleEvents{}}
r.events = &consumers{conn: conn, logf: logf, of: map[string]*moduleEvents{}}
failed := map[string]string{}
var registrations []registration
var stops []func()
for _, s := range served {
for _, entry := range s.Entrypoints {
path, _ := filepath.Abs(entry)
module := s.Module
fail := func(why string) {
failed[module] = why
logf("[mesh-tools] %s's bundle %s failed to load: %s; its tools are not served here", module, entry, why)
}
if !launch.Executable(path) {
fail(path + " is not executable; a bundle the runtime serves is started, never imported, and its build makes it executable (novox/hq ADR 0193)")
continue
}
child, err := launch.Start(module, path, envFor(module), events.forModule(module), logf)
if err != nil {
fail(err.Error())
continue
}
stops = append(stops, child.Stop)
for _, r := range child.Registrations {
registrations = append(registrations, registration{module: r.Module, owner: module, tools: r.Tools})
}
r.of[s.Module] = r.load(s)
}
for _, m := range r.modules {
if err := refusal(r.of[m].own); err != nil {
r.stop()
return nil, err
}
}
claimed := map[string]bool{}
for _, m := range modules {
if mem := conn.Membership(m); mem != nil {
for _, s := range mem.Seats {
claimed[s.Seat] = true
}
}
}
var own []registration
for _, r := range registrations {
switch {
case isServed[r.module]:
own = append(own, r)
case claimed[r.module]:
default:
logf(`[mesh-tools] %s registers tools under "%s", which is neither a module served here nor a seat one of them claims; not served until the mesh issues the claim`, r.owner, r.module)
}
}
stopAll := func() {
events.stopAll()
for i := len(stops) - 1; i >= 0; i-- {
stops[i]()
}
}
for _, r := range own {
seen := map[string]bool{}
for _, t := range r.tools {
if t.Name == ToolsVerb {
stopAll()
return nil, fmt.Errorf(`%s names a tool "%s", which is the verb the runtime answers for every module with what it serves (novox/hq ADR 0152) — refused, rename it`, r.module, ToolsVerb)
}
if seen[t.Name] {
stopAll()
return nil, fmt.Errorf("%s exposes two tools named %s — refused", r.module, t.Name)
}
seen[t.Name] = true
}
}
var names []string
byModule := map[string][]launch.Tool{}
for _, r := range own {
for _, t := range r.tools {
t := t
// Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365).
stop, err := conn.HandleCalled(r.module+"."+t.Name, func(caller string, body json.RawMessage) (any, error) {
return t.Run(caller, argsOf(body))
})
if err != nil {
logf("[mesh-tools] cannot serve %s.%s: %v", r.module, t.Name, err)
continue
}
names = append(names, r.module+"."+t.Name)
stops = append(stops, stop)
}
byModule[r.module] = append(byModule[r.module], r.tools...)
}
for _, module := range modules {
module := module
tools := byModule[module]
why := failed[module]
if len(tools) == 0 && why == "" {
continue // a pure-events module: silent, as it always was
}
stop, err := conn.Handle(module+"."+ToolsVerb, func(json.RawMessage) (any, error) {
answer := ToolsAnswer{Module: module, Tools: []ListedTool{}, Failed: why}
for _, t := range tools {
answer.Tools = append(answer.Tools, ListedTool{Name: t.Name, Description: t.Description,
Input: t.Input, Subjects: subjectsOf(conn, module, t.Name)})
}
return answer, nil
})
if err == nil {
stops = append(stops, stop)
}
}
failedNames := make([]string, 0, len(failed))
for _, m := range modules {
if _, f := failed[m]; f {
r.mu.Lock()
var names, failedNames []string
for _, m := range r.modules {
r.serve(r.of[m])
names = append(names, r.of[m].names...)
if r.of[m].failed != "" {
failedNames = append(failedNames, m)
}
}
line := fmt.Sprintf("[mesh-tools] serving %d tool(s) for %d module(s): %s", len(names), len(modules), orNone(names))
r.bindVerbs()
r.mu.Unlock()
line := fmt.Sprintf("[mesh-tools] serving %d tool(s) for %d module(s): %s", len(names), len(r.modules), orNone(names))
if len(failedNames) > 0 {
line += fmt.Sprintf("; not serving %s, whose bundle(s) failed to load", strings.Join(failedNames, ", "))
}
logf("%s", line)
stops = append(stops, serveSeats(conn, modules, registrations, logf))
stopSeats, again := serveSeats(conn, r.modules, r.registrations, logf)
r.seatsAgain = again
r.stops = append(r.stops, stopSeats)
// **What it serves, it announces** (novox/hq ADR 0197): the NATS services protocol's discovery,
// answered with what is served at the moment it is asked — re-served memberships included.
// answered with what is served at the moment it is asked — re-served memberships and bundles loaded
// again included, so a bundle served after start is announced as one served at start.
announced, err := announce.Serve(conn, announce.Service{
Name: conn.Module(), ID: instanceOf(conn),
Description: "the mesh's tool runtime on " + node + ": every assigned module's tools and the seats they hold",
Metadata: map[string]string{"node": node},
}, func() []announce.Endpoint { return endpointsOf(conn, own, registrations, modules) })
}, func() []announce.Endpoint { return endpointsOf(conn, r.own(), r.registrations(), r.modules) })
if err != nil {
logf("[mesh-tools] cannot announce what it serves: %v", err)
} else {
stops = append(stops, announced)
r.stops = append(r.stops, announced)
}
// **A bundle is served as soon as it is in place** (issue 1945, note 23): the entrypoints'
// directories are watched, and a module whose bundle appears, changes or goes is loaded again.
var paths []string
for _, s := range served {
for _, entry := range s.Entrypoints {
path, _ := filepath.Abs(entry)
paths = append(paths, path)
}
}
if len(paths) > 0 {
stopWatch, err := watchBundles(paths, r.reconcile, logf)
if err != nil {
logf("[mesh-tools] cannot watch the bundles: %v; a bundle placed or changed later is served only after a restart", err)
} else {
r.stops = append(r.stops, stopWatch)
}
}
conn.Flush()
return stopAll, nil
return r.stop, nil
}
// stop ends the watch, the announcement and the seats, unbinds the modules' consumers, and stops every
// module's tools and bundles.
func (r *runner) stop() {
for i := len(r.stops) - 1; i >= 0; i-- {
r.stops[i]()
}
r.stops = nil
r.events.stopAll()
r.mu.Lock()
for _, stop := range r.verbs {
stop()
}
r.verbs = map[string]func(){}
loaded := make([]*loadedModule, 0, len(r.modules))
for _, m := range r.modules {
if l := r.of[m]; l != nil {
loaded = append(loaded, l)
}
}
r.mu.Unlock()
for i := len(loaded) - 1; i >= 0; i-- {
loaded[i].unload()
}
}
// claimed is every seat a served module's membership names.
func (r *runner) claimed() map[string]bool {
out := map[string]bool{}
for _, m := range r.modules {
if mem := r.conn.Membership(m); mem != nil {
for _, s := range mem.Seats {
out[s.Seat] = true
}
}
}
return out
}
// load launches one module's bundles and learns what they register; it serves nothing yet.
func (r *runner) load(s Served) *loadedModule {
l := &loadedModule{served: s, seen: map[string]string{}}
claimed := r.claimed()
for _, entry := range s.Entrypoints {
path, _ := filepath.Abs(entry)
module := s.Module
l.seen[path] = signatureOf(path)
fail := func(why string) {
l.failed = why
r.logf("[mesh-tools] %s's bundle %s failed to load: %s; its tools are not served here", module, entry, why)
}
if why := notStartable(path); why != "" {
fail(why)
continue
}
child, err := launch.Start(module, path, r.envFor(module), r.events.forModule(module), r.logf)
if err != nil {
fail(err.Error())
continue
}
l.children = append(l.children, child.Stop)
for _, reg := range child.Registrations {
got := registration{module: reg.Module, owner: module, tools: reg.Tools}
l.regs = append(l.regs, got)
switch {
case r.isServed[got.module]:
l.own = append(l.own, got)
case claimed[got.module]:
default:
r.logf(`[mesh-tools] %s registers tools under "%s", which is neither a module served here nor a seat one of them claims; not served until the mesh issues the claim`, got.owner, got.module)
}
}
}
return l
}
// refusal is why a module's registrations cannot be served: a tool named as the verb the runtime
// answers for every module, or two tools of one name.
func refusal(own []registration) error {
for _, r := range own {
seen := map[string]bool{}
for _, t := range r.tools {
if t.Name == ToolsVerb {
return fmt.Errorf(`%s names a tool "%s", which is the verb the runtime answers for every module with what it serves (novox/hq ADR 0152) — refused, rename it`, r.module, ToolsVerb)
}
if seen[t.Name] {
return fmt.Errorf("%s exposes two tools named %s — refused", r.module, t.Name)
}
seen[t.Name] = true
}
}
return nil
}
// serve binds a loaded module's tools. r.mu is held.
func (r *runner) serve(l *loadedModule) {
for _, reg := range l.own {
for _, t := range reg.tools {
t := t
// Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365).
stop, err := r.conn.HandleCalled(reg.module+"."+t.Name, func(caller string, body json.RawMessage) (any, error) {
return t.Run(caller, argsOf(body))
})
if err != nil {
r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err)
continue
}
l.names = append(l.names, reg.module+"."+t.Name)
l.serving = append(l.serving, stop)
}
}
}
// toolsOf is every tool served under a module's name, and why its bundle failed, if it did. r.mu is held.
func (r *runner) toolsOf(module string) ([]launch.Tool, string) {
var tools []launch.Tool
for _, m := range r.modules {
for _, reg := range r.of[m].own {
if reg.module == module {
tools = append(tools, reg.tools...)
}
}
}
return tools, r.of[module].failed
}
// bindVerbs answers each module's `tools` verb where it serves a tool or its bundle failed, and not
// where neither is so (a pure-events module is silent, as it always was). r.mu is held.
func (r *runner) bindVerbs() {
for _, module := range r.modules {
module := module
tools, why := r.toolsOf(module)
want := len(tools) > 0 || why != ""
stop, bound := r.verbs[module]
switch {
case want && !bound:
stop, err := r.conn.Handle(module+"."+ToolsVerb, func(json.RawMessage) (any, error) {
r.mu.Lock()
tools, why := r.toolsOf(module)
r.mu.Unlock()
answer := ToolsAnswer{Module: module, Tools: []ListedTool{}, Failed: why}
for _, t := range tools {
answer.Tools = append(answer.Tools, ListedTool{Name: t.Name, Description: t.Description,
Input: t.Input, Subjects: subjectsOf(r.conn, module, t.Name)})
}
return answer, nil
})
if err == nil {
r.verbs[module] = stop
}
case !want && bound:
stop()
delete(r.verbs, module)
}
}
}
// own is every registration served under a served module's name, now.
func (r *runner) own() []registration {
r.mu.Lock()
defer r.mu.Unlock()
var out []registration
for _, m := range r.modules {
out = append(out, r.of[m].own...)
}
return out
}
// registrations is every registration the bundles made, now.
func (r *runner) registrations() []registration {
r.mu.Lock()
defer r.mu.Unlock()
var out []registration
for _, m := range r.modules {
out = append(out, r.of[m].regs...)
}
return out
}
// reconcile loads again every module whose entrypoint appeared, changed or went since it was loaded.
// The watch calls it once a burst of changes settles; it runs on the watch's one goroutine.
func (r *runner) reconcile() {
for _, m := range r.modules {
r.mu.Lock()
l := r.of[m]
r.mu.Unlock()
for _, entry := range l.served.Entrypoints {
path, _ := filepath.Abs(entry)
was, now := l.seen[path], signatureOf(path)
if was == now {
continue
}
what := "changed"
switch {
case was == "":
what = "appeared"
case now == "":
what = "is gone"
}
r.reload(m, entry+" "+what)
break
}
}
}
// reload stops a module's tools and bundles, loads them again, serves what they register now, and
// serves its seats' verbs again; discovery answers with it from then on.
func (r *runner) reload(module, what string) {
r.mu.Lock()
old := r.of[module]
r.of[module] = &loadedModule{served: old.served, seen: old.seen}
r.mu.Unlock()
old.unload()
r.events.dropModule(module)
r.logf("[mesh-tools] %s's bundle %s; loading it again", module, what)
fresh := r.load(old.served)
if err := refusal(fresh.own); err != nil {
fresh.unload()
fresh.regs, fresh.own, fresh.failed = nil, nil, err.Error()
r.events.dropModule(module)
r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err)
}
r.mu.Lock()
r.of[module] = fresh
r.serve(fresh)
r.bindVerbs()
r.mu.Unlock()
if fresh.failed != "" {
r.logf("[mesh-tools] not serving %s, whose bundle failed to load", module)
} else {
r.logf("[mesh-tools] serving %d tool(s) for %s: %s", len(fresh.names), module, orNone(fresh.names))
}
if r.seatsAgain != nil {
r.seatsAgain()
}
r.conn.Flush()
}
// instanceOf is this runtime's instance on the bus: its machine, which is what tells two instances of
@@ -390,26 +604,31 @@ func subjectsOf(conn *bus.Conn, module, tool string) []string {
// serveSeats serves every verb of every seat a served module holds, where the mesh issued it, by
// the tool of the same name registered under the seat's name (ADR 0159, 0160) — and serves again
// whenever a membership changes. Whether this machine holds the seat is the bus's to decide.
func serveSeats(conn *bus.Conn, modules []string, registrations []registration, logf func(string, ...any)) func() {
impl := map[string]map[string]launch.Tool{}
for _, r := range registrations {
if impl[r.module] == nil {
impl[r.module] = map[string]launch.Tool{}
}
for _, t := range r.tools {
impl[r.module][t.Name] = t
}
}
// whenever a membership changes, and when again is called because a bundle was loaded again. Whether
// this machine holds the seat is the bus's to decide. It answers its stop and again.
func serveSeats(conn *bus.Conn, modules []string, current func() []registration, logf func(string, ...any)) (func(), func()) {
var mu sync.Mutex
var stops []func()
stopped := false
serve := func() {
mu.Lock()
defer mu.Unlock()
if stopped {
return
}
for _, s := range stops {
s()
}
stops = nil
impl := map[string]map[string]launch.Tool{}
for _, r := range current() {
if impl[r.module] == nil {
impl[r.module] = map[string]launch.Tool{}
}
for _, t := range r.tools {
impl[r.module][t.Name] = t
}
}
have := map[string]bool{}
for _, module := range modules {
m := conn.Membership(module)
@@ -447,7 +666,8 @@ func serveSeats(conn *bus.Conn, modules []string, registrations []registration,
s()
}
stops = nil
}
stopped = true
}, serve
}
// consumers holds, per module the runtime serves, the one durable consumer its events arrive on and
@@ -485,6 +705,24 @@ func (c *consumers) stopAll() {
c.provisioned.stopAll()
}
// dropModule unbinds one module's consumer, its seat traffic and its provision watches, so the bundle
// loaded again in its place binds them anew and no event is handed to a child that was stopped.
func (c *consumers) dropModule(module string) {
c.mu.Lock()
defer c.mu.Unlock()
if m := c.of[module]; m != nil && m.stop != nil {
m.stop()
}
delete(c.of, module)
for key, stop := range c.seatBound {
if strings.HasPrefix(key, module+" ") {
stop()
delete(c.seatBound, key)
}
}
c.provisioned.stopModule(module)
}
// forModule is the bus one launched bundle of a module reaches the mesh through.
func (c *consumers) forModule(module string) launch.Bus {
return &moduleBus{all: c, module: module}