Serve a tools bundle as soon as it is in place, changed or gone, with no restart; say why a bundle cannot start (issue 1945)
node-tools loaded every bundle once at start, so a bundle delivered after it started (mesh-files on shanks, 2026-10-25 12:08) went unserved until a hand restart, and a missing file was said as "not executable". The runtime now watches the entrypoints' directories with inotify, loads a module again once a burst of changes settles, and discovery answers with what is served now. A bundle that cannot start says it does not exist, is not executable with its mode, or the exec error. Tests: 1945/AC19-AC22 in internal/runtime/reload_test.go.
This commit is contained in:
@@ -0,0 +1,281 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A runtime serves a module's bundle as soon as it is in place, again when it is replaced, and no
|
||||
// more once it is gone, with no restart (issue 1945, note 23: mesh-files' bundle arrived on a machine
|
||||
// a quarter of an hour after its runtime started, and its tools were missing until a hand restart).
|
||||
|
||||
// syncLogs is a logger that keeps the lines, safe to read while the runtime's watcher writes.
|
||||
type syncLogs struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (l *syncLogs) Logf(format string, args ...any) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.lines = append(l.lines, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *syncLogs) Has(fragments ...string) bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
for _, line := range l.lines {
|
||||
all := true
|
||||
for _, f := range fragments {
|
||||
all = all && strings.Contains(line, f)
|
||||
}
|
||||
if all {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (l *syncLogs) All() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return strings.Join(l.lines, "\n")
|
||||
}
|
||||
|
||||
// bundleServing is a tools bundle, MCP over stdio with no SDK, whose one tool is named tool.
|
||||
func bundleServing(tool string) string {
|
||||
return `#!/usr/bin/env python3
|
||||
import json, sys
|
||||
def say(m):
|
||||
m["jsonrpc"] = "2.0"; sys.stdout.write(json.dumps(m) + "\n"); sys.stdout.flush()
|
||||
TOOLS = [{"name": "` + tool + `", "description": "answers its name", "inputSchema": {"type": "object", "properties": {}}}]
|
||||
for line in sys.stdin:
|
||||
req = json.loads(line); rid = req.get("id"); m = req.get("method")
|
||||
if m == "initialize":
|
||||
say({"id": rid, "result": {"protocolVersion": "2025-03-26", "capabilities": {"tools": {}}, "serverInfo": {"name": "ledger", "version": "1"}}})
|
||||
elif m == "tools/list":
|
||||
say({"id": rid, "result": {"tools": TOOLS}})
|
||||
elif m == "tools/call":
|
||||
say({"id": rid, "result": {"content": [{"type": "text", "text": json.dumps({"tool": "` + tool + `"})}]}})
|
||||
elif rid is not None:
|
||||
say({"id": rid, "result": {}})
|
||||
`
|
||||
}
|
||||
|
||||
// place puts a bundle serving tool at path the way a delivery does: written beside it, made
|
||||
// executable, and renamed into place, its directories made as needed.
|
||||
func place(t *testing.T, path, tool string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmp := path + ".new"
|
||||
if err := os.WriteFile(tmp, []byte(bundleServing(tool)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// fastSettle makes the watcher act on a burst of changes quickly in a test.
|
||||
func fastSettle(t *testing.T) {
|
||||
was := Settle
|
||||
Settle = 50 * time.Millisecond
|
||||
t.Cleanup(func() { Settle = was })
|
||||
}
|
||||
|
||||
type reloadBed struct {
|
||||
nodeTools *bus.Conn
|
||||
asker *bus.Conn
|
||||
logs *syncLogs
|
||||
entry string
|
||||
}
|
||||
|
||||
func newReloadBed(t *testing.T) *reloadBed {
|
||||
t.Helper()
|
||||
fastSettle(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
return &reloadBed{
|
||||
nodeTools: connect(t, "node-tools", "anchor"),
|
||||
asker: connect(t, "console", "workstation"),
|
||||
logs: &syncLogs{},
|
||||
// Below directories that do not exist yet, as on a machine before the module's first delivery.
|
||||
entry: filepath.Join(t.TempDir(), "bundles", "ledger", "tools", "serve.py"),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *reloadBed) run(t *testing.T) {
|
||||
t.Helper()
|
||||
stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
|
||||
func (b *reloadBed) answers(t *testing.T, tool string) {
|
||||
t.Helper()
|
||||
mt.Until(t, func() error {
|
||||
got, err := call(t, b.asker, "ledger."+tool+"@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ledger.%s is not served: %v\n%s", tool, err, b.logs.All())
|
||||
}
|
||||
if !strings.Contains(got, `"tool":"`+tool+`"`) {
|
||||
return fmt.Errorf("ledger.%s answered %s", tool, got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (b *reloadBed) silent(t *testing.T, tool string) {
|
||||
t.Helper()
|
||||
mt.Until(t, func() error {
|
||||
if _, err := call(t, b.asker, "ledger."+tool+"@anchor", map[string]any{}); err == nil {
|
||||
return fmt.Errorf("ledger.%s is still served\n%s", tool, b.logs.All())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func announced(t *testing.T) string {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
msg, err := nc.Request("$SRV.INFO.node-tools.anchor", nil, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var i micro.Info
|
||||
if err := json.Unmarshal(msg.Data, &i); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, e := range i.Endpoints {
|
||||
out = append(out, e.Subject)
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
// 1945/AC19: a bundle placed after the runtime started is served, and announced, with no restart.
|
||||
func TestABundlePlacedAfterStartIsServed(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
b.run(t)
|
||||
if !b.logs.Has("ledger's bundle", "does not exist") {
|
||||
t.Errorf("the missing bundle was not said as missing:\n%s", b.logs.All())
|
||||
}
|
||||
place(t, b.entry, "first")
|
||||
b.answers(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
if a := announced(t); !strings.Contains(a, "mesh.mod.ledger.tool.first.anchor") {
|
||||
return fmt.Errorf("discovery does not announce the placed bundle's tool:\n%s", a)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// 1945/AC20: a bundle replaced while served is loaded again: its new tool is served, its old one no more.
|
||||
func TestAReplacedBundleIsReloaded(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
place(t, b.entry, "second")
|
||||
b.answers(t, "second")
|
||||
b.silent(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
a := announced(t)
|
||||
if !strings.Contains(a, "mesh.mod.ledger.tool.second.anchor") || strings.Contains(a, "mesh.mod.ledger.tool.first.anchor") {
|
||||
return fmt.Errorf("discovery does not follow the replaced bundle:\n%s", a)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// The bundle's whole directory swapped for a new one, as an archive unpacked beside it and renamed in.
|
||||
dir := filepath.Dir(b.entry)
|
||||
fresh := dir + ".unpacked"
|
||||
place(t, filepath.Join(fresh, filepath.Base(b.entry)), "third")
|
||||
if err := os.Rename(dir, dir+".old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(fresh, dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.answers(t, "third")
|
||||
b.silent(t, "second")
|
||||
|
||||
// And changed again inside the directory swapped in: the watch follows the new directory.
|
||||
place(t, b.entry, "fourth")
|
||||
b.answers(t, "fourth")
|
||||
}
|
||||
|
||||
// 1945/AC21: a bundle removed while served is no longer served, and its module says why.
|
||||
func TestARemovedBundleIsNoLongerServed(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
if err := os.RemoveAll(filepath.Dir(b.entry)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.silent(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
got, err := call(t, b.asker, "ledger.tools@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var answer ToolsAnswer
|
||||
if err := json.Unmarshal([]byte(got), &answer); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(answer.Tools) != 0 || !strings.Contains(answer.Failed, "does not exist") {
|
||||
return fmt.Errorf("ledger's tools after its bundle was removed: %s", got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if a := announced(t); strings.Contains(a, "mesh.mod.ledger.tool.first") {
|
||||
t.Errorf("discovery still announces the removed bundle's tool:\n%s", a)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC22: a bundle that cannot be started is said with its real reason: a missing file does not
|
||||
// exist, and a file without an execute bit is not executable with its mode.
|
||||
func TestABundleThatCannotStartSaysTheRealReason(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "nowhere", "serve.py")
|
||||
if why := notStartable(missing); !strings.Contains(why, missing+" does not exist") || strings.Contains(why, "not executable") {
|
||||
t.Errorf("a missing bundle said: %q", why)
|
||||
}
|
||||
plain := filepath.Join(dir, "serve.py")
|
||||
if err := os.WriteFile(plain, []byte(bundleServing("first")), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := notStartable(plain); !strings.Contains(why, plain+" is not executable (mode -rw-r--r--)") {
|
||||
t.Errorf("a bundle without an execute bit said: %q", why)
|
||||
}
|
||||
if why := notStartable(dir); !strings.Contains(why, "is a directory") {
|
||||
t.Errorf("a directory said: %q", why)
|
||||
}
|
||||
if err := os.Chmod(plain, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := notStartable(plain); why != "" {
|
||||
t.Errorf("a startable bundle said: %q", why)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user