Wait for every runtime discovery hears from, and name the ones it misses

The console gathered $SRV.INFO answers for a fixed 750 ms. The laptop's
runtime (48 modules, 341 endpoints, 164 kB) answers last every time: its
answer crosses to the broker on another machine and back, a median of
365 ms on a quiet link and 813 ms in one of 25 rounds measured. When it
missed the window the console said its modules ran nowhere ("nothing in
the mesh is called slack") or only on another machine.

Discovery now asks $SRV.PING alongside $SRV.INFO and waits, past the
window and up to 5 s, for every instance that answered PING. A runtime
that never sends what it serves, or answered before and not now, is
named in every answer that might concern it instead of the module being
called missing. An answer still too large after first-line descriptions
drops them, and says so in its metadata.

mesh_runtimes reports, per runtime, its machine, how long its answer
took, its size, its modules and tools, whether it was shortened and when
it was last heard, and the runtimes and machines not heard.
This commit is contained in:
jochen
2026-10-05 15:36:10 +02:00
parent 8b789578c1
commit 379d19d907
9 changed files with 882 additions and 71 deletions
+118 -13
View File
@@ -2,8 +2,8 @@ package console
// The mesh's tools found by address, not announced whole (novox/hq ADR 0195, to-be 34 §3a).
//
// The console announces five tools. Everything the mesh answers is reached through them by one
// address per layer:
// The console announces six tools — five to find and call, and mesh_runtimes to say which runtimes
// discovery heard. Everything the mesh answers is reached through them by one address per layer:
//
// <seat>.<verb> a seat held once for the mesh — its holder answers
// <node>/<seat>.<verb> a seat held once per machine — that machine's holder answers
@@ -32,13 +32,14 @@ import (
// one agent turn's search, describe and call ask once, short enough that nothing goes stale.
var IndexKept = 5 * time.Second
// The five tools the console announces. Names of the API's kind — letters, digits, `_`, `-`.
// The six tools the console announces. Names of the API's kind — letters, digits, `_`, `-`.
const (
verbOverview = "mesh_overview"
verbMachine = "mesh_machine"
verbSearch = "mesh_search"
verbDescribe = "mesh_describe"
verbCall = "mesh_call"
verbRuntimes = "mesh_runtimes"
)
// searchCap is how many matches a search answers before it says how many more there were.
@@ -49,7 +50,7 @@ const grammar = "Addresses: `<seat>.<verb>` for a seat held once for the mesh (e
"`<node>/<module>.<tool>` for a module on one machine (e.g. `novox/postgres.postgres_list_databases`); " +
"and `<module>.<tool>` also for a module whose instances are interchangeable."
// discovery is the five tools as tools/list announces them.
// discovery is the six tools as tools/list announces them.
func discovery() []map[string]any {
str := func(desc string) map[string]any { return map[string]any{"type": "string", "description": desc} }
obj := func(props map[string]any, required ...string) map[string]any {
@@ -77,12 +78,17 @@ func discovery() []map[string]any {
"arguments": map[string]any{"type": "object", "description": "the tool's arguments, as mesh_describe gives its schema"},
}, "address"),
"description": "Call one tool by its address with its arguments; the answer says which machine gave it. " + grammar},
{"name": verbRuntimes, "inputSchema": obj(map[string]any{}),
"description": "Which runtimes answered discovery, asked now: for each, its machine, how long its answer took to " +
"arrive, its size in bytes, how many modules and tools it announced, whether it was shortened to fit the " +
"bus, and when it was last heard — and every runtime or machine expected that was not heard. Read-only; " +
"for when an address is said to be missing or on another machine."},
}
}
func isDiscovery(name string) bool {
switch name {
case verbOverview, verbMachine, verbSearch, verbDescribe, verbCall:
case verbOverview, verbMachine, verbSearch, verbDescribe, verbCall, verbRuntimes:
return true
}
return false
@@ -117,6 +123,44 @@ type index struct {
Modules map[string]*moduleInfo
NotAnswering []string
Listing *Listing // the flat catalogue the call path resolves subjects with
Discovery announce.Discovery
// Recorded is where the controller's records place each module that declares tools.
Recorded map[string][]string
// Unheard is every runtime whose answer this index lacks: it said it was there and did not say
// what it serves in time, or it answered before and not now. An address it might answer is never
// called missing while it is here (2026-10-05).
Unheard []unheard
}
// unheard is one runtime discovery did not hear in full, and why.
type unheard struct {
Runtime string `json:"runtime"`
Machine string `json:"machine,omitempty"`
Why string `json:"why"`
}
// unheardOn is what is unheard on one machine; with no machine, everything unheard.
func (x *index) unheardOn(node string) []unheard {
var out []unheard
for _, u := range x.Unheard {
if node == "" || u.Machine == node || u.Machine == "" {
out = append(out, u)
}
}
return out
}
// sayUnheard is the unheard, as a sentence's end.
func sayUnheard(us []unheard) string {
var parts []string
for _, u := range us {
who := u.Runtime
if u.Machine != "" {
who += " on " + u.Machine
}
parts = append(parts, who+" ("+u.Why+")")
}
return strings.Join(parts, "; ")
}
func (x *index) seat(name string) *seatInfo {
@@ -193,20 +237,24 @@ func indexOn(conn *bus.Conn) (*index, error) {
wg.Add(2)
go func() { defer wg.Done(); nodesOut, _ = controllerOutput(conn, "nodes") }()
go func() { defer wg.Done(); modulesOut, _ = controllerOutput(conn, "modules") }()
infos, err := announce.Gather(conn)
d, err := announce.Gather(conn)
wg.Wait()
if err != nil {
return nil, err
}
l := &Listing{Tools: []Tool{}, NotAnswering: []string{}}
x := &index{Modules: map[string]*moduleInfo{}, Listing: l}
x := &index{Modules: map[string]*moduleInfo{}, Listing: l, Discovery: d, Recorded: map[string][]string{}}
for _, s := range d.Silent {
x.Unheard = append(x.Unheard, unheard{Runtime: s.Name, Machine: s.Machine,
Why: fmt.Sprintf("it answered PING; what it serves did not arrive within %s", announce.Patience)})
}
announced := map[string]map[string]bool{} // module → node → announced something
seats := map[string]*seatInfo{}
toolAt := map[string]int{} // <module>.<tool> or <seat>.<verb> → index in l.Tools
machines := map[string]bool{}
for _, info := range infos {
for _, e := range announce.Endpoints(info) {
for _, h := range d.Heard {
for _, e := range announce.Endpoints(h.Info) {
if e.Node != "" {
machines[e.Node] = true
}
@@ -298,6 +346,7 @@ func indexOn(conn *bus.Conn) (*index, error) {
if !m.Tools {
continue
}
x.Recorded[m.Module] = append([]string{}, m.On...)
for _, n := range m.On {
// A holder of a seat held once for the mesh announces no machine — the seat is the mesh's,
// not a machine's — so what it announced without one answers for wherever it is assigned.
@@ -309,6 +358,9 @@ func indexOn(conn *bus.Conn) (*index, error) {
} else {
l.NotAnswering = append(l.NotAnswering, "mesh-controller (its records of the modules did not answer, so what is missing cannot be said)")
}
for _, u := range x.Unheard {
l.NotAnswering = append(l.NotAnswering, "the runtime "+sayUnheard([]unheard{u}))
}
sort.Strings(l.NotAnswering)
x.NotAnswering = l.NotAnswering
@@ -336,9 +388,12 @@ func containsHolder(hs []holder, h holder) bool {
return false
}
func (s *Surface) index() (*index, error) {
func (s *Surface) index() (*index, error) { return s.indexAsked(false) }
// indexAsked is what the mesh answered, asked again when `fresh` or when the kept answer is old.
func (s *Surface) indexAsked(fresh bool) (*index, error) {
s.mu.Lock()
if s.idx != nil && time.Since(s.idxAt) <= IndexKept {
if !fresh && s.idx != nil && time.Since(s.idxAt) <= IndexKept {
x := s.idx
s.mu.Unlock()
return x, nil
@@ -349,6 +404,7 @@ func (s *Surface) index() (*index, error) {
return nil, err
}
s.mu.Lock()
s.remember(x)
s.idx, s.idxAt = x, time.Now()
s.mu.Unlock()
return x, nil
@@ -397,6 +453,18 @@ func resolve(x *index, address string) (target, error) {
m := x.Modules[prefix]
if m == nil {
if on := x.Recorded[prefix]; len(on) > 0 {
why := "it did not announce itself on the bus"
if us := x.unheardOnAny(on); len(us) > 0 {
why = "discovery did not hear in full from " + sayUnheard(us)
}
return target{}, fmt.Errorf("%s is assigned to %s in the mesh's records, but nothing that answered discovery serves it: %s. "+
"Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, strings.Join(on, ", "), why)
}
if len(x.Unheard) > 0 {
return target{}, fmt.Errorf("nothing that answered discovery is called %s, but not every runtime answered: %s — "+
"%s may be theirs. Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, sayUnheard(x.Unheard), prefix)
}
return target{}, fmt.Errorf("nothing in the mesh is called %s: no seat, and no module that answers tools. "+
"mesh_search finds a tool by words", prefix)
}
@@ -412,11 +480,30 @@ func resolve(x *index, address string) (target, error) {
return target{Address: rest, Key: rest, Name: rest, Tool: *tool}, nil
}
if len(m.On) > 0 && !contains(m.On, node) {
if us := x.unheardOn(node); len(us) > 0 || contains(x.Recorded[prefix], node) {
why := "it did not announce itself there"
if len(us) > 0 {
why = "discovery did not hear in full from " + sayUnheard(us)
}
return target{}, fmt.Errorf("%s on %s did not answer discovery: %s. It was heard on %s. "+
"Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, node, why, orNobody(m.On))
}
return target{}, fmt.Errorf("%s does not run on %s; it runs on %s", prefix, node, orNobody(m.On))
}
return target{Address: node + "/" + rest, Key: rest + "@" + node, Name: rest, Node: node, Tool: *tool}, nil
}
// unheardOnAny is what is unheard on any of these machines.
func (x *index) unheardOnAny(nodes []string) []unheard {
var out []unheard
for _, u := range x.Unheard {
if u.Machine == "" || contains(nodes, u.Machine) {
out = append(out, u)
}
}
return out
}
func contains(xs []string, s string) bool {
for _, x := range xs {
if x == s {
@@ -511,8 +598,11 @@ func failure(text string) map[string]any {
return map[string]any{"content": []map[string]any{{"type": "text", "text": text}}, "isError": true}
}
// discover answers one of the five.
// discover answers one of the discovery verbs.
func (s *Surface) discover(name string, args map[string]any) map[string]any {
if name == verbRuntimes {
return s.runtimesAnswer()
}
x, err := s.index()
if err != nil {
return failure("the mesh's discovery failed: " + err.Error())
@@ -606,7 +696,18 @@ func (s *Surface) discover(name string, args map[string]any) map[string]any {
}
modules = append(modules, entry)
}
return answerText(map[string]any{"machine": node, "seats": seats, "modules": modules})
out := map[string]any{"machine": node, "seats": seats, "modules": modules}
if seats == nil {
out["seats"] = []map[string]any{}
}
if modules == nil {
out["modules"] = []map[string]any{}
}
if us := x.unheardOn(node); len(us) > 0 {
out["incomplete"] = "discovery did not hear in full from " + sayUnheard(us) +
"; what is listed may be missing what it serves. Ask again in a moment; mesh_runtimes shows which runtimes answered"
}
return answerText(out)
case verbSearch:
query := strings.ToLower(str("query"))
@@ -684,6 +785,10 @@ func (s *Surface) discover(name string, args map[string]any) map[string]any {
out["matches"] = []hit{}
out["hint"] = "nothing matched every word; try fewer words, or mesh_overview and mesh_machine to browse"
}
if len(x.Unheard) > 0 {
out["incomplete"] = "discovery did not hear in full from " + sayUnheard(x.Unheard) +
"; what they serve is not searched. mesh_runtimes shows which runtimes answered"
}
return answerText(out)
case verbDescribe:
+3 -3
View File
@@ -36,7 +36,7 @@ func call(t *testing.T, endpoint, tool string, args map[string]any) (string, boo
return text(t, post(t, endpoint, string(body)))
}
// novox/hq ADR 0195: the console announces five tools, and everything the mesh answers is reached
// novox/hq ADR 0195: the console announces six tools, and everything the mesh answers is reached
// through them by one address per layer.
func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
was := IndexKept
@@ -113,7 +113,7 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
defer up.Close()
endpoint := "http://" + up.Address + "/mcp"
// Five tools, nothing else.
// Six tools, nothing else.
listed := post(t, endpoint, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`)["result"].(map[string]any)
var names []string
for _, x := range listed["tools"].([]any) {
@@ -125,7 +125,7 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
}
}
}
if got := strings.Join(names, ","); got != "mesh_overview,mesh_machine,mesh_search,mesh_describe,mesh_call" {
if got := strings.Join(names, ","); got != "mesh_overview,mesh_machine,mesh_search,mesh_describe,mesh_call,mesh_runtimes" {
t.Errorf("announced %s", got)
}
+4 -1
View File
@@ -50,6 +50,8 @@ type Surface struct {
at time.Time
idx *index
idxAt time.Time
// runtimes is every runtime discovery has heard, as it last heard it (mesh_runtimes).
runtimes map[string]*runtimeSeen
// Flat announces the whole catalogue, as the console did before ADR 0195: for a person reading
// it or a client that wants it. Off by default; MESH_CONSOLE_FLAT=1 turns it on.
Flat bool
@@ -300,7 +302,8 @@ func (s *Surface) instructions() string {
return "The tools of a Novox mesh, reached as " + s.who + ", found by address rather than listed " +
"whole (novox/hq ADR 0195). mesh_overview shows the mesh's seats and machines; mesh_machine one " +
"machine's seats and modules; mesh_search finds a tool by words; mesh_describe gives one tool's " +
"arguments; mesh_call calls it. " + grammar + " What may be called was fixed when this account " +
"arguments; mesh_call calls it; mesh_runtimes says which machines' runtimes answered discovery, how fast " +
"and how much — ask it when an address is said to be missing or on another machine. " + grammar + " What may be called was fixed when this account " +
"was issued, so a refusal means the account, not the tool."
}
+189
View File
@@ -0,0 +1,189 @@
package console
// What discovery heard from each runtime (2026-10-05): the console said a module ran nowhere when the
// largest runtime's answer arrived after the window closed, and nothing showed that it had. Every
// round of discovery is remembered per runtime, so a runtime heard before and not now is named, and
// `mesh_runtimes` says who answered, how fast and how much.
import (
"fmt"
"sort"
"time"
"github.com/novox/mesh-tools/node-tools/internal/announce"
)
// RuntimeForgotten is how long a runtime heard once is expected again: within it, its silence is said
// wherever an address it might answer is asked for; after it, it is listed only by mesh_runtimes.
var RuntimeForgotten = 15 * time.Minute
// runtimeSeen is one runtime instance as discovery last heard it.
type runtimeSeen struct {
Name string
ID string
Machine string
LastHeard time.Time
Took time.Duration
Bytes int
Modules int
Tools int
SeatVerbs int
Shortened string
Answered bool // in the latest round
Why string // why not, when it did not
}
func runtimeKey(name, id string) string { return name + "/" + id }
// seenOf is what one answer announced, counted.
func seenOf(h announce.Heard, at time.Time) *runtimeSeen {
modules := map[string]bool{}
tools := map[string]bool{}
verbs := map[string]bool{}
for _, e := range announce.Endpoints(h.Info) {
switch e.Kind {
case announce.KindTool:
modules[e.Module] = true
tools[e.Module+"."+e.Tool] = true
case announce.KindSeat:
verbs[e.Seat+"."+e.Tool] = true
}
}
return &runtimeSeen{Name: h.Info.Name, ID: h.Info.ID, Machine: h.Machine(), LastHeard: at, Took: h.Took,
Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true}
}
// remember records a round of discovery and adds to the index every runtime heard lately that did not
// answer this time. Called with s.mu held.
func (s *Surface) remember(x *index) {
if s.runtimes == nil {
s.runtimes = map[string]*runtimeSeen{}
}
at := x.Discovery.At
if at.IsZero() {
at = time.Now().UTC()
}
heard := map[string]bool{}
for _, h := range x.Discovery.Heard {
k := runtimeKey(h.Info.Name, h.Info.ID)
seen := seenOf(h, at)
if heard[k] {
// Two instances under one name and id: both counted, the slower answer's time kept.
prev := s.runtimes[k]
seen.Modules += prev.Modules
seen.Tools += prev.Tools
seen.SeatVerbs += prev.SeatVerbs
seen.Bytes += prev.Bytes
if prev.Took > seen.Took {
seen.Took = prev.Took
}
}
heard[k] = true
s.runtimes[k] = seen
}
silent := map[string]string{}
for _, u := range x.Discovery.Silent {
silent[runtimeKey(u.Name, u.ID)] = fmt.Sprintf("it answered PING; what it serves did not arrive within %s", announce.Patience)
}
for k, r := range s.runtimes {
if heard[k] {
continue
}
r.Answered = false
if why, ok := silent[k]; ok {
r.Why = why
continue // already in the index's unheard
}
r.Why = "it did not answer this round of discovery"
if time.Since(r.LastHeard) <= RuntimeForgotten {
x.Unheard = append(x.Unheard, unheard{Runtime: r.Name, Machine: r.Machine,
Why: fmt.Sprintf("it answered discovery %s ago, not now", time.Since(r.LastHeard).Round(time.Second))})
}
}
for _, u := range x.Discovery.Silent {
k := runtimeKey(u.Name, u.ID)
if _, known := s.runtimes[k]; !known {
s.runtimes[k] = &runtimeSeen{Name: u.Name, ID: u.ID, Machine: u.Machine, Why: silent[k]}
}
}
}
// runtimesAnswer is mesh_runtimes: discovery asked now, and every runtime as it was heard.
func (s *Surface) runtimesAnswer() map[string]any {
x, err := s.indexAsked(true)
if err != nil {
return failure("the mesh's discovery failed: " + err.Error())
}
s.mu.Lock()
var all []runtimeSeen
for _, r := range s.runtimes {
all = append(all, *r)
}
s.mu.Unlock()
sort.Slice(all, func(i, j int) bool {
if all[i].Machine != all[j].Machine {
return all[i].Machine < all[j].Machine
}
return runtimeKey(all[i].Name, all[i].ID) < runtimeKey(all[j].Name, all[j].ID)
})
now := time.Now()
var answered, notHeard []map[string]any
onMachine := map[string]bool{}
for _, r := range all {
entry := map[string]any{"runtime": r.Name, "instance": r.ID}
if r.Machine != "" {
entry["machine"] = r.Machine
}
if !r.LastHeard.IsZero() {
entry["last heard"] = r.LastHeard.Format(time.RFC3339)
entry["last heard, ago"] = now.Sub(r.LastHeard).Round(time.Second).String()
entry["took"] = r.Took.Round(time.Millisecond).String()
entry["bytes"] = r.Bytes
entry["modules"] = r.Modules
entry["tools"] = r.Tools
entry["seat verbs"] = r.SeatVerbs
entry["shortened"] = r.Shortened != ""
if r.Shortened != "" {
entry["shortened, how"] = r.Shortened
}
}
if r.Answered {
onMachine[r.Machine] = true
answered = append(answered, entry)
} else {
entry["why"] = r.Why
if r.LastHeard.IsZero() {
entry["last heard"] = "never in full, since this console started"
}
notHeard = append(notHeard, entry)
}
}
for _, m := range x.Machines {
if !onMachine[m] && !machineListed(notHeard, m) {
notHeard = append(notHeard, map[string]any{"machine": m,
"why": "the mesh knows this machine and no runtime on it has answered discovery since this console started"})
}
}
if answered == nil {
answered = []map[string]any{}
}
if notHeard == nil {
notHeard = []map[string]any{}
}
return answerText(map[string]any{
"asked": x.Discovery.At.Format(time.RFC3339),
"waited": x.Discovery.Waited.Round(time.Millisecond).String(),
"window": fmt.Sprintf("at least %s; up to %s for a runtime that answered PING", announce.Window, announce.Patience),
"answered": answered,
"not heard": notHeard,
})
}
func machineListed(entries []map[string]any, machine string) bool {
for _, e := range entries {
if e["machine"] == machine {
return true
}
}
return false
}
@@ -0,0 +1,200 @@
package console
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/micro"
"github.com/novox/mesh-tools/node-tools/internal/announce"
"github.com/novox/mesh-tools/node-tools/internal/bus"
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
)
// slowRuntime answers PING at once and INFO after `lag`, or never with a negative lag.
func slowRuntime(t *testing.T, conn *bus.Conn, node string, lag time.Duration, endpoints []announce.Endpoint) {
t.Helper()
id := micro.ServiceIdentity{Name: "node-tools", ID: node, Version: announce.Version, Metadata: map[string]string{"node": node}}
ping, _ := json.Marshal(micro.Ping{ServiceIdentity: id, Type: micro.PingResponseType})
stop, err := conn.Raw("$SRV.PING", func(string, []byte) []byte { return ping })
if err != nil {
t.Fatal(err)
}
t.Cleanup(stop)
stop, err = conn.Raw("$SRV.INFO", func(string, []byte) []byte {
if lag < 0 {
return nil
}
time.Sleep(lag)
body, _ := json.Marshal(announce.Info(announce.Service{Name: id.Name, ID: id.ID, Metadata: id.Metadata}, endpoints))
return body
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(stop)
conn.Flush()
}
// The laptop's symptom (2026-10-05), reproduced: its runtime's one large answer reached the console
// after the 750 ms window, and the console said its modules ran nowhere — "nothing in the mesh is
// called slack" — or ran only on another machine. Now a runtime that said it is there is waited for,
// one that never says what it serves is named wherever an address it might answer is asked for, and
// mesh_runtimes shows who answered, how fast and how much.
func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T) {
keptWas, patienceWas := IndexKept, announce.Patience
// Kept for the test: one round of discovery answers every question but mesh_runtimes, which asks anew.
IndexKept, announce.Patience = time.Minute, 2500*time.Millisecond
t.Cleanup(func() { IndexKept, announce.Patience = keptWas, patienceWas })
mt.New(t)
laptop := connect(t, "node-tools", "laptop")
stopTool, err := laptop.HandleSubject("mesh.mod.slack.tool.slack_check.laptop", func(json.RawMessage) (any, error) {
return map[string]any{"ok": true}, nil
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(stopTool)
slowRuntime(t, laptop, "laptop", announce.Window+400*time.Millisecond, []announce.Endpoint{
{Kind: announce.KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Description: "Is Slack started once?",
Subject: "mesh.mod.slack.tool.slack_check.laptop"},
{Kind: announce.KindTool, Module: "nextcloud-client", Tool: "nextcloud_check", Node: "laptop",
Subject: "mesh.mod.nextcloud-client.tool.nextcloud_check.laptop"},
})
sleeper := connect(t, "node-tools", "sleeper")
slowRuntime(t, sleeper, "sleeper", -1, nil)
desktop := connect(t, "node-tools", "desktop")
slowRuntime(t, desktop, "desktop", 0, []announce.Endpoint{
{Kind: announce.KindTool, Module: "nextcloud-client", Tool: "nextcloud_check", Node: "desktop",
Subject: "mesh.mod.nextcloud-client.tool.nextcloud_check.desktop"},
})
controller := connect(t, "mesh-controller", "bench")
for verb, output := range map[string]string{
"nodes": `[{"name":"laptop"},{"name":"sleeper"},{"name":"desktop"},{"name":"bench"}]`,
"modules": `[{"module":"slack","on":["laptop"],"tools":true},{"module":"nap","on":["sleeper"],"tools":true},` +
`{"module":"nextcloud-client","on":["laptop","desktop","sleeper"],"tools":true}]`,
} {
output := output
stop, err := controller.HandleSubject("mesh.seat.mesh-controller.tool."+verb, func(json.RawMessage) (any, error) {
return map[string]any{"output": output, "ok": true}, nil
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(stop)
}
controller.Flush()
console := connect(t, "node-tools", "laptop-console")
up, err := Serve(NewSurface(console, "laptop.node-tools"), "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer up.Close()
endpoint := "http://" + up.Address + "/mcp"
// The late answer is waited for: the laptop's module is found and called.
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/slack.slack_check"}); isErr || !strings.Contains(got, `"ok": true`) {
t.Errorf("a late runtime's module: %s", got)
}
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/nextcloud-client.nextcloud_check"}); isErr && strings.Contains(got, "does not run on laptop") {
t.Errorf("a late runtime's module was said to run elsewhere: %s", got)
}
machine, _ := call(t, endpoint, "mesh_machine", map[string]any{"node": "laptop"})
if !strings.Contains(machine, "laptop/slack.slack_check") {
t.Errorf("the late runtime's machine lists nothing: %s", machine)
}
// The silent one is named, never called missing.
for address, want := range map[string]string{
"sleeper/nap.doze": "nap is assigned to sleeper in the mesh's records, but nothing that answered discovery serves it: discovery did not hear in full from node-tools on sleeper (it answered PING",
"sleeper/nextcloud-client.nextcloud_check": "nextcloud-client on sleeper did not answer discovery: discovery did not hear in full from node-tools on sleeper",
} {
got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": address})
if !isErr || !strings.Contains(got, want) || strings.Contains(got, "nothing in the mesh is called") || strings.Contains(got, "does not run on") {
t.Errorf("%s: %s\nwant %q", address, got, want)
}
}
machine, _ = call(t, endpoint, "mesh_machine", map[string]any{"node": "sleeper"})
if !strings.Contains(machine, `"incomplete"`) || !strings.Contains(machine, `"modules": []`) {
t.Errorf("a machine whose runtime did not answer: %s", machine)
}
overview, _ := call(t, endpoint, "mesh_overview", nil)
if !strings.Contains(overview, "the runtime node-tools on sleeper") {
t.Errorf("overview does not name the silent runtime: %s", overview)
}
// mesh_runtimes: who answered, how fast, how much; who did not.
got, isErr := call(t, endpoint, "mesh_runtimes", nil)
if isErr {
t.Fatal(got)
}
var report struct {
Answered []map[string]any `json:"answered"`
NotHeard []map[string]any `json:"not heard"`
}
if err := json.Unmarshal([]byte(got), &report); err != nil {
t.Fatalf("%v: %s", err, got)
}
var lap map[string]any
for _, r := range report.Answered {
if r["machine"] == "laptop" {
lap = r
}
}
if lap == nil {
t.Fatalf("the laptop's runtime is not listed as answering: %s", got)
}
took, _ := time.ParseDuration(lap["took"].(string))
if took < announce.Window || lap["bytes"].(float64) <= 0 || lap["modules"].(float64) != 2 || lap["tools"].(float64) != 2 ||
lap["shortened"] != false || lap["last heard"] == nil {
t.Errorf("the laptop's runtime: %v", lap)
}
if !strings.Contains(got, `"machine": "sleeper"`) || !strings.Contains(got, "answered PING") {
t.Errorf("the silent runtime is not listed: %s", got)
}
silent := false
for _, r := range report.NotHeard {
silent = silent || r["machine"] == "sleeper"
}
if !silent {
t.Errorf("sleeper is not among the not heard: %s", got)
}
}
// A runtime heard once and silent now is named in the answers about its machine.
func TestARuntimeHeardBeforeAndNotNowIsNamed(t *testing.T) {
s := &Surface{}
before := &index{Modules: map[string]*moduleInfo{}, Recorded: map[string][]string{}, Discovery: announce.Discovery{At: time.Now().UTC(),
Heard: []announce.Heard{{Info: micro.Info{ServiceIdentity: micro.ServiceIdentity{Name: "node-tools", ID: "laptop",
Metadata: map[string]string{"node": "laptop"}}}, Took: 300 * time.Millisecond, Bytes: 1000}}}}
s.remember(before)
now := &index{Modules: map[string]*moduleInfo{"nextcloud-client": {Module: "nextcloud-client", On: []string{"desktop"},
Tools: []Tool{{Module: "nextcloud-client", Name: "nextcloud_check"}}}}, Recorded: map[string][]string{},
Discovery: announce.Discovery{At: time.Now().UTC()}}
s.remember(now)
if len(now.Unheard) != 1 || now.Unheard[0].Machine != "laptop" {
t.Fatalf("unheard: %+v", now.Unheard)
}
_, err := resolve(now, "laptop/nextcloud-client.nextcloud_check")
if err == nil || !strings.Contains(err.Error(), "nextcloud-client on laptop did not answer discovery") ||
!strings.Contains(err.Error(), "answered discovery") {
t.Errorf("%v", err)
}
_, err = resolve(now, "laptop/slack.slack_check")
if err == nil || strings.Contains(err.Error(), "nothing in the mesh is called") || !strings.Contains(err.Error(), "slack may be theirs") {
t.Errorf("%v", err)
}
// Nobody unheard: the plain answers stand.
quiet := &index{Modules: now.Modules, Recorded: map[string][]string{}}
if _, err := resolve(quiet, "laptop/nextcloud-client.nextcloud_check"); err == nil || !strings.Contains(err.Error(), "does not run on laptop; it runs on desktop") {
t.Errorf("%v", err)
}
if _, err := resolve(quiet, "laptop/slack.slack_check"); err == nil || !strings.Contains(err.Error(), "nothing in the mesh is called slack") {
t.Errorf("%v", err)
}
}