220 lines
9.7 KiB
Go
220 lines
9.7 KiB
Go
package console
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go/micro"
|
|
|
|
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
|
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
|
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
|
)
|
|
|
|
// slowRuntime answers PING at once and INFO after `lag`, or never with a negative lag.
|
|
func slowRuntime(t *testing.T, conn *bus.Conn, node string, lag time.Duration, endpoints []announce.Endpoint) {
|
|
t.Helper()
|
|
id := micro.ServiceIdentity{Name: "node-tools", ID: node, Version: announce.Version, Metadata: map[string]string{"node": node}}
|
|
ping, _ := json.Marshal(micro.Ping{ServiceIdentity: id, Type: micro.PingResponseType})
|
|
stop, err := conn.Raw("$SRV.PING", func(string, []byte) []byte { return ping })
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(stop)
|
|
stop, err = conn.Raw("$SRV.INFO", func(string, []byte) []byte {
|
|
if lag < 0 {
|
|
return nil
|
|
}
|
|
time.Sleep(lag)
|
|
body, _ := json.Marshal(announce.Info(announce.Service{Name: id.Name, ID: id.ID, Metadata: id.Metadata}, endpoints))
|
|
return body
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(stop)
|
|
conn.Flush()
|
|
}
|
|
|
|
// The laptop's symptom (2026-10-05), reproduced: its runtime's one large answer reached the console
|
|
// after the 750 ms window, and the console said its modules ran nowhere — "nothing in the mesh is
|
|
// called slack" — or ran only on another machine. Now a runtime that said it is there is waited for,
|
|
// one that never says what it serves is named wherever an address it might answer is asked for, and
|
|
// mesh_runtimes shows who answered, how fast and how much.
|
|
func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T) {
|
|
keptWas, patienceWas := IndexKept, announce.Patience
|
|
// Kept for the test: one round of discovery answers every question but mesh_runtimes, which asks anew.
|
|
IndexKept, announce.Patience = time.Minute, 2500*time.Millisecond
|
|
t.Cleanup(func() { IndexKept, announce.Patience = keptWas, patienceWas })
|
|
mt.New(t)
|
|
|
|
laptop := connect(t, "node-tools", "laptop")
|
|
stopTool, err := laptop.HandleSubject("mesh.mod.slack.tool.slack_check.laptop", func(json.RawMessage) (any, error) {
|
|
return map[string]any{"ok": true}, nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(stopTool)
|
|
slowRuntime(t, laptop, "laptop", announce.Window+400*time.Millisecond, []announce.Endpoint{
|
|
{Kind: announce.KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Description: "Is Slack started once?",
|
|
Subject: "mesh.mod.slack.tool.slack_check.laptop"},
|
|
{Kind: announce.KindTool, Module: "nextcloud-client", Tool: "nextcloud_check", Node: "laptop",
|
|
Subject: "mesh.mod.nextcloud-client.tool.nextcloud_check.laptop"},
|
|
})
|
|
sleeper := connect(t, "node-tools", "sleeper")
|
|
slowRuntime(t, sleeper, "sleeper", -1, nil)
|
|
desktop := connect(t, "node-tools", "desktop")
|
|
slowRuntime(t, desktop, "desktop", 0, []announce.Endpoint{
|
|
{Kind: announce.KindTool, Module: "nextcloud-client", Tool: "nextcloud_check", Node: "desktop",
|
|
Subject: "mesh.mod.nextcloud-client.tool.nextcloud_check.desktop"},
|
|
})
|
|
|
|
controller := connect(t, "mesh-controller", "bench")
|
|
for verb, output := range map[string]string{
|
|
"nodes": `[{"name":"laptop"},{"name":"sleeper"},{"name":"desktop"},{"name":"bench"}]`,
|
|
"modules": `[{"module":"slack","on":["laptop"],"tools":true},{"module":"nap","on":["sleeper"],"tools":true},` +
|
|
`{"module":"nextcloud-client","on":["laptop","desktop","sleeper"],"tools":true}]`,
|
|
} {
|
|
output := output
|
|
stop, err := controller.HandleSubject("mesh.seat.mesh-controller.tool."+verb, func(json.RawMessage) (any, error) {
|
|
return map[string]any{"output": output, "ok": true}, nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(stop)
|
|
}
|
|
controller.Flush()
|
|
|
|
console := connect(t, "node-tools", "laptop-console")
|
|
up, err := Serve(NewSurface(console, "laptop.node-tools"), "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer up.Close()
|
|
endpoint := "http://" + up.Address + "/mcp"
|
|
|
|
// The late answer is waited for: the laptop's module is found and called.
|
|
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/slack.slack_check"}); isErr || !strings.Contains(got, `"ok": true`) {
|
|
t.Errorf("a late runtime's module: %s", got)
|
|
}
|
|
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/nextcloud-client.nextcloud_check"}); isErr && strings.Contains(got, "does not run on laptop") {
|
|
t.Errorf("a late runtime's module was said to run elsewhere: %s", got)
|
|
}
|
|
machine, _ := call(t, endpoint, "mesh_machine", map[string]any{"node": "laptop"})
|
|
if !strings.Contains(machine, "laptop/slack.slack_check") {
|
|
t.Errorf("the late runtime's machine lists nothing: %s", machine)
|
|
}
|
|
|
|
// The silent one is named, never called missing.
|
|
for address, want := range map[string]string{
|
|
"sleeper/nap.doze": "nap is assigned to sleeper in the mesh's records, but nothing that answered discovery serves it: discovery did not hear in full from node-tools on sleeper (it answered PING",
|
|
"sleeper/nextcloud-client.nextcloud_check": "nextcloud-client on sleeper did not answer discovery: discovery did not hear in full from node-tools on sleeper",
|
|
} {
|
|
got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": address})
|
|
if !isErr || !strings.Contains(got, want) || strings.Contains(got, "nothing in the mesh is called") || strings.Contains(got, "does not run on") {
|
|
t.Errorf("%s: %s\nwant %q", address, got, want)
|
|
}
|
|
}
|
|
machine, _ = call(t, endpoint, "mesh_machine", map[string]any{"node": "sleeper"})
|
|
if !strings.Contains(machine, `"incomplete"`) || !strings.Contains(machine, `"modules": []`) {
|
|
t.Errorf("a machine whose runtime did not answer: %s", machine)
|
|
}
|
|
overview, _ := call(t, endpoint, "mesh_overview", nil)
|
|
if !strings.Contains(overview, "the runtime node-tools on sleeper") {
|
|
t.Errorf("overview does not name the silent runtime: %s", overview)
|
|
}
|
|
|
|
// mesh_runtimes: who answered, how fast, how much; who did not.
|
|
got, isErr := call(t, endpoint, "mesh_runtimes", nil)
|
|
if isErr {
|
|
t.Fatal(got)
|
|
}
|
|
var report struct {
|
|
Answered []map[string]any `json:"answered"`
|
|
NotHeard []map[string]any `json:"not heard"`
|
|
}
|
|
if err := json.Unmarshal([]byte(got), &report); err != nil {
|
|
t.Fatalf("%v: %s", err, got)
|
|
}
|
|
var lap map[string]any
|
|
for _, r := range report.Answered {
|
|
if r["machine"] == "laptop" {
|
|
lap = r
|
|
}
|
|
}
|
|
if lap == nil {
|
|
t.Fatalf("the laptop's runtime is not listed as answering: %s", got)
|
|
}
|
|
took, _ := time.ParseDuration(lap["took"].(string))
|
|
if took < announce.Window || lap["bytes"].(float64) <= 0 || lap["modules"].(float64) != 2 || lap["tools"].(float64) != 2 ||
|
|
lap["shortened"] != false || lap["last heard"] == nil {
|
|
t.Errorf("the laptop's runtime: %v", lap)
|
|
}
|
|
if !strings.Contains(got, `"machine": "sleeper"`) || !strings.Contains(got, "answered PING") {
|
|
t.Errorf("the silent runtime is not listed: %s", got)
|
|
}
|
|
silent := false
|
|
for _, r := range report.NotHeard {
|
|
silent = silent || r["machine"] == "sleeper"
|
|
}
|
|
if !silent {
|
|
t.Errorf("sleeper is not among the not heard: %s", got)
|
|
}
|
|
}
|
|
|
|
// A runtime heard once and silent now is named in the answers about its machine.
|
|
func TestARuntimeHeardBeforeAndNotNowIsNamed(t *testing.T) {
|
|
s := &Surface{}
|
|
before := &index{Modules: map[string]*moduleInfo{}, Recorded: map[string][]string{}, Discovery: announce.Discovery{At: time.Now().UTC(),
|
|
Heard: []announce.Heard{{Info: micro.Info{ServiceIdentity: micro.ServiceIdentity{Name: "node-tools", ID: "laptop",
|
|
Metadata: map[string]string{"node": "laptop"}}}, Took: 300 * time.Millisecond, Bytes: 1000}}}}
|
|
s.remember(before)
|
|
now := &index{Modules: map[string]*moduleInfo{"nextcloud-client": {Module: "nextcloud-client", On: []string{"desktop"},
|
|
Tools: []Tool{{Module: "nextcloud-client", Name: "nextcloud_check"}}}}, Recorded: map[string][]string{},
|
|
Discovery: announce.Discovery{At: time.Now().UTC()}}
|
|
s.remember(now)
|
|
if len(now.Unheard) != 1 || now.Unheard[0].Machine != "laptop" {
|
|
t.Fatalf("unheard: %+v", now.Unheard)
|
|
}
|
|
_, err := resolve(now, "laptop/nextcloud-client.nextcloud_check")
|
|
if err == nil || !strings.Contains(err.Error(), "nextcloud-client on laptop did not answer discovery") ||
|
|
!strings.Contains(err.Error(), "answered discovery") {
|
|
t.Errorf("%v", err)
|
|
}
|
|
_, err = resolve(now, "laptop/slack.slack_check")
|
|
if err == nil || strings.Contains(err.Error(), "nothing in the mesh is called") || !strings.Contains(err.Error(), "slack may be theirs") {
|
|
t.Errorf("%v", err)
|
|
}
|
|
// Nobody unheard: the plain answers stand.
|
|
quiet := &index{Modules: now.Modules, Recorded: map[string][]string{}}
|
|
if _, err := resolve(quiet, "laptop/nextcloud-client.nextcloud_check"); err == nil || !strings.Contains(err.Error(), "does not run on laptop; it runs on desktop") {
|
|
t.Errorf("%v", err)
|
|
}
|
|
if _, err := resolve(quiet, "laptop/slack.slack_check"); err == nil || !strings.Contains(err.Error(), "nothing in the mesh is called slack") {
|
|
t.Errorf("%v", err)
|
|
}
|
|
}
|
|
|
|
// A runtime restarted answers under a new instance id; its old instance is gone, not unheard.
|
|
func TestARestartedRuntimeIsNotNamedUnheard(t *testing.T) {
|
|
heard := func(id string) *index {
|
|
return &index{Modules: map[string]*moduleInfo{}, Recorded: map[string][]string{}, Discovery: announce.Discovery{At: time.Now().UTC(),
|
|
Heard: []announce.Heard{{Info: micro.Info{ServiceIdentity: micro.ServiceIdentity{Name: "node-tools", ID: id,
|
|
Metadata: map[string]string{"node": "laptop"}}}}}}}
|
|
}
|
|
s := &Surface{}
|
|
s.remember(heard("before-restart"))
|
|
after := heard("after-restart")
|
|
s.remember(after)
|
|
if len(after.Unheard) != 0 {
|
|
t.Errorf("a restarted runtime was named unheard: %+v", after.Unheard)
|
|
}
|
|
if len(s.runtimes) != 1 {
|
|
t.Errorf("the old instance is still kept: %d runtimes", len(s.runtimes))
|
|
}
|
|
}
|