Compare commits
80
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
676ce657f1 | ||
|
|
67635ef89f | ||
|
|
4cdf63443d | ||
|
|
1e9995faba | ||
|
|
f6c401ecd2 | ||
|
|
c2f01bf10b | ||
|
|
be2fc2e9b6 | ||
|
|
aa05629375 | ||
|
|
35a7ff70fe | ||
|
|
747bfa71aa | ||
|
|
316a5a2fc3 | ||
|
|
0edcd2501d | ||
|
|
ac645e2cae | ||
|
|
a479a90fb2 | ||
|
|
3c9068f99a | ||
|
|
250aa73a56 | ||
|
|
e32eef2379 | ||
|
|
070ae53128 | ||
|
|
8c0618cfc5 | ||
|
|
54680238a0 | ||
|
|
200249a84f | ||
|
|
67850457ef | ||
|
|
9d67b0995f | ||
|
|
e9009b7f04 | ||
|
|
9fc97e45c9 | ||
|
|
1583fa85d9 | ||
|
|
95a12105cf | ||
|
|
2966c1e32c | ||
|
|
1a0424117a | ||
|
|
23f5babe33 | ||
|
|
27266f99a3 | ||
|
|
a9b02bc676 | ||
|
|
f016c67a72 | ||
|
|
6cc3398880 | ||
|
|
8cdb7c3032 | ||
|
|
3baeaece61 | ||
|
|
a5097dc69a | ||
|
|
9f25e5c9f6 | ||
|
|
bb6a0e3192 | ||
|
|
3331719407 | ||
|
|
5d6cee9d84 | ||
|
|
2cd5550dc7 | ||
|
|
d8ac69904d | ||
|
|
b0ded57294 | ||
|
|
77b05089c8 | ||
|
|
0211844e6a | ||
|
|
0e985f9bf1 | ||
|
|
860043fd7f | ||
|
|
589bfc0bc5 | ||
|
|
3b5a38e4a4 | ||
|
|
755f77ca45 | ||
|
|
9153a280f8 | ||
|
|
8ee6db7d9a | ||
|
|
5dffb920ff | ||
|
|
e0fadc43d9 | ||
|
|
d295fd4250 | ||
|
|
18260ce9be | ||
|
|
4165ad59b5 | ||
|
|
f183b86fa0 | ||
|
|
06e6721ed6 | ||
|
|
7412c5b6c9 | ||
|
|
33d136e702 | ||
|
|
6b3e15042f | ||
|
|
21d038d339 | ||
|
|
5d70eb3e35 | ||
|
|
d3b0330ed1 | ||
|
|
9921486fd5 | ||
|
|
d0dcae53a8 | ||
|
|
52e275ad2f | ||
|
|
954e1a53ee | ||
|
|
424a7eec82 | ||
|
|
09c5f809cb | ||
|
|
847055c758 | ||
|
|
aee834aeb6 | ||
|
|
494d0787e2 | ||
|
|
172b5af589 | ||
|
|
3f0c1a6cec | ||
|
|
4348b42844 | ||
|
|
82aefe1988 | ||
|
|
d535dd5322 |
-80
@@ -1,80 +0,0 @@
|
||||
ARG NODE_BASE=node:22-bookworm-slim
|
||||
# The mesh-tools module: the two images every TypeScript module is COMPILED in and may RUN in. The
|
||||
# runtime itself ships as the node-tools module's bundle (node-tools/, novox/hq ADR 0175, to-be 38
|
||||
# WP3); these images are the toolchain for TypeScript bundles and the base a module's own service
|
||||
# may still be built on. They are no longer how tools reach a node.
|
||||
#
|
||||
# **They were the same image, and that was a mistake.** A module's recipe starts from this and
|
||||
# invokes the compiler out of it, so the compiler had to be here — and because the same image was
|
||||
# also what every module ran in, every running container on every machine carried a TypeScript
|
||||
# compiler it would never invoke. The answer is separate stages rather than one image bad at both
|
||||
# jobs. `build.artifacts` in module.json names the published stage each — `toolchain` and `runtime`.
|
||||
#
|
||||
# The `deps` stage is neither published nor named there: it is where the mesh's package registry is
|
||||
# reached, so it is where — and only where — the credential to reach it exists. The toolchain copies
|
||||
# resolved node_modules out of it, so the credential is in no image any machine ever holds
|
||||
# (novox/hq ADR 0076). This is the buildkit-secret's job done without buildkit, because a machine's
|
||||
# docker may carry no buildx.
|
||||
|
||||
# ---- deps: node_modules resolved from the mesh's registry, credential and all ----------------
|
||||
FROM ${NODE_BASE} AS deps
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY node-tools/package.json ./
|
||||
# The builder writes .npmrc into the build context; it authenticates to the mesh's package registry
|
||||
# for the @novox scope, which is where @novox/mesh-sdk resolves. This stage is not published, so the
|
||||
# credential travels no further than here. Development dependencies included: the compiler is one.
|
||||
COPY .npmrc ./.npmrc
|
||||
RUN npm install --no-audit --no-fund
|
||||
# **The SDK this image carries is the one the mesh last published** (novox/hq issue 212). The range in
|
||||
# package.json is resolved once and the layer above is cached, so a release reached no toolchain until
|
||||
# that file changed. The exact version arrives as a build argument from the SDK module's published
|
||||
# package (module.json `build.on`), so a new release is a new argument, this layer runs again — and
|
||||
# the planner orders this module after the SDK, so a release rebuilds the toolchain and, after it,
|
||||
# every bundle compiled in it (issue 211).
|
||||
ARG MESH_SDK=@novox/mesh-sdk@latest
|
||||
RUN npm install --no-audit --no-fund "${MESH_SDK}"
|
||||
|
||||
# ---- compiling: the runtime's own code built, WITHOUT the credential -------------------------
|
||||
FROM ${NODE_BASE} AS compiling
|
||||
WORKDIR /app
|
||||
COPY node-tools/package.json ./
|
||||
# The resolved libraries, but not the .npmrc that resolved them.
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
COPY node-tools/tsconfig.json ./
|
||||
COPY node-tools/src ./src
|
||||
RUN npm run build
|
||||
|
||||
# ---- what a running bundle needs, and nothing else -------------------------------------------
|
||||
# Its own stage so the toolchain image keeps its build tools while the runtime image does not.
|
||||
FROM compiling AS lean
|
||||
RUN npm prune --omit=dev
|
||||
|
||||
# ---- toolchain: what a TypeScript bundle is compiled in ---------------------------------------
|
||||
# The compiler, and esbuild (a development dependency) at /app/node_modules/esbuild: the builder
|
||||
# bundles every entrypoint and launcher into one file with it (novox/hq ADR 0193), so a bundle
|
||||
# carries what it imports and not this image's node_modules.
|
||||
# Beside the compiler, at /app/runtime, what every TypeScript bundle runs with: the production
|
||||
# dependencies the SDK and the runtime need, and a package.json saying the compiled files are ES
|
||||
# modules. The builder copies this directory whole into a compiled bundle (novox/hq ADR 0188 §5),
|
||||
# so a bundle unpacked on a machine starts — a `.js` without that package.json is read as
|
||||
# CommonJS, and an import of `nats` without node_modules beside it resolves to nothing.
|
||||
FROM compiling AS toolchain
|
||||
COPY --from=lean /app/node_modules /app/runtime/node_modules
|
||||
RUN printf '{"type":"module","private":true}\n' > /app/runtime/package.json
|
||||
# **git, because a pull request's merge check runs in this image too** (novox/hq ADR 0237): a repository's
|
||||
# own merge-check.sh that declares this toolchain runs here, and a suite that reads a repository's history
|
||||
# (the lab's last-run record) ran into "spawnSync git ENOENT". Nothing compiled here calls it.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ---- runtime: what a module's own service may run in ------------------------------------------
|
||||
FROM ${NODE_BASE} AS runtime
|
||||
WORKDIR /app
|
||||
COPY node-tools/package.json ./
|
||||
COPY --from=lean /app/node_modules ./node_modules
|
||||
COPY --from=compiling /app/dist ./dist
|
||||
ENTRYPOINT ["node", "dist/main.js"]
|
||||
@@ -1,13 +1,11 @@
|
||||
# mesh-tools
|
||||
|
||||
Two modules in one repository (novox/hq ADR 0069), one piece of software:
|
||||
One module, `node-tools`; the toolchain image module that stood beside it is retired (novox/hq ADR 0524 §5):
|
||||
|
||||
- **`node-tools`** (`node-tools/`) — the node's **tool runtime** as a module (ADR 0175, to-be 38
|
||||
WP3): one process per machine the host runs from this bundle, serving every assigned module's tools
|
||||
and every held seat's verbs on the bus, and answering MCP on the machine's loopback — the console
|
||||
(design 34). The code, its tests and the `mesh` client all live there.
|
||||
- **`mesh-tools`** (this directory) — the two images TypeScript bundles are compiled in and a module's
|
||||
own *service* may still run in. Built from the same code; no longer how tools reach a node.
|
||||
|
||||
The runtime:
|
||||
|
||||
@@ -49,8 +47,7 @@ MESH_BROKER_URL a plain URL instead of the credential, for the bootstrap
|
||||
|
||||
`node dist/main.js`. On a node the controller composes the variables and the host supervises the
|
||||
process like any other host-side workload (novox/hq to-be 38). As `node-tools` the same process is
|
||||
the console: MCP on `127.0.0.1:4270` (or `MESH_CONSOLE_LISTEN`). The container (`Dockerfile`) is how
|
||||
a module's own *service* may still be built; it is no longer how tools reach a node.
|
||||
the console: MCP on `127.0.0.1:4270` (or `MESH_CONSOLE_LISTEN`).
|
||||
|
||||
## `mesh` — the tools for whoever is on a machine
|
||||
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
#!/bin/sh
|
||||
# mesh-check-toolchain: go
|
||||
#
|
||||
# The tool runtime's own check (novox/hq ADR 0237 as amended): the second layer of a pull request's merge
|
||||
# check, `mesh/repo-check`, run by the build seat in the mesh's Go toolchain — and by hand.
|
||||
#
|
||||
# The gate — every machine of the facts snapshot composed with this change — is the build seat's first
|
||||
# layer (`mesh/merge-gate`), run because the mesh's module graph builds node-tools and mesh-tools from
|
||||
# here. This is the code's own: node-tools (Go) formatted, vetted and its suite, every test on a bus of its
|
||||
# own at the release the mesh runs (internal/meshtest), packages in parallel, under the race detector when
|
||||
# the toolchain has a C compiler.
|
||||
#
|
||||
# **Said, never passed silently**: the runtime's and the console's tests launch TypeScript and Python
|
||||
# bundles that import the mesh's own package (@novox/mesh-sdk), which comes from the package registry — and
|
||||
# a check of code nobody has approved is given no credential for it. Where node and that package are not
|
||||
# present those two packages are not tested here, and the TypeScript in node-tools/src neither.
|
||||
set -eu
|
||||
cd node-tools
|
||||
|
||||
unformatted=$(gofmt -l cmd internal)
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "not gofmt'd:"
|
||||
echo "$unformatted"
|
||||
exit 1
|
||||
fi
|
||||
CGO_ENABLED=0 go vet ./...
|
||||
|
||||
packages=$(go list ./...)
|
||||
bundleless=""
|
||||
if ! command -v node >/dev/null 2>&1 || [ ! -d node_modules/@novox/mesh-sdk ]; then
|
||||
echo "NOT TESTED HERE: internal/runtime and internal/console launch bundles that need node and @novox/mesh-sdk"
|
||||
packages=$(printf '%s\n' "$packages" | grep -v -e '/internal/runtime$' -e '/internal/console$')
|
||||
# What of the console launches no bundle runs anyway: how an address resolves (novox/hq issue 287),
|
||||
# and how a search ranks (novox/hq ADR 0245).
|
||||
bundleless='^(TestASeatAndAModuleOfOneNameAreEachReached|TestSearch.*|TestASearchByCommandSaysWhatItReplaces)$'
|
||||
# And what of the runtime launches no bundle: how a bundle's seat traffic reaches the bus, what its
|
||||
# environment holds, what a runtime on a module's own account serves (novox/hq ADR 0259), and that a write
|
||||
# by compare-and-set loses no update under contention (novox/hq research 042).
|
||||
runtimeless='^(TestARuntimeOnAModulesOwnAccountServesThatModuleAlone|TestNoBusWordReachesABundle|TestAToolCallNeverReachesASeatsEventAcceptOrProof|TestABundlesSeatTrafficGoesThroughItsModulesBus|TestALostCompareAndSetCarriesItsCode|TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate|TestAStaleWriteIsRefusedAndWritesNothing|TestACallOnTheCallerNamedSubjectReachesTheModuleWithItsCaller|TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone)$'
|
||||
fi
|
||||
if command -v gcc >/dev/null 2>&1; then
|
||||
CGO_ENABLED=1 go test -race -count=1 $packages
|
||||
else
|
||||
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
|
||||
CGO_ENABLED=0 go test -count=1 $packages
|
||||
fi
|
||||
if [ -n "$bundleless" ]; then
|
||||
CGO_ENABLED=0 go test -count=1 -run "$bundleless" ./internal/console
|
||||
fi
|
||||
if [ -n "${runtimeless:-}" ]; then
|
||||
CGO_ENABLED=0 go test -count=1 -run "$runtimeless" ./internal/runtime
|
||||
fi
|
||||
echo "NOT TESTED HERE: node-tools/src (TypeScript) needs @novox/mesh-sdk from the package registry"
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"module": "mesh-tools",
|
||||
"version": "1",
|
||||
"slug": "tools",
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "build",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile",
|
||||
"target": "toolchain"
|
||||
},
|
||||
{
|
||||
"name": "runtime",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile",
|
||||
"target": "runtime"
|
||||
}
|
||||
],
|
||||
"on": [
|
||||
{
|
||||
"arg": "NODE_BASE",
|
||||
"image": "node@sha256:48e4b67d85f87bd551df43704e24d252f56cc5f8e9718841aace50f19948f0f9"
|
||||
},
|
||||
{
|
||||
"arg": "MESH_SDK",
|
||||
"module": "mesh-sdk",
|
||||
"artifact": "lib"
|
||||
}
|
||||
]
|
||||
},
|
||||
"resources": []
|
||||
}
|
||||
@@ -12,7 +12,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand/v2"
|
||||
@@ -24,6 +26,8 @@ import (
|
||||
"github.com/novox/mesh-tools/node-tools/internal/alive"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/console"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/provide"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
@@ -40,8 +44,9 @@ func main() {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %q is not a mode; this runtime serves (novox/hq ADR 0193)\n", os.Args[1])
|
||||
os.Exit(2)
|
||||
}
|
||||
cred := credential()
|
||||
conn := connectPatiently(cred)
|
||||
until := time.Now().Add(startWait)
|
||||
cred := credential(until)
|
||||
conn := connectPatiently(cred, until)
|
||||
|
||||
served, err := runtime.ServedModulesFrom(os.Getenv(runtime.ToolModules), cred.Module)
|
||||
if err != nil {
|
||||
@@ -62,6 +67,10 @@ func main() {
|
||||
stopSaying = alive.Keep(conn, cred.Node, alive.Every, log.Printf)
|
||||
}
|
||||
|
||||
// As the machine's runtime it is also the provider of `mcp-endpoint`: it answers its consumers through the
|
||||
// SDK's helper wherever it runs, whether or not the module is handed (novox/hq ADR 0349 clause 12, issue 791).
|
||||
stopProviding := provide.Start(conn, cred.Module, cred.Node, time.Minute, log.Printf)
|
||||
|
||||
listen := os.Getenv("MESH_CONSOLE_LISTEN")
|
||||
if listen == "" && cred.Module == runtimeModule {
|
||||
listen = consoleListen
|
||||
@@ -83,7 +92,11 @@ func main() {
|
||||
signals := make(chan os.Signal, 1)
|
||||
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
|
||||
<-signals
|
||||
// First of all: the service manager signalled every bundle with this process, and none is started again
|
||||
// (issue 1126).
|
||||
launch.ShutDown()
|
||||
stopSaying()
|
||||
stopProviding()
|
||||
stop()
|
||||
if up != nil {
|
||||
_ = up.Close()
|
||||
@@ -91,48 +104,149 @@ func main() {
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
// credential reads the credential the mesh delivered; a missing or unreadable one is a fault of
|
||||
// configuration, said and final.
|
||||
func credential() bus.Credential {
|
||||
// startWait is how long a runtime waits, from its start, for its broker credential to be on the machine and for the bus
|
||||
// to accept the account it names (issue 1496): the node-engine starts a module's runtime before the provisioner's client
|
||||
// fills the credential, and the bus carries a new account only after it reloads its users. Past it the runtime gives up,
|
||||
// loudly; inside it the wait is said once.
|
||||
var startWait = 5 * time.Minute
|
||||
|
||||
// say is a line in the runtime's journal.
|
||||
func say(format string, args ...any) { fmt.Fprintf(os.Stderr, "node-tools: "+format+"\n", args...) }
|
||||
|
||||
// credential reads the credential the mesh delivered, waiting for it within the start's bound (issue 1496); one that can
|
||||
// never be a credential (no url) is a fault of configuration, said and final.
|
||||
func credential(until time.Time) bus.Credential {
|
||||
file := os.Getenv("MESH_BROKER_FILE")
|
||||
if file == "" {
|
||||
if url := os.Getenv("MESH_BROKER_URL"); url != "" {
|
||||
return bus.Credential{URL: url, Module: runtimeModule}
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "node-tools: set MESH_BROKER_FILE (a sealed credential) or MESH_BROKER_URL — there is no broker to reach")
|
||||
say("set MESH_BROKER_FILE (a sealed credential) or MESH_BROKER_URL — there is no broker to reach")
|
||||
os.Exit(1)
|
||||
}
|
||||
raw, err := os.ReadFile(file)
|
||||
cred, err := awaitCredential(file, until, say)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: cannot read the broker credential at %s: %v\n", file, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
var cred bus.Credential
|
||||
if err := json.Unmarshal(raw, &cred); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: cannot read the broker credential at %s: %v\n", file, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if cred.URL == "" {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %s carries no url — it is not a broker credential\n", file)
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
// connectPatiently retries while the bus is merely not reachable yet — the normal case at startup —
|
||||
// and gives up at once on what waiting cannot fix (issue 058).
|
||||
func connectPatiently(cred bus.Credential) *bus.Conn {
|
||||
for delay := 2 * time.Second; ; delay = min(delay*2, 30*time.Second) {
|
||||
conn, err := bus.Connect(cred)
|
||||
if err == nil {
|
||||
return conn
|
||||
// errNotCredential is a file that is there and readable and still no broker credential: waiting will not fix it.
|
||||
var errNotCredential = errors.New("it carries no url — it is not a broker credential")
|
||||
|
||||
// readCredential reads the credential at file. A file not there yet, empty, or not yet whole JSON is one the node-engine
|
||||
// has still to write: the error says so and waiting may fix it. errNotCredential is final.
|
||||
func readCredential(file string) (bus.Credential, error) {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return bus.Credential{}, err
|
||||
}
|
||||
if len(bytes.TrimSpace(raw)) == 0 {
|
||||
return bus.Credential{}, errors.New("the file is empty")
|
||||
}
|
||||
var cred bus.Credential
|
||||
if err := json.Unmarshal(raw, &cred); err != nil {
|
||||
return bus.Credential{}, err
|
||||
}
|
||||
if cred.URL == "" {
|
||||
return bus.Credential{}, errNotCredential
|
||||
}
|
||||
return cred, nil
|
||||
}
|
||||
|
||||
// awaitCredential is the credential at file, read at once and, while it is not there or not whole, again each time its
|
||||
// directory changes (runtime.WatchFile, inotify), until the time until. The wait is said once; at until it ends with an
|
||||
// error naming the file, how long it waited and what the last read found. A file that is not a credential ends it at once.
|
||||
func awaitCredential(file string, until time.Time, say func(string, ...any)) (bus.Credential, error) {
|
||||
cred, err := readCredential(file)
|
||||
if err == nil {
|
||||
return cred, nil
|
||||
}
|
||||
if errors.Is(err, errNotCredential) {
|
||||
return bus.Credential{}, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
began := time.Now()
|
||||
say("waits for its broker credential at %s (%v): the node-engine writes it once the provisioner's client fills it; "+
|
||||
"it gives up at %s", file, err, until.Format("15:04:05"))
|
||||
changed := make(chan struct{}, 1)
|
||||
stop, werr := runtime.WatchFile(file, func() {
|
||||
select {
|
||||
case changed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
if fatal := bus.Fatal(err); fatal != "" {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %s — waiting will not fix this; giving up\n", fatal)
|
||||
os.Exit(1)
|
||||
}, func(format string, args ...any) { say(format, args...) },
|
||||
fmt.Sprintf("the credential is read again every %s instead", rereadEvery))
|
||||
if werr != nil {
|
||||
say("cannot watch %s (%v): it is read again every %s instead", file, werr, rereadEvery)
|
||||
} else {
|
||||
defer stop()
|
||||
}
|
||||
for {
|
||||
left := time.Until(until)
|
||||
if left <= 0 {
|
||||
return bus.Credential{}, fmt.Errorf("cannot read the broker credential at %s: waited %s for it, and the last read "+
|
||||
"found: %v", file, time.Since(began).Round(time.Second), err)
|
||||
}
|
||||
// The watch wakes the read; the clock is only the backstop for a watch that could not be made or missed a change.
|
||||
select {
|
||||
case <-changed:
|
||||
case <-time.After(min(left, rereadEvery)):
|
||||
}
|
||||
if cred, err = readCredential(file); err == nil {
|
||||
say("read its broker credential at %s after %s", file, time.Since(began).Round(time.Second))
|
||||
return cred, nil
|
||||
}
|
||||
if errors.Is(err, errNotCredential) {
|
||||
return bus.Credential{}, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rereadEvery is the backstop of the wait for the credential, beside the watch of its directory.
|
||||
var rereadEvery = 30 * time.Second
|
||||
|
||||
// connectPatiently retries while the bus is merely not reachable yet — the normal case at startup — and, until the start's
|
||||
// bound, while the bus refuses the account: the bus carries a new account only after it reloads its users, which may come
|
||||
// after the credential is on the machine (issue 1496, issue 1423). What waiting cannot fix (a pin that does not match, an
|
||||
// address that is no URL) gives up at once (issue 058), and so does a refusal past the bound.
|
||||
func connectPatiently(cred bus.Credential, until time.Time) *bus.Conn {
|
||||
conn, err := connectWithin(cred, until, bus.Connect, time.Sleep, say)
|
||||
if err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
// connectWithin is connectPatiently with its dial, its sleep and its words given, as a test gives them.
|
||||
func connectWithin(cred bus.Credential, until time.Time, dial func(bus.Credential) (*bus.Conn, error), sleep func(time.Duration),
|
||||
say func(string, ...any)) (*bus.Conn, error) {
|
||||
refusedSaid := false
|
||||
for delay := 2 * time.Second; ; delay = min(delay*2, 30*time.Second) {
|
||||
conn, err := dial(cred)
|
||||
if err == nil {
|
||||
if refusedSaid {
|
||||
say("the bus accepts the account now")
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
wait := delay + time.Duration(rand.IntN(1000))*time.Millisecond
|
||||
fmt.Fprintf(os.Stderr, "node-tools: the broker is not reachable yet (%v); retrying in %ds\n", err, int(wait.Round(time.Second)/time.Second))
|
||||
time.Sleep(wait)
|
||||
switch fatal := bus.Fatal(err); {
|
||||
case bus.Refused(err) && time.Now().Add(wait).Before(until):
|
||||
if !refusedSaid {
|
||||
say("the bus refuses this account (%v): it may not carry it yet, since it loads a new account by a reload; "+
|
||||
"retrying until %s", err, until.Format("15:04:05"))
|
||||
refusedSaid = true
|
||||
}
|
||||
case bus.Refused(err) && refusedSaid:
|
||||
return nil, fmt.Errorf("the bus still refuses this account (%v) at the end of the start's bound, %s; giving up", err,
|
||||
until.Format("15:04:05"))
|
||||
case fatal != "":
|
||||
return nil, fmt.Errorf("%s — waiting will not fix this; giving up", fatal)
|
||||
default:
|
||||
say("the broker is not reachable yet (%v); retrying in %ds", err, int(wait.Round(time.Second)/time.Second))
|
||||
}
|
||||
sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
// A runtime started before its credential is on the machine waits for it, woken by the watch of its directory, and gives
|
||||
// up loudly at its bound; a file that is no credential ends the wait at once (issue 1496: on ace mesh-build-node's and
|
||||
// mesh-build-image's runtimes exited twice on "cannot read the broker credential ... no such file").
|
||||
func TestAwaitCredential(t *testing.T) {
|
||||
const good = `{"url":"tls://bus.example:4222","module":"m"}`
|
||||
rereadEvery = time.Hour // only the watch can wake the read in time
|
||||
settle := runtime.Settle
|
||||
runtime.Settle = 10 * time.Millisecond // a unit test stays well under a second
|
||||
t.Cleanup(func() { rereadEvery, runtime.Settle = 30*time.Second, settle })
|
||||
place := func(path, body string) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
before string // written before the wait, "" none
|
||||
later string // written 20ms into the wait, "" none
|
||||
deeper bool // the file's directory is not there at the start
|
||||
bound time.Duration
|
||||
want string // "" a credential; else a word of the error
|
||||
said int
|
||||
}{
|
||||
{name: "there at the start", before: good, bound: time.Second, said: 0},
|
||||
{name: "written during the wait", later: good, bound: 5 * time.Second, said: 2},
|
||||
{name: "empty, then filled", before: " ", later: good, bound: 5 * time.Second, said: 2},
|
||||
{name: "its directory made during the wait", later: good, deeper: true, bound: 5 * time.Second, said: 2},
|
||||
{name: "never written", bound: 100 * time.Millisecond, want: "waited", said: 1},
|
||||
{name: "no url", before: `{"module":"m"}`, bound: 5 * time.Second, want: "not a broker credential", said: 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
file := filepath.Join(dir, "broker")
|
||||
if c.deeper {
|
||||
file = filepath.Join(dir, "var", "lib", "m", "broker")
|
||||
}
|
||||
if c.before != "" {
|
||||
place(file, c.before)
|
||||
}
|
||||
if c.later != "" {
|
||||
go func() { time.Sleep(20 * time.Millisecond); place(file, c.later) }()
|
||||
}
|
||||
var said []string
|
||||
began := time.Now()
|
||||
cred, err := awaitCredential(file, time.Now().Add(c.bound), func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
switch {
|
||||
case c.want == "" && (err != nil || cred.URL == ""):
|
||||
t.Fatalf("no credential: %v", err)
|
||||
case c.want != "" && (err == nil || !strings.Contains(err.Error(), c.want)):
|
||||
t.Fatalf("error %v, want one saying %q", err, c.want)
|
||||
case c.want == "" && c.later != "" && time.Since(began) > 3*time.Second:
|
||||
t.Fatalf("read after %s: the watch did not wake it", time.Since(began))
|
||||
}
|
||||
if len(said) != c.said {
|
||||
t.Fatalf("said %d lines, want %d: %q", len(said), c.said, said)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A bus that refuses the account at a start is asked again until the start's bound, said once; a refusal past it, a pin
|
||||
// that does not match and an address that is no URL give up (issue 1496, issue 058).
|
||||
func TestConnectWithin(t *testing.T) {
|
||||
refused := errors.New("nats: Authorization Violation")
|
||||
unreachable := errors.New("nats: no servers available for connection")
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
errs []error // what each dial answers, then a connection
|
||||
bound time.Duration
|
||||
fails string
|
||||
saidAt int // lines said
|
||||
}{
|
||||
{name: "refused twice, then accepted", errs: []error{refused, refused}, bound: time.Hour, saidAt: 2},
|
||||
{name: "unreachable, then accepted", errs: []error{unreachable}, bound: time.Hour, saidAt: 1},
|
||||
{name: "refused past the bound", errs: []error{refused, refused}, bound: 0, fails: "refused this account"},
|
||||
{name: "refused, then the bound ends", errs: []error{refused, refused, refused, refused, refused, refused}, bound: 5 * time.Second,
|
||||
fails: "still refuses"},
|
||||
{name: "pin", errs: []error{fmt.Errorf("tls: %w", bus.ErrPin)}, bound: time.Hour, fails: "pin"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
dials := 0
|
||||
dial := func(bus.Credential) (*bus.Conn, error) {
|
||||
dials++
|
||||
if dials <= len(c.errs) {
|
||||
return nil, c.errs[dials-1]
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
until := time.Now().Add(c.bound)
|
||||
var said []string
|
||||
_, err := connectWithin(bus.Credential{URL: "tls://bus.example:4222"}, until, dial, func(time.Duration) {},
|
||||
func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
switch {
|
||||
case c.fails == "" && err != nil:
|
||||
t.Fatalf("gave up: %v", err)
|
||||
case c.fails != "" && (err == nil || !strings.Contains(err.Error(), c.fails)):
|
||||
t.Fatalf("error %v, want one saying %q", err, c.fails)
|
||||
case c.fails == "" && len(said) != c.saidAt:
|
||||
t.Fatalf("said %d lines, want %d: %q", len(said), c.saidAt, said)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -8,15 +8,15 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
+2
-2
@@ -1,3 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
@@ -16,8 +18,6 @@ github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
|
||||
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -35,6 +37,12 @@ var Window = 750 * time.Millisecond
|
||||
// serves is waited for until it arrives or this passes, and a service that never sends it is named.
|
||||
var Patience = 5 * time.Second
|
||||
|
||||
// Digest is the PING answer's metadata key for a digest of what the instance serves now: a console
|
||||
// that holds the answer of that digest does not ask for it again (novox/hq issue 554). Every round
|
||||
// pulled every runtime's whole announcement — some 880 kB, every five seconds while a session works —
|
||||
// to the laptop, and the hundred-byte PING answers queued behind it.
|
||||
const Digest = "digest"
|
||||
|
||||
// Shortened is the service metadata key a runtime sets when its announcement was cut to fit the bus.
|
||||
const Shortened = "shortened"
|
||||
|
||||
@@ -157,7 +165,14 @@ func Serve(conn *bus.Conn, s Service, current func() []Endpoint) (func(), error)
|
||||
var v any
|
||||
switch parts[1] {
|
||||
case "PING":
|
||||
v = micro.Ping{ServiceIdentity: s.identity(), Type: micro.PingResponseType}
|
||||
id := s.identity()
|
||||
md := map[string]string{}
|
||||
for k, v := range id.Metadata {
|
||||
md[k] = v
|
||||
}
|
||||
md[Digest] = DigestOf(Info(s, current()))
|
||||
id.Metadata = md
|
||||
v = micro.Ping{ServiceIdentity: id, Type: micro.PingResponseType}
|
||||
case "INFO":
|
||||
v = Info(s, current())
|
||||
case "STATS":
|
||||
@@ -205,6 +220,16 @@ func Serve(conn *bus.Conn, s Service, current func() []Endpoint) (func(), error)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DigestOf is a digest of an info_response: equal for equal announcements, whatever the moment.
|
||||
func DigestOf(i micro.Info) string {
|
||||
body, err := json.Marshal(i)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return hex.EncodeToString(sum[:12])
|
||||
}
|
||||
|
||||
// shorten is an info_response cut to fit the bus: descriptions to their first line, then none, the
|
||||
// service's metadata saying which (so discovery can tell a person what it was given).
|
||||
func shorten(logf func(string, ...any), info micro.Info, body []byte, limit int64) []byte {
|
||||
@@ -244,6 +269,11 @@ type Heard struct {
|
||||
Took time.Duration
|
||||
Bytes int
|
||||
Shortened string // why its descriptions are not whole, or ""
|
||||
// Digest is the digest its PING gave for this answer, or "" when it gave none.
|
||||
Digest string
|
||||
// Reused is true when the instance's PING said it serves what the console already held, so its
|
||||
// announcement was not asked for again; Took is then how long its PING took.
|
||||
Reused bool
|
||||
}
|
||||
|
||||
// Machine is the machine an instance runs on: its metadata's, or the one its endpoints name.
|
||||
@@ -273,12 +303,24 @@ type Discovery struct {
|
||||
Waited time.Duration
|
||||
Heard []Heard
|
||||
Silent []Instance
|
||||
// Pinged is how many PING answers arrived.
|
||||
Pinged int
|
||||
}
|
||||
|
||||
// Gather asks every service on the bus who it is and what it serves, and waits at least the window
|
||||
// and at most the patience: until every instance that answered PING has answered INFO. An answer
|
||||
// Empty is a round in which nothing arrived at all, not even a PING: no answer about the mesh, but a
|
||||
// round that failed (novox/hq issue 554).
|
||||
func (d Discovery) Empty() bool { return d.Pinged == 0 && len(d.Heard) == 0 }
|
||||
|
||||
// Known is the announcement a console already holds for an instance at this digest, if any.
|
||||
type Known func(name, id, digest string) (Heard, bool)
|
||||
|
||||
// Gather asks every service on the bus who it is (`$SRV.PING`), and each instance that answers what it
|
||||
// serves (`$SRV.INFO.<name>.<id>`) unless its PING's digest is one `known` holds; it waits at least the
|
||||
// window and at most the patience: until at least one instance answered PING and every instance that
|
||||
// did has said what it serves. A round in which no PING arrives within the window is not taken as the
|
||||
// mesh's answer that nothing is there (novox/hq issue 554): it waits on, up to the patience. An answer
|
||||
// that is not the protocol's is skipped.
|
||||
func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
func Gather(conn *bus.Conn, known Known) (Discovery, error) {
|
||||
start := time.Now()
|
||||
d := Discovery{At: start.UTC()}
|
||||
pings, stopPings, err := conn.Collect("$SRV.PING", nil)
|
||||
@@ -286,7 +328,7 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
return d, err
|
||||
}
|
||||
defer stopPings()
|
||||
infos, stopInfos, err := conn.Collect("$SRV.INFO", nil)
|
||||
infos, askInfo, stopInfos, err := conn.Collector("$SRV.INFO")
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
@@ -296,8 +338,14 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
pinged := map[key]int{}
|
||||
answered := map[key]int{}
|
||||
where := map[key]string{}
|
||||
digest := map[key]string{}
|
||||
asked := map[key]bool{}
|
||||
reused := map[key]int{} // index in d.Heard of an answer held already
|
||||
var order []key
|
||||
complete := func() bool {
|
||||
if len(pinged) == 0 {
|
||||
return false // nobody said they are there: not an answer, a round that has not arrived
|
||||
}
|
||||
for k, n := range pinged {
|
||||
if answered[k] < n {
|
||||
return false
|
||||
@@ -305,6 +353,15 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
}
|
||||
return true
|
||||
}
|
||||
ask := func(k key) {
|
||||
if asked[k] {
|
||||
return
|
||||
}
|
||||
asked[k] = true
|
||||
if err := askInfo("$SRV.INFO."+k.name+"."+k.id, nil); err != nil {
|
||||
conn.Logf("[mesh-tools] discovery could not ask %s/%s what it serves: %v", k.name, k.id, err)
|
||||
}
|
||||
}
|
||||
least := time.NewTimer(Window)
|
||||
defer least.Stop()
|
||||
most := time.NewTimer(Patience)
|
||||
@@ -317,21 +374,49 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
if json.Unmarshal(a.Data, &p) != nil || p.Type != micro.PingResponseType {
|
||||
continue
|
||||
}
|
||||
d.Pinged++
|
||||
k := key{p.Name, p.ID}
|
||||
if _, seen := pinged[k]; !seen {
|
||||
_, seen := pinged[k]
|
||||
if !seen {
|
||||
order = append(order, k)
|
||||
}
|
||||
pinged[k]++
|
||||
if where[k] == "" {
|
||||
where[k] = p.Metadata["node"]
|
||||
}
|
||||
if seen {
|
||||
// Two instances under one name and id: what each serves is asked of both, never assumed.
|
||||
if i, ok := reused[k]; ok {
|
||||
d.Heard = append(d.Heard[:i], d.Heard[i+1:]...)
|
||||
for kk, j := range reused {
|
||||
if j > i {
|
||||
reused[kk] = j - 1
|
||||
}
|
||||
}
|
||||
delete(reused, k)
|
||||
answered[k]--
|
||||
}
|
||||
ask(k)
|
||||
continue
|
||||
}
|
||||
digest[k] = p.Metadata[Digest]
|
||||
if h, ok := heldAt(known, k.name, k.id, digest[k]); ok {
|
||||
h.Took, h.Reused, h.Digest = a.At.Sub(start), true, digest[k]
|
||||
reused[k] = len(d.Heard)
|
||||
d.Heard = append(d.Heard, h)
|
||||
answered[k]++
|
||||
break
|
||||
}
|
||||
ask(k)
|
||||
case a := <-infos:
|
||||
var i micro.Info
|
||||
if json.Unmarshal(a.Data, &i) != nil || i.Type != micro.InfoResponseType {
|
||||
continue
|
||||
}
|
||||
answered[key{i.Name, i.ID}]++
|
||||
d.Heard = append(d.Heard, Heard{Info: i, Took: a.At.Sub(start), Bytes: len(a.Data), Shortened: i.Metadata[Shortened]})
|
||||
k := key{i.Name, i.ID}
|
||||
answered[k]++
|
||||
d.Heard = append(d.Heard, Heard{Info: i, Took: a.At.Sub(start), Bytes: len(a.Data), Shortened: i.Metadata[Shortened],
|
||||
Digest: digest[k]})
|
||||
case <-least.C:
|
||||
windowOver = true
|
||||
case <-most.C:
|
||||
@@ -350,6 +435,14 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// heldAt is the answer `known` holds for an instance at this digest; never for a PING without one.
|
||||
func heldAt(known Known, name, id, digest string) (Heard, bool) {
|
||||
if known == nil || digest == "" {
|
||||
return Heard{}, false
|
||||
}
|
||||
return known(name, id, digest)
|
||||
}
|
||||
|
||||
// Endpoints reads an info_response's endpoints back into what they announce.
|
||||
func Endpoints(i micro.Info) []Endpoint {
|
||||
var out []Endpoint
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
)
|
||||
|
||||
// novox/hq issue 554: the laptop's console took a round in which not even a PING arrived within the
|
||||
// 750 ms window as the mesh's answer that nothing is there, and refused every seat and module call. A
|
||||
// PING that arrives after the window is waited for, up to the patience.
|
||||
func TestARoundWaitsForAPingThatArrivesAfterTheWindow(t *testing.T) {
|
||||
was := Patience
|
||||
Patience = 2500 * time.Millisecond
|
||||
t.Cleanup(func() { Patience = was })
|
||||
|
||||
late := connect(t, "node-tools", "laptop")
|
||||
s := Service{Name: "node-tools", ID: "laptop", Metadata: map[string]string{"node": "laptop"}}
|
||||
ping, _ := json.Marshal(micro.Ping{ServiceIdentity: s.identity(), Type: micro.PingResponseType})
|
||||
stop, err := late.Raw("$SRV.PING", func(string, []byte) []byte { time.Sleep(Window + 300*time.Millisecond); return ping })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
body, _ := json.Marshal(Info(s, []Endpoint{{Kind: KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Subject: "mesh.mod.slack.tool.slack_check.laptop"}}))
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO.node-tools.laptop"} {
|
||||
stop, err := late.Raw(subject, func(string, []byte) []byte { return body })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
late.Flush()
|
||||
|
||||
d, err := Gather(connect(t, "console", "laptop"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.Empty() || len(d.Heard) != 1 || d.Heard[0].Machine() != "laptop" {
|
||||
t.Fatalf("a PING after the window was not waited for: waited %s, pinged %d, heard %+v", d.Waited, d.Pinged, d.Heard)
|
||||
}
|
||||
if d.Waited < Window || d.Waited >= Patience {
|
||||
t.Errorf("waited %s; the PING came after %s and patience is %s", d.Waited, Window+300*time.Millisecond, Patience)
|
||||
}
|
||||
}
|
||||
|
||||
// A round in which nothing at all arrives is said to be empty, after the patience — not answered as an
|
||||
// empty mesh at the window.
|
||||
func TestARoundThatHearsNothingIsEmptyAfterThePatience(t *testing.T) {
|
||||
was := Patience
|
||||
Patience = 1500 * time.Millisecond
|
||||
t.Cleanup(func() { Patience = was })
|
||||
d, err := Gather(connect(t, "console", "laptop"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !d.Empty() || d.Waited < Patience {
|
||||
t.Errorf("empty %v after %s (patience %s)", d.Empty(), d.Waited, Patience)
|
||||
}
|
||||
}
|
||||
|
||||
// What a runtime announces is fetched again only when its PING's digest changed: every round pulled
|
||||
// some 880 kB of announcements to the laptop, and the PING answers queued behind them (issue 554).
|
||||
func TestAnUnchangedAnnouncementIsNotFetchedAgain(t *testing.T) {
|
||||
runtime := connect(t, "node-tools", "desk")
|
||||
var endpoints atomic.Value
|
||||
endpoints.Store([]Endpoint{{Kind: KindTool, Module: "alpha", Tool: "one", Node: "desk", Subject: "mesh.mod.alpha.tool.one.desk"}})
|
||||
stop, err := Serve(runtime, Service{Name: "node-tools", ID: "desk", Metadata: map[string]string{"node": "desk"}},
|
||||
func() []Endpoint { return endpoints.Load().([]Endpoint) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
runtime.Flush()
|
||||
// Every request for what desk serves, counted where it is answered.
|
||||
var asked atomic.Int32
|
||||
spy := connect(t, "node-tools", "spy")
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO.node-tools", "$SRV.INFO.node-tools.desk"} {
|
||||
stop, err := spy.Raw(subject, func(string, []byte) []byte { asked.Add(1); return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
spy.Flush()
|
||||
|
||||
console := connect(t, "console", "laptop")
|
||||
first, err := Gather(console, nil)
|
||||
if err != nil || len(first.Heard) != 1 || first.Heard[0].Digest == "" || first.Heard[0].Reused {
|
||||
t.Fatalf("first round: %v %+v", err, first.Heard)
|
||||
}
|
||||
if asked.Load() != 1 {
|
||||
t.Errorf("the first round asked what desk serves %d time(s)", asked.Load())
|
||||
}
|
||||
held := first.Heard[0]
|
||||
known := func(name, id, digest string) (Heard, bool) {
|
||||
if name == held.Info.Name && id == held.Info.ID && digest == held.Digest {
|
||||
return held, true
|
||||
}
|
||||
return Heard{}, false
|
||||
}
|
||||
asked.Store(0)
|
||||
second, err := Gather(console, known)
|
||||
if err != nil || len(second.Heard) != 1 || !second.Heard[0].Reused || len(Endpoints(second.Heard[0].Info)) != 1 {
|
||||
t.Fatalf("an unchanged announcement: %v %+v", err, second.Heard)
|
||||
}
|
||||
if asked.Load() != 0 {
|
||||
t.Errorf("an unchanged announcement was asked for %d time(s)", asked.Load())
|
||||
}
|
||||
|
||||
endpoints.Store([]Endpoint{
|
||||
{Kind: KindTool, Module: "alpha", Tool: "one", Node: "desk", Subject: "mesh.mod.alpha.tool.one.desk"},
|
||||
{Kind: KindTool, Module: "alpha", Tool: "two", Node: "desk", Subject: "mesh.mod.alpha.tool.two.desk"},
|
||||
})
|
||||
third, err := Gather(console, known)
|
||||
if err != nil || len(third.Heard) != 1 || third.Heard[0].Reused || len(Endpoints(third.Heard[0].Info)) != 2 ||
|
||||
third.Heard[0].Digest == held.Digest {
|
||||
t.Fatalf("a changed announcement was not fetched: %v %+v", err, third.Heard)
|
||||
}
|
||||
if asked.Load() != 1 {
|
||||
t.Errorf("a changed announcement was asked for %d time(s)", asked.Load())
|
||||
}
|
||||
}
|
||||
@@ -33,18 +33,22 @@ func lagging(t *testing.T, conn *bus.Conn, s Service, lag time.Duration, endpoin
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
stop, err = conn.Raw("$SRV.INFO", func(string, []byte) []byte {
|
||||
// Asked of every service, and of this instance by name and id — as discovery asks it (issue 554).
|
||||
info := func(string, []byte) []byte {
|
||||
if lag < 0 {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(lag)
|
||||
body, _ := json.Marshal(Info(s, endpoints))
|
||||
return body
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO." + s.Name + "." + s.ID} {
|
||||
stop, err := conn.Raw(subject, info)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
conn.Flush()
|
||||
}
|
||||
|
||||
@@ -73,7 +77,7 @@ func TestDiscoveryWaitsForEveryRuntimeThatSaidItIsThere(t *testing.T) {
|
||||
|
||||
asker := connect(t, "console", "desk")
|
||||
start := time.Now()
|
||||
d, err := Gather(asker)
|
||||
d, err := Gather(asker, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -109,7 +113,7 @@ func TestDiscoveryDoesNotWaitWhenEveryoneAnswered(t *testing.T) {
|
||||
t.Cleanup(stop)
|
||||
fast.Flush()
|
||||
start := time.Now()
|
||||
d, err := Gather(connect(t, "console", "desk"))
|
||||
d, err := Gather(connect(t, "console", "desk"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
+233
-80
@@ -18,6 +18,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -66,9 +68,20 @@ type Membership struct {
|
||||
Tools string `json:"tools"`
|
||||
// State is every bucket the module's code may reach, by the name it uses (novox/hq ADR 0201).
|
||||
State []StateIssued `json:"state,omitempty"`
|
||||
// Logs is every log the module keeps, by its local name (novox/hq ADR 0297): its own alone.
|
||||
Logs []LogIssued `json:"logs,omitempty"`
|
||||
// SeatTraffic is what the module's code may submit, say, take, ask, answer and read on seats that name
|
||||
// their caller or their kind (novox/hq ADR 0259): the runtime does for it only what is listed here.
|
||||
SeatTraffic *SeatTraffic `json:"seat-traffic,omitempty"`
|
||||
// Streams are the module's own streams (novox/hq ADR 0363 §2): an event one of them carries is published
|
||||
// on that stream's subject, never on EVENTS (streams.go).
|
||||
Streams []StreamIssued `json:"streams,omitempty"`
|
||||
// Hears are the module streams the module hears, each through a durable consumer on that stream the
|
||||
// runtime binds beside the one on EVENTS (ADR 0363 §8).
|
||||
Hears []HeardIssued `json:"hears,omitempty"`
|
||||
// Waits are the event families a session may wait on through the mesh MCP server (novox/hq ADR 0367 §2):
|
||||
// the tool runner's alone, composed from its manifest with the grant that lets it subscribe them.
|
||||
Waits []WaitIssued `json:"waits,omitempty"`
|
||||
}
|
||||
|
||||
// Served is an address a tool is answered on; `{tool}` stands for the tool's name.
|
||||
@@ -119,12 +132,19 @@ func Fatal(err error) string {
|
||||
strings.Contains(msg, "nats: invalid url") {
|
||||
return "the bus address is not a usable URL"
|
||||
}
|
||||
if regexp.MustCompile(`(?i)authorization violation|user authentication expired|permissions violation`).MatchString(msg) {
|
||||
if Refused(err) || regexp.MustCompile(`(?i)user authentication expired|permissions violation`).MatchString(msg) {
|
||||
return "the bus refused this account"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Refused says whether the bus refused the account a connection named (an authorization violation): final for a running
|
||||
// runtime, but at a start possibly only "not yet", since the bus carries a new account after it reloads its users (issue
|
||||
// 1496). Fatal still names it; the caller decides how long a start waits.
|
||||
func Refused(err error) bool {
|
||||
return err != nil && regexp.MustCompile(`(?i)authorization violation`).MatchString(err.Error())
|
||||
}
|
||||
|
||||
func normalizeFingerprint(f string) string {
|
||||
f = strings.TrimSpace(f)
|
||||
if len(f) > 7 && strings.EqualFold(f[:7], "sha256:") {
|
||||
@@ -163,9 +183,12 @@ type Conn struct {
|
||||
cred Credential
|
||||
mu sync.Mutex
|
||||
issued map[string]*Membership // module → membership; present with nil = followed, none issued
|
||||
onNew []func(Membership)
|
||||
subs []*nats.Subscription
|
||||
Logf func(format string, args ...any)
|
||||
// fromGrants is every module whose membership the grants part's state memberships has given (novox/hq ADR 0571
|
||||
// clause 7): its assignments stream copy is then a fallback no longer applied.
|
||||
fromGrants map[string]bool
|
||||
onNew []*func(Membership)
|
||||
subs []*nats.Subscription
|
||||
Logf func(format string, args ...any)
|
||||
// answering is every subject this runtime answers on, so a subscription the bus refused can be
|
||||
// asked for again (novox/hq issue 222).
|
||||
answering map[string]*answered
|
||||
@@ -174,6 +197,8 @@ type Conn struct {
|
||||
RetryAfter []time.Duration
|
||||
// calls are the caller-named calls in flight (caller.go, novox/hq issue 365).
|
||||
calls callers
|
||||
// waits are the waits in flight by subject, each told when the bus refuses its subscription (wait.go).
|
||||
waits map[string]map[chan string]bool
|
||||
}
|
||||
|
||||
// answered is one subject the runtime answers on: what to subscribe again with, and how often it was
|
||||
@@ -211,6 +236,9 @@ func (c *Conn) refused(err error) {
|
||||
if c.publishRefused(err) {
|
||||
return
|
||||
}
|
||||
if subject := subjectRefused(err); subject != "" && c.waitRefused(subject) {
|
||||
return
|
||||
}
|
||||
m := refusedSubject.FindStringSubmatch(err.Error())
|
||||
if len(m) < 2 {
|
||||
return
|
||||
@@ -304,7 +332,7 @@ func Connect(cred Credential) (*Conn, error) {
|
||||
return nil, err
|
||||
}
|
||||
c = &Conn{nc: nc, js: js, self: cred.Module, node: cred.Node, cred: cred,
|
||||
issued: map[string]*Membership{}, Logf: log.Printf, answering: map[string]*answered{}}
|
||||
issued: map[string]*Membership{}, fromGrants: map[string]bool{}, Logf: log.Printf, answering: map[string]*answered{}}
|
||||
c.Follow(cred.Module)
|
||||
return c, nil
|
||||
}
|
||||
@@ -324,7 +352,10 @@ func MembershipSubject(node, module string) string {
|
||||
}
|
||||
|
||||
// Follow reads a module's membership on this machine once and follows it live, so its tools are
|
||||
// served where the mesh issued them (ADR 0175).
|
||||
// served where the mesh issued them (ADR 0175). It is read from the grants part's state memberships, its key
|
||||
// `<node>.<module>` (novox/hq ADR 0571 clause 7, mesh-sdk buscompose.GrantsMembershipsBucket); the controller's
|
||||
// assignments stream is read beside it as the fallback until the lease write, and applied only while the state has given
|
||||
// the module no membership.
|
||||
func (c *Conn) Follow(module string) {
|
||||
c.mu.Lock()
|
||||
if c.node == "" {
|
||||
@@ -337,16 +368,21 @@ func (c *Conn) Follow(module string) {
|
||||
}
|
||||
c.issued[module] = nil
|
||||
c.mu.Unlock()
|
||||
c.followGrants(module)
|
||||
subject := MembershipSubject(c.node, module)
|
||||
// The subject-addressed direct get: the one address the mesh grants this account on the
|
||||
// stream's API.
|
||||
if got, err := c.nc.Request("$JS.API.DIRECT.GET."+assignmentsStream+"."+subject, nil, 5*time.Second); err == nil {
|
||||
if got.Header.Get("Status") == "" && len(got.Data) > 0 {
|
||||
var m Membership
|
||||
if json.Unmarshal(got.Data, &m) == nil {
|
||||
c.mu.Lock()
|
||||
c.issued[module] = &m
|
||||
c.mu.Unlock()
|
||||
if c.Membership(module) == nil {
|
||||
if got, err := c.nc.Request("$JS.API.DIRECT.GET."+assignmentsStream+"."+subject, nil, 5*time.Second); err == nil {
|
||||
if got.Header.Get("Status") == "" && len(got.Data) > 0 {
|
||||
var m Membership
|
||||
if json.Unmarshal(got.Data, &m) == nil {
|
||||
c.mu.Lock()
|
||||
if !c.fromGrants[module] {
|
||||
c.issued[module] = &m
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -359,21 +395,124 @@ func (c *Conn) Follow(module string) {
|
||||
c.Logf("[mesh-tools] a membership arrived that is not one: %v", err)
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.issued[module] = &m
|
||||
handlers := append([]func(Membership){}, c.onNew...)
|
||||
c.mu.Unlock()
|
||||
c.Logf("[mesh-tools] %s on %s was issued a new membership; re-serving on it", module, c.node)
|
||||
for _, h := range handlers {
|
||||
h(m)
|
||||
}
|
||||
_ = c.nc.Flush()
|
||||
c.issue(module, m, false)
|
||||
})
|
||||
if err == nil {
|
||||
c.track(sub)
|
||||
}
|
||||
}
|
||||
|
||||
// GrantsPatience bounds the first read of a membership from the grants part's state.
|
||||
var GrantsPatience = 2 * time.Second
|
||||
|
||||
// issue applies a membership that arrived for module, live, and re-serves on it. One from the assignments stream is
|
||||
// passed over once the grants part's state has given the module one.
|
||||
func (c *Conn) issue(module string, m Membership, fromGrants bool) {
|
||||
c.mu.Lock()
|
||||
if !fromGrants && c.fromGrants[module] {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if fromGrants {
|
||||
c.fromGrants[module] = true
|
||||
}
|
||||
c.issued[module] = &m
|
||||
handlers := make([]func(Membership), 0, len(c.onNew))
|
||||
for _, h := range c.onNew {
|
||||
handlers = append(handlers, *h)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
c.Logf("[mesh-tools] %s on %s was issued a new membership; re-serving on it", module, c.node)
|
||||
for _, h := range handlers {
|
||||
h(m)
|
||||
}
|
||||
_ = c.nc.Flush()
|
||||
}
|
||||
|
||||
// followGrants reads module's key of the grants part's state memberships and watches it: the value there now is applied
|
||||
// before Follow returns, each change after it live. A state that is not there yet, or not readable, is said once, and
|
||||
// the assignments stream answers alone.
|
||||
func (c *Conn) followGrants(module string) {
|
||||
key := buscompose.MembershipKey(c.node, module)
|
||||
// First the key's direct get, which the account is granted alone: a bus that refuses it, or a state not there yet,
|
||||
// answers at once, where binding the bucket would wait out its bound on a refusal.
|
||||
bucket := buscompose.GrantsMembershipsBucket
|
||||
probe, err := c.nc.Request("$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+key, nil, GrantsPatience)
|
||||
if err != nil || probe.Header.Get("Status") == "503" {
|
||||
why := "it did not answer"
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
c.Logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%s); %s's membership is read from the "+
|
||||
"assignments stream alone", bucket, why, module)
|
||||
return
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
c.Logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%v); %s's membership is read from the "+
|
||||
"assignments stream alone", buscompose.GrantsMembershipsBucket, err, module)
|
||||
return
|
||||
}
|
||||
w, err := kv.Watch(context.Background(), key)
|
||||
if err != nil {
|
||||
c.Logf("[mesh-tools] %s's key %s of the grants part's memberships cannot be watched (%v); read from the "+
|
||||
"assignments stream alone", module, key, err)
|
||||
return
|
||||
}
|
||||
apply := func(e jetstream.KeyValueEntry) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return // a membership withdrawn is the stream's, or the module's unassignment's, to say
|
||||
}
|
||||
// Read through mesh-sdk's one codec of this state, the grants part's writer's (issue 774): it takes the bare
|
||||
// membership and a wrapper the part wrote before of one published or unchanged, and refuses a held-back one's
|
||||
// wrapper or a value naming no node or module, which then leaves the assignments stream's copy in force instead.
|
||||
// A refused value arriving after a good one keeps the last good membership: nothing here clears it.
|
||||
var m Membership
|
||||
body, err := membership.BareGrantsMembership(e.Value())
|
||||
if err == nil {
|
||||
err = json.Unmarshal(body, &m)
|
||||
}
|
||||
if err != nil {
|
||||
c.Logf("[mesh-tools] the membership at %s is not one: %v", key, err)
|
||||
return
|
||||
}
|
||||
c.issue(module, m, true)
|
||||
}
|
||||
timer := time.NewTimer(StateTimeout)
|
||||
defer timer.Stop()
|
||||
for current := true; current; {
|
||||
select {
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
current = false
|
||||
continue
|
||||
}
|
||||
apply(e)
|
||||
case <-timer.C:
|
||||
_ = w.Stop()
|
||||
c.Logf("[mesh-tools] %s's key %s of the grants part's memberships did not answer in %s; read from the "+
|
||||
"assignments stream alone", module, key, StateTimeout)
|
||||
return
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
for e := range w.Updates() {
|
||||
if e != nil {
|
||||
apply(e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Membership is what the mesh issued a module here, or nil when nothing has been issued.
|
||||
func (c *Conn) Membership(module string) *Membership {
|
||||
c.mu.Lock()
|
||||
@@ -403,10 +542,28 @@ func (c *Conn) Serving() []string {
|
||||
// OnMembership is called with every new membership any followed module is issued.
|
||||
func (c *Conn) OnMembership(h func(Membership)) {
|
||||
c.mu.Lock()
|
||||
c.onNew = append(c.onNew, h)
|
||||
c.onNew = append(c.onNew, &h)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// onMembership is OnMembership until the answered function is called, which drops the handler.
|
||||
func (c *Conn) onMembership(h func(Membership)) func() {
|
||||
mine := &h
|
||||
c.mu.Lock()
|
||||
c.onNew = append(c.onNew, mine)
|
||||
c.mu.Unlock()
|
||||
return func() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for i, had := range c.onNew {
|
||||
if had == mine {
|
||||
c.onNew = append(c.onNew[:i:i], c.onNew[i+1:]...)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Conn) track(s *nats.Subscription) {
|
||||
c.mu.Lock()
|
||||
c.subs = append(c.subs, s)
|
||||
@@ -566,9 +723,13 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) {
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var stops []func()
|
||||
stopped := false
|
||||
serve := func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if stopped {
|
||||
return
|
||||
}
|
||||
for _, s := range stops {
|
||||
s()
|
||||
}
|
||||
@@ -582,14 +743,18 @@ func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) {
|
||||
}
|
||||
}
|
||||
serve()
|
||||
c.OnMembership(func(m Membership) {
|
||||
// Its stop drops the membership handler too: a handler left behind would bind the stopped tool again
|
||||
// on the module's next membership, and call a handler whose bundle was replaced (issue 1945).
|
||||
unfollow := c.onMembership(func(m Membership) {
|
||||
if m.Module == module {
|
||||
serve()
|
||||
}
|
||||
})
|
||||
return func() {
|
||||
unfollow()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
stopped = true
|
||||
for _, s := range stops {
|
||||
s()
|
||||
}
|
||||
@@ -672,12 +837,16 @@ var HandoverPause = 3 * time.Second
|
||||
// (mesh-controller internal/link, `"retry": "handing-over"`).
|
||||
const RetryHandingOver = "handing-over"
|
||||
|
||||
// ErrNoResponders is the bus's own refusal: nothing subscribes to the subject asked. It is told apart
|
||||
// from a tool's own error by errors.Is, never by its text (a tool may say "503" itself, issue 1529).
|
||||
var ErrNoResponders = errors.New("503 no responders")
|
||||
|
||||
// askOnce asks once, and says whether the answer was a refusal marked to be asked again.
|
||||
func (c *Conn) askOnce(subject string, data []byte) (Answered, bool, error) {
|
||||
msg, err := c.nc.Request(subject, data, RequestTimeout)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return Answered{}, false, errors.New("503 no responders")
|
||||
return Answered{}, false, ErrNoResponders
|
||||
}
|
||||
if errors.Is(err, nats.ErrTimeout) {
|
||||
return Answered{}, false, errors.New("timeout")
|
||||
@@ -713,7 +882,9 @@ func (c *Conn) answerOf(subject string, data []byte) (Answered, bool, error) {
|
||||
// PublishAs emits an event as a module: published into JetStream and awaited, de-duplicated by its
|
||||
// own id (ADR 0042). The body is the payload; the metadata rides as headers.
|
||||
func (c *Conn) PublishAs(module string, env Envelope) error {
|
||||
msg := nats.NewMsg("mesh.mod." + module + ".event." + env.Key)
|
||||
// On the module's own stream when its membership names one carrying the event (ADR 0363 §2), on EVENTS
|
||||
// otherwise — and only there, as before, for a membership that names no stream.
|
||||
msg := nats.NewMsg(eventSubject(c.Membership(module), module, env.Key))
|
||||
for k, v := range env.Headers {
|
||||
msg.Header.Set(k, v)
|
||||
}
|
||||
@@ -773,6 +944,21 @@ type Arrival struct {
|
||||
// that said it is there, not for a fixed moment (2026-10-05). Answers beyond what the caller has
|
||||
// taken are kept up to the channel's room; an empty answer is no answer.
|
||||
func (c *Conn) Collect(subject string, body []byte) (<-chan Arrival, func(), error) {
|
||||
out, ask, stop, err := c.Collector(subject)
|
||||
if err != nil {
|
||||
return nil, func() {}, err
|
||||
}
|
||||
if err := ask(subject, body); err != nil {
|
||||
stop()
|
||||
return nil, func() {}, err
|
||||
}
|
||||
return out, stop, nil
|
||||
}
|
||||
|
||||
// Collector is one inbox for many requests: every answer to every request `ask` publishes arrives on
|
||||
// the one channel, until stopped — for discovery, which asks each service that said it is there for
|
||||
// what it serves, as each says so (novox/hq issue 554). `what` names the requests in a log line.
|
||||
func (c *Conn) Collector(what string) (<-chan Arrival, func(subject string, body []byte) error, func(), error) {
|
||||
inbox := c.nc.NewRespInbox()
|
||||
out := make(chan Arrival, 1024)
|
||||
var mu sync.Mutex
|
||||
@@ -790,11 +976,11 @@ func (c *Conn) Collect(subject string, body []byte) (<-chan Arrival, func(), err
|
||||
select {
|
||||
case out <- a:
|
||||
default:
|
||||
c.Logf("[mesh-tools] more answers to %s than are kept; one (%d bytes) dropped", subject, len(msg.Data))
|
||||
c.Logf("[mesh-tools] more answers to %s than are kept; one (%d bytes) dropped", what, len(msg.Data))
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, func() {}, err
|
||||
return nil, nil, func() {}, err
|
||||
}
|
||||
stop := func() {
|
||||
_ = sub.Unsubscribe()
|
||||
@@ -802,11 +988,8 @@ func (c *Conn) Collect(subject string, body []byte) (<-chan Arrival, func(), err
|
||||
stopped = true
|
||||
mu.Unlock()
|
||||
}
|
||||
if err := c.nc.PublishRequest(subject, inbox, body); err != nil {
|
||||
stop()
|
||||
return nil, func() {}, err
|
||||
}
|
||||
return out, stop, nil
|
||||
ask := func(subject string, body []byte) error { return c.nc.PublishRequest(subject, inbox, body) }
|
||||
return out, ask, stop, nil
|
||||
}
|
||||
|
||||
// MaxPayload is the largest message the bus carries, as the server told this connection.
|
||||
@@ -923,64 +1106,34 @@ var NakDelay = 5 * time.Second
|
||||
// handed over, and a burst would end in max-deliver with nothing failed (novox/hq issue 387).
|
||||
var EventHeartbeat = 10 * time.Second
|
||||
|
||||
// ConsumeAs reads a module's durable consumer and hands each event to deliver (novox/hq ADR 0198):
|
||||
// acknowledged when deliver returns nil, negatively acknowledged after NakDelay when it returns an
|
||||
// error, terminated when it is not an event at all. The consumer is the controller's to create; this
|
||||
// binds to it and never makes one. Runs until stopped.
|
||||
// ConsumeAs reads a module's durable consumer on EVENTS, and each one on a module's stream its
|
||||
// membership names (novox/hq ADR 0363 §8), and hands each event to deliver one at a time (novox/hq
|
||||
// ADR 0198): acknowledged when deliver returns nil, negatively acknowledged after NakDelay when it
|
||||
// returns an error, terminated when it is not an event at all. The consumers are the controller's to
|
||||
// create; this binds to them and never makes one. Runs until stopped.
|
||||
func (c *Conn) ConsumeAs(module string, deliver func(Envelope) error) (func(), error) {
|
||||
durable := ConsumerOf(c.node, module)
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
h := newHearing(c, js, module, deliver)
|
||||
// Bound by name, never created and never matched against a subject: the consumer and its
|
||||
// filters are the controller's (design 29 §3), exactly as the module's own runtime bound it.
|
||||
consumer, err := js.Consumer(ctx, EventsStream, durable)
|
||||
events, _, doing, err := h.bind(EventsStream, durable, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("binding %s's consumer %s: %w", module, durable, err)
|
||||
}
|
||||
reading, err := consumer.Consume(func(msg jetstream.Msg) {
|
||||
env, ok := envelopeOf(msg.Subject(), msg.Headers(), msg.Data())
|
||||
if !ok {
|
||||
// Unparseable: redelivering bytes no version of a handler can read is a loop.
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
// Kept alive while the bundle works on it, as a worker's work is (seat.go): the consumer's ack wait
|
||||
// (30 seconds) is for a runtime that died, not a handler that is slow — the runtime gives a handler
|
||||
// launch.EventTimeout. Without it an event handled longer than the ack wait was offered again while
|
||||
// it was handled, and counted toward max-deliver (novox/hq issue 387).
|
||||
done, stopped := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(stopped)
|
||||
tick := time.NewTicker(EventHeartbeat)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
_ = msg.InProgress()
|
||||
}
|
||||
}
|
||||
}()
|
||||
err := deliver(env)
|
||||
close(done)
|
||||
// No in-progress after the answer: one after a Nak would stretch its delay to the whole ack wait.
|
||||
<-stopped
|
||||
if err != nil {
|
||||
_ = msg.NakWithDelay(NakDelay)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
}, jetstream.PullMaxMessages(1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s's consumer %s: %w", module, durable, err)
|
||||
return nil, fmt.Errorf("%s %s's consumer %s: %w", doing, module, durable, err)
|
||||
}
|
||||
// And every consumer on a module's stream its membership names (novox/hq ADR 0363 §8), followed as the
|
||||
// membership changes. A membership naming none binds nothing more: the module hears EVENTS alone.
|
||||
h.start()
|
||||
var once sync.Once
|
||||
return func() { once.Do(reading.Stop) }, nil
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
h.stop()
|
||||
events.Stop()
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
|
||||
// envelopeOf rebuilds the envelope a module sees from a delivered event: its key is the emitter and
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// heardMsg is a delivered event as a consumer's loop hands it, saying how it was settled.
|
||||
type heardMsg struct {
|
||||
jetstream.Msg
|
||||
mu sync.Mutex
|
||||
settled []string
|
||||
}
|
||||
|
||||
func (m *heardMsg) Subject() string { return "mesh.mod.alpha.event.happened" }
|
||||
func (m *heardMsg) Headers() nats.Header { return nats.Header{} }
|
||||
func (m *heardMsg) Data() []byte { return []byte(`{"n":1}`) }
|
||||
func (m *heardMsg) InProgress() error { return nil }
|
||||
func (m *heardMsg) Ack() error { return m.say("ack") }
|
||||
func (m *heardMsg) Nak() error { return m.say("nak") }
|
||||
func (m *heardMsg) NakWithDelay(time.Duration) error { return m.say("nak-delayed") }
|
||||
func (m *heardMsg) Term() error { return m.say("term") }
|
||||
func (m *heardMsg) say(how string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.settled = append(m.settled, how)
|
||||
return nil
|
||||
}
|
||||
|
||||
// An event the bus hands a hearing after it was stopped (a module's bundle loaded again, issue 1945) is
|
||||
// offered again at once and never handed to the module: the module's handler that heard it may be gone.
|
||||
func TestAHearingStoppedHandsNoEventOnAndOffersItAgain(t *testing.T) {
|
||||
delivered := 0
|
||||
h := newHearing(nil, nil, "alpha", func(Envelope) error { delivered++; return nil })
|
||||
h.stop()
|
||||
msg := &heardMsg{}
|
||||
h.handle(msg)
|
||||
if delivered != 0 {
|
||||
t.Errorf("a stopped hearing handed the event to the module")
|
||||
}
|
||||
if len(msg.settled) != 1 || msg.settled[0] != "nak" {
|
||||
t.Errorf("a stopped hearing settled the event as %v, want it offered again at once (nak)", msg.settled)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,491 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's log on the bus (novox/hq ADR 0297): a stream of operations, one subject per key, that the
|
||||
// controller creates from what the module declared under `logs` and issues to the module's own instances
|
||||
// in its membership. It mirrors a bucket (ADR 0201) in how it is reached: what a module may reach is
|
||||
// decided here, from its membership, and refused with the reason before anything is sent.
|
||||
//
|
||||
// **The runtime uses no consumer of a log.** The bus grants it, per issued log, only the publish of an
|
||||
// entry, the stream's info and the direct get: an append is a publish, a read is a batch direct get
|
||||
// (NATS 2.11), the last entry of a key is a direct get by its subject.
|
||||
|
||||
// LogIssued is one log an assignment may reach (ADR 0297): its owner's alone, so there are no reads.
|
||||
type LogIssued struct {
|
||||
Name string `json:"name"`
|
||||
Stream string `json:"stream"`
|
||||
Subject string `json:"subject"`
|
||||
Writes bool `json:"writes,omitempty"`
|
||||
}
|
||||
|
||||
// LogEntry is one operation in a log: its sequence is its identity.
|
||||
type LogEntry struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// LogPage is one page of a read: Next is the sequence to continue from, 0 when nothing follows; Last
|
||||
// is the stream's last sequence.
|
||||
type LogPage struct {
|
||||
Entries []LogEntry `json:"entries"`
|
||||
Next uint64 `json:"next"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
// LogInfo is how full a log is.
|
||||
type LogInfo struct {
|
||||
Entries uint64 `json:"entries"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
MaxBytes uint64 `json:"max_bytes"`
|
||||
First uint64 `json:"first"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
const (
|
||||
// LogValueMax is the most one entry's value may hold (ADR 0297 §3); the stream's MaxMsgSize is 4 KiB
|
||||
// more, for the headers an append carries.
|
||||
LogValueMax = 256 * 1024
|
||||
// LogKeyMax is the longest key.
|
||||
LogKeyMax = 255
|
||||
// LogReadDefault and LogReadMax are a page's entries when none is asked, and at most.
|
||||
LogReadDefault = 100
|
||||
LogReadMax = 1000
|
||||
// LogPageBytes bounds the bytes of one page.
|
||||
LogPageBytes = 4 * 1024 * 1024
|
||||
)
|
||||
|
||||
// logToken is one name of a key: letters, digits, `_` and `-`.
|
||||
var logToken = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
|
||||
func checkLogKey(key string) error {
|
||||
if key == "" || len(key) > LogKeyMax {
|
||||
return fmt.Errorf("%q is not a log key: one or more names of letters, digits, _ and -, joined by "+
|
||||
"dots, at most %d bytes (novox/hq ADR 0297)", key, LogKeyMax)
|
||||
}
|
||||
for _, t := range strings.Split(key, ".") {
|
||||
if !logToken.MatchString(t) {
|
||||
return fmt.Errorf("%q is not a log key: one or more names of letters, digits, _ and -, joined by "+
|
||||
"dots, at most %d bytes (novox/hq ADR 0297)", key, LogKeyMax)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// logFilter is a read's key pattern as a subject filter under the log's subject: `*` for one name, `>`
|
||||
// for the rest; empty is every key.
|
||||
func logFilter(l LogIssued, pattern string) (string, error) {
|
||||
if pattern == "" {
|
||||
return l.Subject + ".>", nil
|
||||
}
|
||||
parts := strings.Split(pattern, ".")
|
||||
for i, p := range parts {
|
||||
if p == "**" && i == len(parts)-1 {
|
||||
// `**` for the rest, as a state's key pattern and mesh-sdk's log reader say it.
|
||||
parts[i], p = ">", ">"
|
||||
}
|
||||
switch {
|
||||
case p == ">" && i == len(parts)-1, p == "*", logToken.MatchString(p):
|
||||
default:
|
||||
return "", fmt.Errorf("%q is not a key pattern: names, `*` for one and `>` for the rest, last "+
|
||||
"(novox/hq ADR 0297)", pattern)
|
||||
}
|
||||
}
|
||||
if len(pattern) > LogKeyMax {
|
||||
return "", fmt.Errorf("%q is longer than a key can be (%d bytes)", pattern, LogKeyMax)
|
||||
}
|
||||
return l.Subject + "." + strings.Join(parts, "."), nil
|
||||
}
|
||||
|
||||
// issuedLog is the log a module may reach by a name, or the reason it may not.
|
||||
func (c *Conn) issuedLog(module, name string) (LogIssued, error) {
|
||||
m := c.Membership(module)
|
||||
if m == nil {
|
||||
return LogIssued{}, fmt.Errorf("%s has no membership issued on %s yet, so no log of it is reachable "+
|
||||
"until the mesh issues one (novox/hq ADR 0297)", module, c.node)
|
||||
}
|
||||
var names []string
|
||||
for _, l := range m.Logs {
|
||||
if l.Name == name {
|
||||
return l, nil
|
||||
}
|
||||
names = append(names, l.Name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
issued := "none"
|
||||
if len(names) > 0 {
|
||||
issued = strings.Join(names, ", ")
|
||||
}
|
||||
return LogIssued{}, fmt.Errorf("%s keeps no log called %q: a module declares the logs it keeps under `logs`, and "+
|
||||
"another module's it reads under `reads` as <module>.<log>; it was issued: %s (novox/hq ADR 0297, ADR 0571)",
|
||||
module, name, issued)
|
||||
}
|
||||
|
||||
// notOnTheBus says a log is issued and its stream is not there yet.
|
||||
func notOnTheBus(l LogIssued) error {
|
||||
return fmt.Errorf("the log %q is issued and its stream %s is not on the bus yet: the controller creates "+
|
||||
"it from the catalogue on its next raise (novox/hq ADR 0297)", l.Name, l.Stream)
|
||||
}
|
||||
|
||||
// LogAppend writes one entry for a key, as the module, and answers its sequence. With expect, it is
|
||||
// written only when the key's last entry is still at that sequence (0: the key has none); otherwise it
|
||||
// is refused as a lost compare-and-set, the same error a lost state update answers (ADR 0259).
|
||||
func (c *Conn) LogAppend(module, name, key string, value json.RawMessage, expect *uint64) (uint64, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !l.Writes {
|
||||
return 0, fmt.Errorf("%s was issued the log %s without writing it (novox/hq ADR 0297)", module, name)
|
||||
}
|
||||
if err := checkLogKey(key); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(value) == 0 || !json.Valid(value) {
|
||||
return 0, fmt.Errorf("a log entry's value is JSON (novox/hq ADR 0297)")
|
||||
}
|
||||
if len(value) > LogValueMax {
|
||||
return 0, fmt.Errorf("a log entry's value is at most %d bytes (256 KiB), and this one is %d (novox/hq ADR 0297)",
|
||||
LogValueMax, len(value))
|
||||
}
|
||||
if field := credentialField(value); field != "" {
|
||||
return 0, fmt.Errorf("%s's %s.%s carries a field %q, which names a credential: no secret is kept in a log, "+
|
||||
"which keeps it for good. Name the secret and fetch it on request/reply (novox/hq ADR 0201 §13, ADR 0297)",
|
||||
module, name, key, field)
|
||||
}
|
||||
msg := nats.NewMsg(l.Subject + "." + key)
|
||||
msg.Data = value
|
||||
if expect != nil {
|
||||
msg.Header.Set(jetstream.ExpectedLastSubjSeqHeader, strconv.FormatUint(*expect, 10))
|
||||
}
|
||||
reply, err := c.nc.RequestMsg(msg, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return 0, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var ack struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
ErrCode uint16 `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &ack); err != nil {
|
||||
return 0, fmt.Errorf("the bus answered the append to %s with what is not an acknowledgement: %q", name, reply.Data)
|
||||
}
|
||||
if ack.Error != nil {
|
||||
// The key's last entry is no longer the one expected: a lost compare-and-set, answered as a lost
|
||||
// state update is (10164 is the same refusal from a replicated stream).
|
||||
if code := jetstream.ErrorCode(ack.Error.ErrCode); code == jetstream.JSErrCodeStreamWrongLastSequence ||
|
||||
code == jetstream.JSErrCodeStreamWrongLastSequenceConstant {
|
||||
return 0, changed{name + "." + key}
|
||||
}
|
||||
return 0, fmt.Errorf("the bus refused the append to %s.%s: %s (%d)", name, key, ack.Error.Description, ack.Error.ErrCode)
|
||||
}
|
||||
if ack.Seq == 0 {
|
||||
return 0, fmt.Errorf("the bus answered the append to %s.%s with no sequence", name, key)
|
||||
}
|
||||
return ack.Seq, nil
|
||||
}
|
||||
|
||||
// LogRead is the entries from a sequence on, or from a time on (since, when not zero), of the keys matching a pattern, at
|
||||
// most limit of them and about 4 MiB, by one batch direct get. A log another module keeps is read as its own is, by the
|
||||
// name its reads declare, `<module>.<log>` (novox/hq ADR 0571).
|
||||
func (c *Conn) LogRead(module, name string, from uint64, since time.Time, keys string, limit int) (*LogPage, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case limit == 0:
|
||||
limit = LogReadDefault
|
||||
case limit < 0 || limit > LogReadMax:
|
||||
return nil, fmt.Errorf("a read's limit is 1 to %d entries, %d when none is given, not %d (novox/hq ADR 0297)",
|
||||
LogReadMax, LogReadDefault, limit)
|
||||
}
|
||||
filter, err := logFilter(l, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if from != 0 && !since.IsZero() {
|
||||
return nil, fmt.Errorf("a read of %s starts from a sequence or from a time, not both", name)
|
||||
}
|
||||
if from == 0 {
|
||||
from = 1
|
||||
}
|
||||
req := map[string]any{"seq": from, "next_by_subj": filter, "batch": limit, "max_bytes": LogPageBytes}
|
||||
if !since.IsZero() {
|
||||
delete(req, "seq")
|
||||
req["start_time"] = since.UTC()
|
||||
}
|
||||
entries, more, err := c.logBatch(l, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page := &LogPage{Entries: entries}
|
||||
if more && len(entries) > 0 {
|
||||
page.Next = entries[len(entries)-1].Sequence + 1
|
||||
}
|
||||
info, err := c.logInfo(l)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page.Last = info.Last
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// LogNewest is the newest entry of each key matching a pattern, in the log's order, by one batch direct get (mesh-sdk
|
||||
// process.LogReader.Newest).
|
||||
func (c *Conn) LogNewest(module, name, keys string) ([]LogEntry, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter, err := logFilter(l, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, more, err := c.logBatch(l, map[string]any{"multi_last": []string{filter}, "batch": LogReadMax})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if more {
|
||||
return nil, fmt.Errorf("%s holds more than %d keys matching %q: name fewer", name, LogReadMax, keys)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// logBatch is one batch direct get of a log: its entries, and whether more are pending after them.
|
||||
func (c *Conn) logBatch(l LogIssued, request map[string]any) ([]LogEntry, bool, error) {
|
||||
req, _ := json.Marshal(request)
|
||||
inbox := c.nc.NewInbox()
|
||||
sub, err := c.nc.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := c.nc.PublishRequest("$JS.API.DIRECT.GET."+l.Stream, inbox, req); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
entries := []LogEntry{}
|
||||
deadline := time.Now().Add(StateTimeout)
|
||||
for {
|
||||
msg, err := sub.NextMsg(time.Until(deadline))
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, false, notOnTheBus(l)
|
||||
}
|
||||
if errors.Is(err, nats.ErrTimeout) {
|
||||
return nil, false, fmt.Errorf("the read of %s did not end in %s", l.Name, StateTimeout)
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
switch status := msg.Header.Get("Status"); status {
|
||||
case "":
|
||||
e, err := logEntryOf(l, msg)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
entries = append(entries, *e)
|
||||
case "204": // the end of the batch: what is still pending after the last entry sent
|
||||
pending, _ := strconv.ParseUint(msg.Header.Get("Nats-Num-Pending"), 10, 64)
|
||||
return entries, pending > 0, nil
|
||||
case "404": // nothing from there on
|
||||
return entries, false, nil
|
||||
case "503":
|
||||
return nil, false, notOnTheBus(l)
|
||||
default:
|
||||
return nil, false, fmt.Errorf("the read of %s was refused by the bus: %s %s", l.Name, status, msg.Header.Get("Description"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// LogWatch hands deliver every entry of the keys matching a pattern from a sequence on (from), from a time on (since),
|
||||
// or, with neither, each new one, in the log's order, one at a time, through an ordered consumer it lets go of when
|
||||
// stopped (mesh-sdk process.LogReader.Watch). It returns once the entries there at its start are handed over; an error
|
||||
// from deliver is said, and the watch goes on with the next entry.
|
||||
func (c *Conn) LogWatch(module, name, keys string, from uint64, since time.Time, deliver func(LogEntry) error) (func(), error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter, err := logFilter(l, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg := jetstream.OrderedConsumerConfig{FilterSubjects: []string{filter}, DeliverPolicy: jetstream.DeliverNewPolicy}
|
||||
switch {
|
||||
case from != 0 && !since.IsZero():
|
||||
return nil, fmt.Errorf("a watch of %s starts from a sequence or from a time, not both", name)
|
||||
case from != 0:
|
||||
cfg.DeliverPolicy, cfg.OptStartSeq = jetstream.DeliverByStartSequencePolicy, from
|
||||
case !since.IsZero():
|
||||
at := since.UTC()
|
||||
cfg.DeliverPolicy, cfg.OptStartTime = jetstream.DeliverByStartTimePolicy, &at
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
cons, err := js.OrderedConsumer(ctx, l.Stream, cfg)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watch of %s (stream %s): %w", name, l.Stream, err)
|
||||
}
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watch of %s (stream %s): %w", name, l.Stream, err)
|
||||
}
|
||||
it, err := cons.Messages()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watch of %s (stream %s): %w", name, l.Stream, err)
|
||||
}
|
||||
hand := func(m jetstream.Msg) error {
|
||||
meta, err := m.Metadata()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e := LogEntry{Sequence: meta.Sequence.Stream, Key: strings.TrimPrefix(m.Subject(), l.Subject+"."),
|
||||
Value: valueOf(m.Data()), At: meta.Timestamp.UTC()}
|
||||
return deliver(e)
|
||||
}
|
||||
for pending := info.NumPending; pending > 0; pending-- {
|
||||
m, err := it.Next()
|
||||
if err == nil {
|
||||
err = hand(m)
|
||||
}
|
||||
if err != nil {
|
||||
it.Stop()
|
||||
return nil, fmt.Errorf("the watch of %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
var stopped atomic.Bool
|
||||
go func() {
|
||||
for {
|
||||
m, err := it.Next()
|
||||
if err != nil || stopped.Load() {
|
||||
return // stopped, or the connection closed; a message fetched as it stopped is not handed over
|
||||
}
|
||||
if err := hand(m); err != nil {
|
||||
c.Logf("[mesh-tools] %s did not take entry of %s: %v; it is in the log, a read finds it", module, name, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { stopped.Store(true); it.Stop() }, nil
|
||||
}
|
||||
|
||||
// LogLast is a key's last entry, or nil when it has none.
|
||||
func (c *Conn) LogLast(module, name, key string) (*LogEntry, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkLogKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := c.nc.Request("$JS.API.DIRECT.GET."+l.Stream+"."+l.Subject+"."+key, nil, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch status := reply.Header.Get("Status"); status {
|
||||
case "":
|
||||
return logEntryOf(l, reply)
|
||||
case "404":
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("the last entry of %s.%s could not be read: %s %s", name, key, status, reply.Header.Get("Description"))
|
||||
}
|
||||
}
|
||||
|
||||
// LogInfo is a log's entries, bytes, cap and first and last sequences.
|
||||
func (c *Conn) LogInfo(module, name string) (*LogInfo, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.logInfo(l)
|
||||
}
|
||||
|
||||
func (c *Conn) logInfo(l LogIssued) (*LogInfo, error) {
|
||||
reply, err := c.nc.Request("$JS.API.STREAM.INFO."+l.Stream, nil, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var info struct {
|
||||
Config struct {
|
||||
MaxBytes int64 `json:"max_bytes"`
|
||||
} `json:"config"`
|
||||
State struct {
|
||||
Msgs uint64 `json:"messages"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
} `json:"state"`
|
||||
Error *struct {
|
||||
ErrCode uint16 `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &info); err != nil {
|
||||
return nil, fmt.Errorf("the bus answered the info of %s with what is not one: %q", l.Name, reply.Data)
|
||||
}
|
||||
if info.Error != nil {
|
||||
if info.Error.ErrCode == uint16(jetstream.JSErrCodeStreamNotFound) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
return nil, fmt.Errorf("the info of the log %s was refused: %s (%d)", l.Name, info.Error.Description, info.Error.ErrCode)
|
||||
}
|
||||
var maxBytes uint64
|
||||
if info.Config.MaxBytes > 0 {
|
||||
maxBytes = uint64(info.Config.MaxBytes)
|
||||
}
|
||||
return &LogInfo{Entries: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: maxBytes,
|
||||
First: info.State.FirstSeq, Last: info.State.LastSeq}, nil
|
||||
}
|
||||
|
||||
// logEntryOf is an entry a direct get answered: its key is its subject under the log's.
|
||||
func logEntryOf(l LogIssued, msg *nats.Msg) (*LogEntry, error) {
|
||||
subject := msg.Header.Get("Nats-Subject")
|
||||
key, ok := strings.CutPrefix(subject, l.Subject+".")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the bus answered an entry of %s on %q, which is not under %s", l.Name, subject, l.Subject)
|
||||
}
|
||||
seq, err := strconv.ParseUint(msg.Header.Get("Nats-Sequence"), 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus answered an entry of %s with no sequence", l.Name)
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339Nano, msg.Header.Get("Nats-Time-Stamp"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus answered entry %d of %s with no time: %v", seq, l.Name, err)
|
||||
}
|
||||
return &LogEntry{Sequence: seq, Key: key, Value: valueOf(msg.Data), At: at.UTC()}, nil
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A module's code asks its machine's provisioner client for its values at runtime (novox/hq ADR 0349 clause 14 as
|
||||
// ADR 0370 changes it, issue 685).
|
||||
//
|
||||
// While a module's code runs here, its asks reach the bus under this runtime's account, and the bus proves only
|
||||
// that account. So the runtime asks its own machine's client on ProvisionAskSubject — which the controller grants
|
||||
// this machine's tool runner alone to publish (ADR 0370 §3) — naming the module itself, from the bundle that asks
|
||||
// and never from the module's code or arguments (§4). The client answers that module's config values, for what its
|
||||
// manifest declares, and no secret: the vault hands a secret only to a caller the bus proves (§2).
|
||||
|
||||
// ProvisionAskSubject is where this machine's tool runner asks its provisioner client (mesh-controller
|
||||
// internal/broker ProvisionerAskSubject): `prov.ask.<node>`.
|
||||
func ProvisionAskSubject(node string) string { return "prov.ask." + node }
|
||||
|
||||
// ProvisionChangedSubjects is every module's word of what moved on a machine (mesh-controller internal/broker
|
||||
// ProvisionerChangedSubject): `prov.changed.<node>.<module>`, a module's name dots and all.
|
||||
func ProvisionChangedSubjects(node string) string { return "prov.changed." + node + ".>" }
|
||||
|
||||
// ProvisionTimeout bounds one ask of the client; a variable so a test need not wait.
|
||||
var ProvisionTimeout = 10 * time.Second
|
||||
|
||||
// ProvisionChanged is what the client says moved of one module: the kind, and the key, or every key of the kind when
|
||||
// empty. Never the value.
|
||||
type ProvisionChanged struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// ProvisionRefused is the client's refusal, in its own words: a key the module does not declare, a module not placed
|
||||
// on the machine, a secret, a kind it does not answer yet.
|
||||
type ProvisionRefused struct{ Reason string }
|
||||
|
||||
func (r *ProvisionRefused) Error() string { return r.Reason }
|
||||
|
||||
// AskProvisioner asks this machine's provisioner client for one value of module: kind `config` or `bound`, and the
|
||||
// key. The module is the caller's to name from the bundle it runs; this never takes it from what a bundle sent.
|
||||
func (c *Conn) AskProvisioner(module, kind, key string) (json.RawMessage, error) {
|
||||
if c.node == "" {
|
||||
return nil, errors.New("this runtime names no machine, so it has no provisioner client to ask")
|
||||
}
|
||||
body, err := json.Marshal(map[string]string{"module": module, "kind": kind, "key": key})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
msg, err := c.nc.Request(ProvisionAskSubject(c.node), body, ProvisionTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, fmt.Errorf("%s's provisioner client does not answer: it is not running there, or the bus does "+
|
||||
"not let %s ask it", c.node, c.cred.User)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("asking %s's provisioner client for %s's %s %s: %w", c.node, module, kind, key, err)
|
||||
}
|
||||
var answered struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &answered); err != nil {
|
||||
return nil, fmt.Errorf("%s's provisioner client answered something that is not its answer: %w", c.node, err)
|
||||
}
|
||||
if answered.Error != "" {
|
||||
return nil, &ProvisionRefused{Reason: answered.Error}
|
||||
}
|
||||
if len(answered.Value) == 0 {
|
||||
return nil, fmt.Errorf("%s's provisioner client answered no value and no reason", c.node)
|
||||
}
|
||||
return answered.Value, nil
|
||||
}
|
||||
|
||||
// OnProvisionChanged hands told every word of this machine's provisioner client of what moved, by module, until
|
||||
// stopped.
|
||||
func (c *Conn) OnProvisionChanged(told func(module string, changed ProvisionChanged)) (func(), error) {
|
||||
if c.node == "" {
|
||||
return nil, errors.New("this runtime names no machine, so it hears no provisioner client")
|
||||
}
|
||||
prefix := "prov.changed." + c.node + "."
|
||||
sub, err := c.nc.Subscribe(ProvisionChangedSubjects(c.node), func(m *nats.Msg) {
|
||||
var changed ProvisionChanged
|
||||
if json.Unmarshal(m.Data, &changed) != nil || changed.Kind == "" {
|
||||
return
|
||||
}
|
||||
told(strings.TrimPrefix(m.Subject, prefix), changed)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() { _ = sub.Unsubscribe() }, c.nc.Flush()
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// **A holder that pulls one piece of work when it has room** (novox/hq ADR 0525 §4, §6; issue 1887 A11). SeatTake
|
||||
// hands a bundle every piece a worker holds, one at a time and prefetched; a build module must instead take one ask
|
||||
// only when it has a free slot, from its workers in their order (delivery, then check, then recheck), and decide each
|
||||
// ask's fate when its step ends. SeatPull is one fetch of at most one message from a worker the module's membership
|
||||
// lists, never a prefetch; SeatSettle gives a pulled message its fate. A message the bundle never settles (its process
|
||||
// died) is delivered again by the worker's own acknowledgement wait.
|
||||
|
||||
// MaxPullWait bounds how long one pull waits for a message: under the bundle's 30 s wait for its answer.
|
||||
var MaxPullWait = 25 * time.Second
|
||||
|
||||
// Pulled is one message a module pulled from a seat's worker, named by an id the runtime gave it.
|
||||
type Pulled struct {
|
||||
ID string `json:"id"`
|
||||
Worker string `json:"worker"`
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
Delivered uint64 `json:"delivered"`
|
||||
}
|
||||
|
||||
// pulls are the messages modules pulled and have not settled, by id; each kept with the module it was pulled for, so
|
||||
// one module never settles another's.
|
||||
type pulls struct {
|
||||
mu sync.Mutex
|
||||
held map[string]pulledMsg
|
||||
cons map[string]jetstream.Consumer // by module and consumer
|
||||
}
|
||||
|
||||
type pulledMsg struct {
|
||||
module string
|
||||
msg jetstream.Msg
|
||||
}
|
||||
|
||||
var seatPulls = &pulls{held: map[string]pulledMsg{}, cons: map[string]jetstream.Consumer{}}
|
||||
|
||||
func pullID() string {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// SeatPull fetches at most one message from consumer for module, waiting at most wait (none when 0): nil when none
|
||||
// came. The consumer must be one the module's membership lists as its worker (novox/hq ADR 0259); the runtime binds to
|
||||
// it by name and never makes one.
|
||||
func (c *Conn) SeatPull(module, consumer string, wait time.Duration) (*Pulled, error) {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var w *SeatWorker
|
||||
var names []string
|
||||
for i := range t.Workers {
|
||||
names = append(names, t.Workers[i].Consumer)
|
||||
if t.Workers[i].Consumer == consumer {
|
||||
w = &t.Workers[i]
|
||||
}
|
||||
}
|
||||
if w == nil {
|
||||
return nil, fmt.Errorf("%s pulls no work from %s: its membership lists %s (novox/hq ADR 0259)", module,
|
||||
consumer, strings.Join(names, ", "))
|
||||
}
|
||||
if wait > MaxPullWait {
|
||||
wait = MaxPullWait
|
||||
}
|
||||
bound, err := c.pullConsumer(module, *w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var batch jetstream.MessageBatch
|
||||
if wait <= 0 {
|
||||
batch, err = bound.FetchNoWait(1)
|
||||
} else {
|
||||
batch, err = bound.Fetch(1, jetstream.FetchMaxWait(wait))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pulling %s's worker %s: %w", module, w.Consumer, err)
|
||||
}
|
||||
msg, ok := <-batch.Messages()
|
||||
if !ok {
|
||||
if err := batch.Error(); err != nil {
|
||||
return nil, fmt.Errorf("pulling %s's worker %s: %w", module, w.Consumer, err)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if !json.Valid(msg.Data()) {
|
||||
_ = msg.Term()
|
||||
return nil, nil
|
||||
}
|
||||
headers := map[string]string{}
|
||||
for k := range msg.Headers() {
|
||||
headers[k] = msg.Headers().Get(k)
|
||||
}
|
||||
delivered := uint64(1)
|
||||
if meta, merr := msg.Metadata(); merr == nil && meta != nil {
|
||||
delivered = meta.NumDelivered
|
||||
}
|
||||
p := &Pulled{ID: pullID(), Worker: w.Consumer, Subject: msg.Subject(), Body: json.RawMessage(msg.Data()),
|
||||
Headers: headers, Delivered: delivered}
|
||||
seatPulls.mu.Lock()
|
||||
seatPulls.held[p.ID] = pulledMsg{module: module, msg: msg}
|
||||
seatPulls.mu.Unlock()
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (c *Conn) pullConsumer(module string, w SeatWorker) (jetstream.Consumer, error) {
|
||||
key := module + " " + w.Consumer
|
||||
seatPulls.mu.Lock()
|
||||
bound, ok := seatPulls.cons[key]
|
||||
seatPulls.mu.Unlock()
|
||||
if ok {
|
||||
return bound, nil
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
bound, err = js.Consumer(ctx, w.Stream, w.Consumer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("binding %s's worker %s on %s: %w", module, w.Consumer, w.Stream, err)
|
||||
}
|
||||
seatPulls.mu.Lock()
|
||||
seatPulls.cons[key] = bound
|
||||
seatPulls.mu.Unlock()
|
||||
return bound, nil
|
||||
}
|
||||
|
||||
// The fates SeatSettle gives a pulled message.
|
||||
const (
|
||||
SettleAck = "ack" // done: never delivered again
|
||||
SettleNak = "nak" // handed back, to be delivered again at once
|
||||
SettleTerm = "term" // dropped
|
||||
SettleProgress = "progress" // still being worked: its acknowledgement wait starts again
|
||||
)
|
||||
|
||||
// SeatSettle gives the message module pulled under id its fate. A message pulled for another module, or settled
|
||||
// already, is refused.
|
||||
func (c *Conn) SeatSettle(module, id, fate string) error {
|
||||
seatPulls.mu.Lock()
|
||||
p, ok := seatPulls.held[id]
|
||||
if ok && p.module != module {
|
||||
ok = false
|
||||
}
|
||||
if ok && fate != SettleProgress {
|
||||
delete(seatPulls.held, id)
|
||||
}
|
||||
seatPulls.mu.Unlock()
|
||||
if !ok {
|
||||
return fmt.Errorf("%s holds no pulled message %s", module, id)
|
||||
}
|
||||
switch fate {
|
||||
case SettleAck:
|
||||
return p.msg.Ack()
|
||||
case SettleNak:
|
||||
return p.msg.Nak()
|
||||
case SettleTerm:
|
||||
return p.msg.Term()
|
||||
case SettleProgress:
|
||||
return p.msg.InProgress()
|
||||
}
|
||||
seatPulls.mu.Lock()
|
||||
seatPulls.held[id] = p // an unknown fate settles nothing
|
||||
seatPulls.mu.Unlock()
|
||||
return fmt.Errorf("a pulled message's fate is ack, nak, term or progress, not %q", fate)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// 1887/A11: a holder pulls one message at a time, only from a worker its membership lists, never a prefetch; it settles
|
||||
// only what it pulled, and a message handed back comes again.
|
||||
func TestAHolderPullsOneMessageWhenItHasRoom(t *testing.T) {
|
||||
_, conn, js := seatMesh(t)
|
||||
for _, id := range []string{"a1", "a2"} {
|
||||
if _, err := conn.SeatPublish("mesh-delivery", "mesh.seat.operator-channel.accept.ask.mesh-delivery",
|
||||
json.RawMessage(`{"id":"`+id+`"}`), id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
first, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 0)
|
||||
if err != nil || first == nil || string(first.Body) != `{"id":"a1"}` || first.Delivered != 1 {
|
||||
t.Fatalf("the first pull = %+v, %v", first, err)
|
||||
}
|
||||
// One pulled, one still waiting on the stream: nothing was fetched ahead.
|
||||
info, err := js.ConsumerInfo("SEAT_OPERATOR_CHANNEL", "SEAT_OPERATOR_CHANNEL_worker")
|
||||
if err != nil || info.NumPending != 1 || info.NumAckPending != 1 {
|
||||
t.Fatalf("after one pull: pending %d, in hand %d (%v); want 1 and 1", info.NumPending, info.NumAckPending, err)
|
||||
}
|
||||
second, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 0)
|
||||
if err != nil || second == nil || string(second.Body) != `{"id":"a2"}` {
|
||||
t.Fatalf("the second pull = %+v, %v", second, err)
|
||||
}
|
||||
if none, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 0); err != nil || none != nil {
|
||||
t.Fatalf("an empty worker answered %+v, %v", none, err)
|
||||
}
|
||||
began := time.Now()
|
||||
if none, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 200*time.Millisecond); err != nil || none != nil || time.Since(began) < 150*time.Millisecond {
|
||||
t.Fatalf("a waiting pull answered %+v, %v after %v", none, err, time.Since(began))
|
||||
}
|
||||
if err := conn.SeatSettle("telegram", first.ID, bus.SettleAck); err == nil {
|
||||
t.Error("a module settled another's message")
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", first.ID, bus.SettleProgress); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", first.ID, bus.SettleAck); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", first.ID, bus.SettleAck); err == nil {
|
||||
t.Error("a message was settled twice")
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", second.ID, bus.SettleNak); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", time.Second)
|
||||
if err != nil || again == nil || string(again.Body) != `{"id":"a2"}` || again.Delivered != 2 {
|
||||
t.Fatalf("the message handed back = %+v, %v", again, err)
|
||||
}
|
||||
if _, err := conn.SeatPull("telegram", "SEAT_OPERATOR_CHANNEL_worker", 0); err == nil || !strings.Contains(err.Error(), "membership lists") {
|
||||
t.Errorf("a channel pulled the router's work: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,8 @@ type StateEntry struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// StateChange is one change a watch delivers: a key put or deleted.
|
||||
// StateChange is one change a watch delivers: a key put, deleted, or expired (its own time to live, or the
|
||||
// bucket's age, ended; issue 1952).
|
||||
type StateChange struct {
|
||||
State string `json:"state"`
|
||||
Key string `json:"key"`
|
||||
@@ -129,6 +130,95 @@ func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) {
|
||||
return &StateEntry{Key: e.Key(), Value: valueOf(e.Value()), Revision: e.Revision()}, nil
|
||||
}
|
||||
|
||||
// StateKept is one value a bucket still keeps for a key: a put, or a delete (a purge reads as one).
|
||||
type StateKept struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Op string `json:"op"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// StateHistory is every value the bucket still keeps for one key, oldest first, or none (novox/hq ADR
|
||||
// 0307). It is a read like StateGet: served to the owner and to every reader, and writes nothing.
|
||||
func (c *Conn) StateHistory(module, name, key string) ([]StateKept, error) {
|
||||
s, err := c.issuedState(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := kv.History(ctx, key)
|
||||
kept := []StateKept{}
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return kept, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
k := StateKept{Key: e.Key(), Revision: e.Revision(), Op: "delete", At: e.Created().UTC()}
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
k.Op = "put"
|
||||
k.Value = valueOf(e.Value())
|
||||
}
|
||||
kept = append(kept, k)
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// StateInfo is how full a state's bucket is against its caps (tools-are-how-a-module-is-spoken-to): a bundle's tools
|
||||
// say fill against cap from it.
|
||||
type StateInfo struct {
|
||||
State string `json:"state"`
|
||||
Bucket string `json:"bucket"`
|
||||
// Bytes is what the bucket holds now, every kept value and marker counted; Keys how many keys it holds, a key
|
||||
// whose last value is a delete or expiry marker counted until the marker goes.
|
||||
Bytes uint64 `json:"bytes"`
|
||||
Keys uint64 `json:"keys"`
|
||||
// MaxBytes and MaxValueBytes are the bucket's caps; -1 is none.
|
||||
MaxBytes int64 `json:"max_bytes"`
|
||||
MaxValueBytes int32 `json:"max_value_bytes"`
|
||||
// History is how many values a key keeps; TTLSeconds the bucket's age for a value (0 none); KeyTTL whether a key may
|
||||
// carry its own time to live (issue 1952).
|
||||
History int64 `json:"history"`
|
||||
TTLSeconds int64 `json:"ttl_seconds"`
|
||||
KeyTTL bool `json:"key_ttl"`
|
||||
}
|
||||
|
||||
// StateInfo reads a state's bucket's fill and caps. It is a read like StateGet: served to the owner and every reader.
|
||||
func (c *Conn) StateInfo(module, name string) (*StateInfo, error) {
|
||||
s, err := c.issuedState(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
status, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b, ok := status.(*jetstream.KeyValueBucketStatus)
|
||||
if !ok || b.StreamInfo() == nil {
|
||||
return nil, fmt.Errorf("the bus answered no stream behind %s's state %s", module, name)
|
||||
}
|
||||
info := b.StreamInfo()
|
||||
return &StateInfo{State: name, Bucket: status.Bucket(), Bytes: info.State.Bytes, Keys: uint64(info.State.NumSubjects),
|
||||
MaxBytes: info.Config.MaxBytes, MaxValueBytes: info.Config.MaxMsgSize, History: status.History(),
|
||||
TTLSeconds: int64(status.TTL() / time.Second), KeyTTL: info.Config.AllowMsgTTL && status.LimitMarkerTTL() > 0}, nil
|
||||
}
|
||||
|
||||
// StatePut writes one key, as the module, where the module keeps the state. It answers the revision.
|
||||
func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64, error) {
|
||||
s, err := c.writable(module, name)
|
||||
@@ -155,6 +245,61 @@ func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64
|
||||
return kv.Put(ctx, key, value)
|
||||
}
|
||||
|
||||
// MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk
|
||||
// process.MinTTL).
|
||||
const MinKeyTTL = time.Second
|
||||
|
||||
// StateCreateTTL writes one key only when it has no value, with its own time to live (novox/hq ADR 0564 clauses 4 and
|
||||
// 7, issue 1952; mesh-sdk process.State.CreateTTL): the bus removes it when ttl passes and every watch hears the
|
||||
// removal as an expiry, so a bundle's wake needs no timer. A key that has a value is refused as ErrStateChanged. The
|
||||
// bucket must take keys with their own time to live (a state declared `key-ttl`); one that does not is refused, naming
|
||||
// it, before anything is written.
|
||||
func (c *Conn) StateCreateTTL(module, name, key string, value json.RawMessage, ttl time.Duration) (uint64, error) {
|
||||
if ttl < MinKeyTTL {
|
||||
return 0, fmt.Errorf("%s's state %s, key %s: a time to live of %s is shorter than the bus keeps (%s)",
|
||||
module, name, key, ttl, MinKeyTTL)
|
||||
}
|
||||
return c.stateWrite(module, name, key, value, func(ctx context.Context, kv jetstream.KeyValue) (uint64, error) {
|
||||
if err := takesKeyTTL(ctx, kv, module, name); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return kv.Create(ctx, key, value, jetstream.KeyTTL(ttl))
|
||||
})
|
||||
}
|
||||
|
||||
// StateRenewTTL writes one key whole with a fresh time to live, whatever it held (mesh-sdk process.State.RenewTTL): a
|
||||
// time to live is set only when a key is created, so a live key is deleted first, and its watches hear the delete and
|
||||
// then the new value.
|
||||
func (c *Conn) StateRenewTTL(module, name, key string, value json.RawMessage, ttl time.Duration) (uint64, error) {
|
||||
rev, err := c.StateCreateTTL(module, name, key, value, ttl)
|
||||
if !errors.Is(err, ErrStateChanged) {
|
||||
return rev, err
|
||||
}
|
||||
if err := c.StateDelete(module, name, key); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.StateCreateTTL(module, name, key, value, ttl)
|
||||
}
|
||||
|
||||
// takesKeyTTL refuses a bucket that does not allow a time to live per key, or leaves no marker when one ends: there the
|
||||
// server would refuse the write, or the key would vanish and no watch would hear it.
|
||||
func takesKeyTTL(ctx context.Context, kv jetstream.KeyValue, module, name string) error {
|
||||
status, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allows := status.LimitMarkerTTL() > 0
|
||||
if b, ok := status.(*jetstream.KeyValueBucketStatus); ok && b.StreamInfo() != nil {
|
||||
allows = allows && b.StreamInfo().Config.AllowMsgTTL
|
||||
}
|
||||
if !allows {
|
||||
return fmt.Errorf("%s's state %s does not take keys with their own time to live: its bucket %s allows none, or "+
|
||||
"leaves no marker a watch hears when one ends; %s declares the state `key-ttl` for it (novox/hq ADR 0564, "+
|
||||
"issue 1952)", module, name, status.Bucket(), module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StateDelete removes one key, as the module, where the module keeps the state. A key that was not
|
||||
// there is not an error: what is asked for is that it is gone.
|
||||
func (c *Conn) StateDelete(module, name, key string) error {
|
||||
@@ -217,7 +362,12 @@ func (c *Conn) writable(module, name string) (StateIssued, error) {
|
||||
|
||||
// StateWatch hands deliver the current value of every key matching the pattern — none that is
|
||||
// deleted — and then every change, in order (ADR 0201). It returns once the current values are
|
||||
// delivered; deliver is called from one goroutine, one change at a time, and an error from it is said
|
||||
// delivered. **StateTimeout bounds the time without progress, not the whole set** (novox/hq issue
|
||||
// 1726): each current value is a round trip to the bundle, so a state of a few hundred keys outlasts
|
||||
// any fixed bound for all of them; the watch then failed at every attempt, the bundle dropped its
|
||||
// handler, and the newest keys, handed over last, were never taken (the checker's owed, whose newest
|
||||
// verdicts never reached mesh/check). It fails only when no current value arrives, or one is not
|
||||
// taken, within StateTimeout; deliver is called from one goroutine, one change at a time, and an error from it is said
|
||||
// and the watch goes on: state is not a queue, and the next change, or the next start, reads it again.
|
||||
// The pattern is a key, with `*` for one name and `**` for the rest; empty is every key.
|
||||
func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange) error) (stop func(), err error) {
|
||||
@@ -262,9 +412,23 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
|
||||
})
|
||||
}
|
||||
current := make(chan struct{})
|
||||
// handed says a current value was handed over or passed by: progress, which restarts the bound.
|
||||
handed := make(chan struct{}, 1)
|
||||
progress := func() {
|
||||
select {
|
||||
case handed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
initial := true
|
||||
for e := range w.Updates() {
|
||||
// A watch given up, or stopped, hands over nothing more, though values stay buffered in the
|
||||
// watcher: they would reach a watch id the bundle no longer holds (issue 774: 1257 orphan
|
||||
// changes after 51 failed watches).
|
||||
if watching.Err() != nil {
|
||||
break
|
||||
}
|
||||
if e == nil {
|
||||
// The end of what was there when the watch began.
|
||||
if initial {
|
||||
@@ -273,6 +437,11 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
|
||||
}
|
||||
continue
|
||||
}
|
||||
if initial {
|
||||
// Arrival from the bus is progress before the bundle is asked: the bound measures the current
|
||||
// values coming in, and restarts when the bundle has taken one.
|
||||
progress()
|
||||
}
|
||||
change := StateChange{State: name, Key: e.Key(), Revision: e.Revision(), Current: initial}
|
||||
switch e.Operation() {
|
||||
case jetstream.KeyValuePut:
|
||||
@@ -282,25 +451,50 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
|
||||
// A deletion among the current values is a key that is not there: not handed over
|
||||
// (measured, research 024 — the server sends its marker among the initial values).
|
||||
if initial {
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
change.Op = "delete"
|
||||
if e.Operation() == jetstream.KeyValuePurge {
|
||||
// The marker the server leaves when a key's time to live ends (Nats-Marker-Reason MaxAge,
|
||||
// read by the client as a purge): nothing else purges a module's state, since neither the
|
||||
// runtime nor mesh-sdk's State offers a purge (issue 1952).
|
||||
change.Op = "expire"
|
||||
}
|
||||
}
|
||||
if watching.Err() != nil {
|
||||
break
|
||||
}
|
||||
if err := deliver(change); err != nil {
|
||||
c.Logf("[mesh-tools] %s did not take %s %s of %s: %v; its next change, or its next start, reads it again",
|
||||
module, change.Op, change.Key, name, err)
|
||||
}
|
||||
if initial {
|
||||
progress()
|
||||
}
|
||||
}
|
||||
if initial {
|
||||
close(current)
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-current:
|
||||
return stop, nil
|
||||
case <-time.After(StateTimeout):
|
||||
stop()
|
||||
return nil, fmt.Errorf("the current values of %s did not arrive in %s", name, StateTimeout)
|
||||
bound := time.NewTimer(StateTimeout)
|
||||
defer bound.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-current:
|
||||
return stop, nil
|
||||
case <-handed:
|
||||
if !bound.Stop() {
|
||||
select {
|
||||
case <-bound.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
bound.Reset(StateTimeout)
|
||||
case <-bound.C:
|
||||
stop()
|
||||
return nil, fmt.Errorf("the current values of %s did not arrive in %s: none was handed over in that time", name, StateTimeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,7 +512,10 @@ func valueOf(raw []byte) json.RawMessage {
|
||||
var credentialEndings = []string{"password", "passwd", "secret", "token", "credential", "credentials",
|
||||
"authorization", "apikey", "privatekey", "accesskey", "cookie"}
|
||||
|
||||
// credentialField is the first field anywhere in a JSON value whose name says it is a credential, or "".
|
||||
// credentialField is the first field anywhere in a JSON value whose name says it is a credential, or "". The one shape a
|
||||
// credential-shaped name may hold is a provider reference, an object of exactly `node` and `module`, both strings: a
|
||||
// placement's binding of the provision "secret" to the vault (issue 1126) names who provides it, never its value. Any
|
||||
// other shape under such a name is refused.
|
||||
func credentialField(value json.RawMessage) string {
|
||||
var v any
|
||||
if json.Unmarshal(value, &v) != nil {
|
||||
@@ -336,7 +533,7 @@ func credentialField(value json.RawMessage) string {
|
||||
for _, k := range keys {
|
||||
norm := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(k))
|
||||
for _, end := range credentialEndings {
|
||||
if strings.HasSuffix(norm, end) {
|
||||
if strings.HasSuffix(norm, end) && !providerReference(t[k]) {
|
||||
return k
|
||||
}
|
||||
}
|
||||
@@ -355,3 +552,17 @@ func credentialField(value json.RawMessage) string {
|
||||
}
|
||||
return walk(v)
|
||||
}
|
||||
|
||||
// providerReference says whether v is exactly {"node": <name>, "module": <name>}, each a name as mesh-sdk names them.
|
||||
func providerReference(v any) bool {
|
||||
m, ok := v.(map[string]any)
|
||||
if !ok || len(m) != 2 {
|
||||
return false
|
||||
}
|
||||
node, okN := m["node"].(string)
|
||||
module, okM := m["module"].(string)
|
||||
return okN && okM && referenceName.MatchString(node) && referenceName.MatchString(module)
|
||||
}
|
||||
|
||||
// referenceName is a node's or a module's name as mesh-sdk names them: lower case, digits and dashes.
|
||||
var referenceName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
@@ -15,6 +15,14 @@ func TestACredentialNamedFieldIsFoundAnywhereInAValue(t *testing.T) {
|
||||
`{"clientSecret":"s"}`: "clientSecret",
|
||||
`{"maxTokens":4096,"name":"a","generation":3}`: "",
|
||||
`"just a string"`: "",
|
||||
// Issue 1126: a placement binds the provision "secret" to the vault; only an exact provider reference passes.
|
||||
`{"bindings":{"secret":{"node":"anchor","module":"mesh-vault"}}}`: "",
|
||||
`{"secret":{"value":"s"}}`: "secret",
|
||||
`{"secret":{"node":"anchor","module":"mesh-vault","value":"s"}}`: "secret",
|
||||
`{"secret":{"node":"anchor","module":7}}`: "secret",
|
||||
`{"secret":{"node":"anchor","module":"hunter2 Bearer s3cr3t"}}`: "secret",
|
||||
`{"secret":["s"]}`: "secret",
|
||||
`{"bindings":{"secret":{"token":"s"}}}`: "secret",
|
||||
} {
|
||||
if got := credentialField(json.RawMessage(value)); got != want {
|
||||
t.Errorf("%s: found %q, want %q", value, got, want)
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
//go:build integration
|
||||
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// **A watch whose current values take longer than the bound to hand over still holds** (novox/hq issue
|
||||
// 1726). The gitea module watches the checker's owed: one key per pull request head, each a round trip
|
||||
// to the bundle. With a few hundred keys the hand-over outlasted StateTimeout at every attempt, the
|
||||
// watch was refused, and the newest keys, handed over last, never reached mesh/check: a group's passing
|
||||
// verdict was never written on its members' heads. The bound is on progress: a watch fails only when
|
||||
// no current value is handed over within it.
|
||||
func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) {
|
||||
was := bus.StateTimeout
|
||||
bus.StateTimeout = 400 * time.Millisecond
|
||||
t.Cleanup(func() { bus.StateTimeout = was })
|
||||
|
||||
conn, kv := owedWatcher(t)
|
||||
// A group of three pull requests judged last, after many older heads: their keys come last.
|
||||
const older = 30
|
||||
var keys []string
|
||||
for i := 0; i < older; i++ {
|
||||
keys = append(keys, fmt.Sprintf("novox/mesh-catalog.%d.%040x.status=mesh/check", 400+i, i))
|
||||
}
|
||||
for _, member := range []string{"novox/mesh-sdk.76.34f210e5", "novox/mesh-controller.453.c06f75d0", "novox/mesh-delivery.33.28ee22ef"} {
|
||||
keys = append(keys, member+strings.Repeat("0", 32)+".status=mesh/check")
|
||||
}
|
||||
for _, k := range keys {
|
||||
if _, err := kv.Put(k, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"success"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var current, live []string
|
||||
// Each value takes 40 ms to be taken: 33 values take about 1.3 s, three times the bound.
|
||||
stop, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(ch bus.StateChange) error {
|
||||
if ch.Current {
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if ch.Current {
|
||||
current = append(current, ch.Key)
|
||||
} else {
|
||||
live = append(live, ch.Key)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a watch whose current values kept arriving was refused: %v", err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mu.Lock()
|
||||
got := append([]string(nil), current...)
|
||||
mu.Unlock()
|
||||
if len(got) != len(keys) {
|
||||
t.Fatalf("the watch returned after %d of %d current values", len(got), len(keys))
|
||||
}
|
||||
for i, k := range keys {
|
||||
if got[i] != k {
|
||||
t.Fatalf("current value %d is %s, want %s", i, got[i], k)
|
||||
}
|
||||
}
|
||||
|
||||
// And it holds: a verdict written afterwards reaches it as a change.
|
||||
next := "novox/mesh-sdk.77." + strings.Repeat("a", 40) + ".status=mesh/check"
|
||||
if _, err := kv.Put(next, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"pending"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(live) != 1 || live[0] != next {
|
||||
return fmt.Errorf("live changes so far: %v", live)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// A watch whose bundle takes nothing for longer than the bound is still refused, so a bundle that hangs
|
||||
// is said, not waited on for ever.
|
||||
func TestAWatchIsRefusedWhenNoCurrentValueIsTakenWithinTheBound(t *testing.T) {
|
||||
was := bus.StateTimeout
|
||||
bus.StateTimeout = 300 * time.Millisecond
|
||||
t.Cleanup(func() { bus.StateTimeout = was })
|
||||
|
||||
conn, kv := owedWatcher(t)
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
release := make(chan struct{})
|
||||
t.Cleanup(func() { close(release) })
|
||||
began := time.Now()
|
||||
_, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(bus.StateChange) error {
|
||||
<-release
|
||||
return nil
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "did not arrive in") {
|
||||
t.Fatalf("a watch whose first value was never taken: want refused, got %v", err)
|
||||
}
|
||||
if took := time.Since(began); took > 3*time.Second {
|
||||
t.Fatalf("the refusal took %s, far past the bound", took)
|
||||
}
|
||||
}
|
||||
|
||||
// **A membership the grants part wrote in its old wrapper is served whole** (issue 774). Until 2026-10-30 the part wrote
|
||||
// its pass record, the membership inside, into its state memberships; the tool runner read the value as a bare
|
||||
// membership, took it over the assignments stream's copy, and served no tool and no state. This is what the live mesh
|
||||
// held: read through mesh-sdk's codec, its tools are answered where the membership says and its state is reached.
|
||||
func TestAMembershipTheGrantsPartWroteWrappedIsServed(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Bucket(t, "gitea_repos")
|
||||
gitea := mt.MembershipOf("gitea", "anchor", false, nil)
|
||||
gitea.State = []bus.StateIssued{{Name: "repos", Bucket: "gitea_repos", Writes: true}}
|
||||
mesh.IssueInGrantsWrapped(t, gitea, "published")
|
||||
// The old wrapper of a membership the part held back carries its unpublished composition: refused, so the
|
||||
// assignments stream's copy is served, never what the hold kept back.
|
||||
pages := mt.MembershipOf("pages", "anchor", false, nil)
|
||||
mesh.Issue(t, pages)
|
||||
heldBack := mt.MembershipOf("pages", "anchor", true, nil)
|
||||
mesh.IssueInGrantsWrapped(t, heldBack, "not published, a shadow difference: publishing would lose the last holder of x")
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("gitea")
|
||||
mt.Until(t, func() error {
|
||||
if m := conn.Membership("gitea"); m == nil || len(m.Serves) == 0 {
|
||||
return fmt.Errorf("gitea's membership is %+v, want the one inside the wrapper", m)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if got, want := fmt.Sprint(conn.ServedOn("gitea", "gitea_list")), "[{mesh.mod.gitea.tool.gitea_list.anchor }]"; got != want {
|
||||
t.Errorf("gitea_list is served on %s, want %s", got, want)
|
||||
}
|
||||
if _, err := conn.StatePut("gitea", "repos", "novox.hq", []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatalf("the state the membership issues is refused: %v", err)
|
||||
}
|
||||
if e, err := conn.StateGet("gitea", "repos", "novox.hq"); err != nil || e == nil || string(e.Value) != `{"v":1}` {
|
||||
t.Errorf("the state reads back %+v, %v", e, err)
|
||||
}
|
||||
conn.Follow("pages")
|
||||
mt.Until(t, func() error {
|
||||
if m := conn.Membership("pages"); m == nil || len(m.Serves) == 0 {
|
||||
return fmt.Errorf("pages' membership is %+v, want the assignments stream's", m)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if got, want := fmt.Sprint(conn.ServedOn("pages", "pages_list")), "[{mesh.mod.pages.tool.pages_list.anchor }]"; got != want {
|
||||
t.Errorf("a held-back membership's wrapper is served: pages_list on %s, want the stream's %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// owedWatcher is the gitea module's view of the checker's owed: the bucket, and a runtime connection
|
||||
// that serves gitea with the owed issued for reading.
|
||||
func owedWatcher(t *testing.T) (*bus.Conn, interface {
|
||||
Put(string, []byte) (uint64, error)
|
||||
}) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
kv := mesh.Bucket(t, "mesh-delivery-checker_owed")
|
||||
gitea := mt.MembershipOf("gitea", "anchor", false, nil)
|
||||
gitea.State = []bus.StateIssued{{Name: "mesh-delivery-checker.owed", Bucket: "mesh-delivery-checker_owed"}}
|
||||
mesh.Issue(t, gitea)
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("gitea")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("gitea") == nil {
|
||||
return errors.New("gitea not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return conn, kv
|
||||
}
|
||||
|
||||
// **A watch given up, or stopped, hands over nothing more** (issue 774). The runtime's watch stopped its watcher
|
||||
// when the current values did not arrive within the bound, but the watcher still held values, and its goroutine
|
||||
// went on handing them over: to a watch id the bundle no longer held (on the provisioner, 51 failed watches were
|
||||
// followed by 1257 orphan changes). Each row blocks the bundle on its first value, gives the watch up, lets the
|
||||
// bundle go and counts what it is still handed.
|
||||
func TestAWatchGivenUpHandsOverNothingMore(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
refused bool // the bound gives it up, or the caller stops it
|
||||
}{
|
||||
{"refused at the bound", true},
|
||||
{"stopped by its caller", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
was := bus.StateTimeout
|
||||
bus.StateTimeout = 300 * time.Millisecond
|
||||
t.Cleanup(func() { bus.StateTimeout = was })
|
||||
conn, kv := owedWatcher(t)
|
||||
put := func(i int) {
|
||||
if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.refused {
|
||||
for i := 0; i < 4; i++ {
|
||||
put(i)
|
||||
}
|
||||
}
|
||||
var calls atomic.Int32
|
||||
first := make(chan struct{}, 1)
|
||||
release := make(chan struct{})
|
||||
deliver := func(ch bus.StateChange) error {
|
||||
if calls.Add(1) == 1 {
|
||||
first <- struct{}{}
|
||||
<-release
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var stop func()
|
||||
if tc.refused {
|
||||
if _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err == nil {
|
||||
t.Fatal("the watch whose bundle took nothing was not refused")
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
if stop, err = conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
put(i)
|
||||
}
|
||||
select {
|
||||
case <-first:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the bundle was handed no change")
|
||||
}
|
||||
stop()
|
||||
}
|
||||
// Let the watcher's server side deliver what it held, then free the bundle.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
close(release)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if n := calls.Load(); n != 1 {
|
||||
t.Fatalf("the bundle was handed %d changes of a watch given up, want 1 (the one in hand)", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's own stream, as the runtime publishes to it and hears it (novox/hq ADR 0363 §8, issue 725).
|
||||
//
|
||||
// A module may declare a stream of its own; the controller creates it and says, in the module's
|
||||
// membership, which of its events go there and on which subject (`streams`), and, in the membership of
|
||||
// each module granted to hear it, which durable consumer on it to bind (`hears`). The runtime publishes and
|
||||
// binds exactly as the membership says. A membership naming neither is what every membership was before:
|
||||
// every event on EVENTS, and only the consumer on EVENTS bound.
|
||||
|
||||
// StreamIssued is one of a module's own streams as its membership names it: the events it carries, as the
|
||||
// bus spells them (`*` one token, `>` the rest), and the subject an event of them is published on, `{event}`
|
||||
// standing for the event.
|
||||
type StreamIssued struct {
|
||||
Stream string `json:"stream"`
|
||||
Events []string `json:"events"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
|
||||
// HeardIssued is one module stream an assignment hears: the stream and the durable consumer on it the
|
||||
// runtime binds. The consumer is the controller's to make, to filter and to delete.
|
||||
type HeardIssued struct {
|
||||
Stream string `json:"stream"`
|
||||
Consumer string `json:"consumer"`
|
||||
}
|
||||
|
||||
// eventSubject is where a module's event is published: on the subject of the first of its membership's
|
||||
// streams that carries it, else on EVENTS — `mesh.mod.<module>.event.<event>`, as every event was before.
|
||||
func eventSubject(m *Membership, module, event string) string {
|
||||
if m != nil {
|
||||
for _, s := range m.Streams {
|
||||
if s.Subject == "" || !strings.Contains(s.Subject, "{event}") {
|
||||
continue
|
||||
}
|
||||
for _, pattern := range s.Events {
|
||||
if eventMatches(pattern, event) {
|
||||
return strings.ReplaceAll(s.Subject, "{event}", event)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return "mesh.mod." + module + ".event." + event
|
||||
}
|
||||
|
||||
// eventMatches says whether an event's name falls under a pattern as the bus matches subjects: `*` is one
|
||||
// token, `>` (or `**`, the manifest's spelling) the one or more tokens left.
|
||||
func eventMatches(pattern, event string) bool {
|
||||
if pattern == "" || event == "" {
|
||||
return false
|
||||
}
|
||||
ps, es := strings.Split(pattern, "."), strings.Split(event, ".")
|
||||
for i, p := range ps {
|
||||
if p == ">" || p == "**" {
|
||||
return i == len(ps)-1 && len(es) > i
|
||||
}
|
||||
if i >= len(es) || (p != "*" && p != es[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(ps) == len(es)
|
||||
}
|
||||
|
||||
// RebindEvery is how often a consumer the membership names and the bus did not let the runtime bind is
|
||||
// tried again: the controller makes the consumer and sends the user list carrying its grant in the
|
||||
// composition that issues the membership, so one may arrive before the other (ADR 0363 §6).
|
||||
var RebindEvery = 30 * time.Second
|
||||
|
||||
// hearing is one module's consumers: the one on EVENTS, and each on a module stream its membership names.
|
||||
// Every event of all of them is handed to the module one at a time, as the consumer on EVENTS alone did.
|
||||
type hearing struct {
|
||||
c *Conn
|
||||
js jetstream.JetStream
|
||||
module string
|
||||
deliver func(Envelope) error
|
||||
// serial hands the module one event at a time across its consumers (novox/hq issue 387).
|
||||
serial sync.Mutex
|
||||
|
||||
// halted is stopped, read by handle without mu: the bus's loops hand events while mu is held.
|
||||
halted atomic.Bool
|
||||
|
||||
mu sync.Mutex
|
||||
stopped bool
|
||||
bound map[HeardIssued]jetstream.ConsumeContext
|
||||
// made is when the bus made each bound consumer: a consumer deleted and made again under its name has
|
||||
// another time, and a loop bound to the old one is not reading the new one.
|
||||
made map[HeardIssued]time.Time
|
||||
said map[HeardIssued]bool
|
||||
// why is the last error the bus gave a consumer's loop, said when the loop ends on it.
|
||||
// Guarded by whyMu, not mu: the bus hands errors from its own goroutines, possibly while mu is held.
|
||||
whyMu sync.Mutex
|
||||
why map[HeardIssued]error
|
||||
kick chan struct{}
|
||||
done chan struct{}
|
||||
unhear func()
|
||||
}
|
||||
|
||||
func newHearing(c *Conn, js jetstream.JetStream, module string, deliver func(Envelope) error) *hearing {
|
||||
return &hearing{c: c, js: js, module: module, deliver: deliver,
|
||||
bound: map[HeardIssued]jetstream.ConsumeContext{}, made: map[HeardIssued]time.Time{}, said: map[HeardIssued]bool{}, why: map[HeardIssued]error{},
|
||||
kick: make(chan struct{}, 1), done: make(chan struct{})}
|
||||
}
|
||||
|
||||
// bind reads one durable consumer by name, and on failure says what it was doing. Errors the bus gives
|
||||
// the loop afterwards are handed to onErr, when given.
|
||||
func (h *hearing) bind(stream, durable string, onErr func(error)) (jetstream.ConsumeContext, time.Time, string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
consumer, err := h.js.Consumer(ctx, stream, durable)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, "binding", err
|
||||
}
|
||||
var created time.Time
|
||||
if info := consumer.CachedInfo(); info != nil {
|
||||
created = info.Created
|
||||
}
|
||||
opts := []jetstream.PullConsumeOpt{jetstream.PullMaxMessages(1)}
|
||||
if onErr != nil {
|
||||
opts = append(opts, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { onErr(err) }))
|
||||
}
|
||||
reading, err := consumer.Consume(h.handle, opts...)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, "reading", err
|
||||
}
|
||||
return reading, created, "", nil
|
||||
}
|
||||
|
||||
// handle hands one delivered event to the module: acknowledged when it took it, offered again after NakDelay
|
||||
// when it did not, terminated when it is not an event at all.
|
||||
func (h *hearing) handle(msg jetstream.Msg) {
|
||||
env, ok := envelopeOf(msg.Subject(), msg.Headers(), msg.Data())
|
||||
if !ok {
|
||||
// Unparseable: redelivering bytes no version of a handler can read is a loop.
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
// Kept alive while the bundle works on it, as a worker's work is (seat.go): the consumer's ack wait
|
||||
// (30 seconds) is for a runtime that died, not a handler that is slow — the runtime gives a handler
|
||||
// launch.EventTimeout. Without it an event handled longer than the ack wait was offered again while
|
||||
// it was handled, and counted toward max-deliver (novox/hq issue 387). Kept alive too while it waits
|
||||
// for an event from another of the module's consumers to be handled.
|
||||
done, stopped := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(stopped)
|
||||
tick := time.NewTicker(EventHeartbeat)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
_ = msg.InProgress()
|
||||
}
|
||||
}
|
||||
}()
|
||||
h.serial.Lock()
|
||||
halted := h.halted.Load()
|
||||
var err error
|
||||
if !halted {
|
||||
err = h.deliver(env)
|
||||
}
|
||||
h.serial.Unlock()
|
||||
close(done)
|
||||
// No in-progress after the answer: one after a Nak would stretch its delay to the whole ack wait.
|
||||
<-stopped
|
||||
if halted {
|
||||
// **A hearing stopped hands nothing on** (issue 1945): the module's bundle that subscribed may be
|
||||
// gone, loaded again in its place; offered again at once, to the consumer bound after it.
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
_ = msg.NakWithDelay(NakDelay)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
}
|
||||
|
||||
// start binds what the membership names now, and follows it: a new membership binds what it adds and stops
|
||||
// what it no longer names, and a consumer not bound yet is tried again every RebindEvery.
|
||||
func (h *hearing) start() {
|
||||
h.reconcile()
|
||||
h.unhear = h.c.onMembership(func(m Membership) {
|
||||
if m.Module != h.module {
|
||||
return
|
||||
}
|
||||
h.nudge()
|
||||
})
|
||||
go func() {
|
||||
tick := time.NewTicker(RebindEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-h.done:
|
||||
return
|
||||
case <-h.kick:
|
||||
case <-tick.C:
|
||||
}
|
||||
h.reconcile()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// forget drops the last error a consumer's loop was given.
|
||||
func (h *hearing) forget(heard HeardIssued) {
|
||||
h.whyMu.Lock()
|
||||
delete(h.why, heard)
|
||||
h.whyMu.Unlock()
|
||||
}
|
||||
|
||||
// nudge asks the loop to reconcile now.
|
||||
func (h *hearing) nudge() {
|
||||
select {
|
||||
case h.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// stop stops every consumer on a module stream this module reads; the one on EVENTS is its caller's.
|
||||
func (h *hearing) stop() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.stopped {
|
||||
return
|
||||
}
|
||||
h.stopped = true
|
||||
h.halted.Store(true)
|
||||
close(h.done)
|
||||
if h.unhear != nil {
|
||||
h.unhear()
|
||||
}
|
||||
for heard, reading := range h.bound {
|
||||
reading.Stop()
|
||||
delete(h.bound, heard)
|
||||
delete(h.made, heard)
|
||||
}
|
||||
}
|
||||
|
||||
// replaced says whether the consumer a loop is bound to is gone from the bus or was made again since. A bus
|
||||
// that cannot be asked right now says no: the loop is left as it is.
|
||||
func (h *hearing) replaced(heard HeardIssued) bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
consumer, err := h.js.Consumer(ctx, heard.Stream, heard.Consumer)
|
||||
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
return true
|
||||
}
|
||||
if err != nil || consumer.CachedInfo() == nil {
|
||||
return false
|
||||
}
|
||||
created := h.made[heard]
|
||||
return !created.IsZero() && !consumer.CachedInfo().Created.Equal(created)
|
||||
}
|
||||
|
||||
// reconcile makes what is bound what the module's membership names now.
|
||||
func (h *hearing) reconcile() {
|
||||
want := map[HeardIssued]bool{}
|
||||
if m := h.c.Membership(h.module); m != nil {
|
||||
for _, heard := range m.Hears {
|
||||
if heard.Stream != "" && heard.Consumer != "" {
|
||||
want[heard] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.stopped {
|
||||
return
|
||||
}
|
||||
for heard, reading := range h.bound {
|
||||
if !want[heard] {
|
||||
reading.Stop()
|
||||
delete(h.bound, heard)
|
||||
delete(h.made, heard)
|
||||
h.c.Logf("[mesh-tools] %s no longer hears %s: its membership stopped naming its consumer %s",
|
||||
h.module, heard.Stream, heard.Consumer)
|
||||
}
|
||||
}
|
||||
for heard := range h.said {
|
||||
if !want[heard] {
|
||||
delete(h.said, heard)
|
||||
h.forget(heard)
|
||||
}
|
||||
}
|
||||
// **A loop the bus ended is no longer reading** (nats.go ends a pull loop on its own when its consumer is
|
||||
// deleted): the controller deletes and makes again under the same name a restricted consumer it cannot
|
||||
// narrow (ADR 0363 §6), and the membership, unchanged, says nothing — so it is bound again here.
|
||||
// **A loop the bus did not end is asked about**: nats.go learns of a deleted consumer only from the answer
|
||||
// to a pull it has pending, and a consumer deleted and made again between two pulls, or while none is
|
||||
// pending, leaves the loop waiting out its heartbeat (30 seconds) on a consumer that is gone. So each
|
||||
// bound consumer is looked up by name: gone, or made at another time, ends its loop here.
|
||||
for heard, reading := range h.bound {
|
||||
if h.replaced(heard) {
|
||||
h.whyMu.Lock()
|
||||
if h.why[heard] == nil {
|
||||
h.why[heard] = errors.New("the consumer was deleted or made again")
|
||||
}
|
||||
h.whyMu.Unlock()
|
||||
reading.Stop()
|
||||
}
|
||||
}
|
||||
for heard, reading := range h.bound {
|
||||
select {
|
||||
case <-reading.Closed():
|
||||
delete(h.bound, heard)
|
||||
delete(h.made, heard)
|
||||
h.whyMu.Lock()
|
||||
why := h.why[heard]
|
||||
h.whyMu.Unlock()
|
||||
if why == nil {
|
||||
why = errors.New("no reason given")
|
||||
}
|
||||
h.c.Logf("[mesh-tools] %s stopped hearing %s: the bus ended reading its consumer %s (%v); bound again as it exists",
|
||||
h.module, heard.Stream, heard.Consumer, why)
|
||||
h.said[heard] = true
|
||||
default:
|
||||
}
|
||||
}
|
||||
missing := make([]HeardIssued, 0, len(want))
|
||||
for heard := range want {
|
||||
if h.bound[heard] == nil {
|
||||
missing = append(missing, heard)
|
||||
}
|
||||
}
|
||||
sort.Slice(missing, func(i, j int) bool { return missing[i].Stream < missing[j].Stream })
|
||||
for _, heard := range missing {
|
||||
heard := heard
|
||||
h.forget(heard)
|
||||
reading, created, doing, err := h.bind(heard.Stream, heard.Consumer, func(err error) {
|
||||
h.whyMu.Lock()
|
||||
h.why[heard] = err
|
||||
h.whyMu.Unlock()
|
||||
})
|
||||
if err != nil {
|
||||
// Said once, not at every attempt; said again when it is bound.
|
||||
if !h.said[heard] {
|
||||
h.c.Logf("[mesh-tools] %s does not hear %s yet: %s its consumer %s failed: %v; tried again every %s",
|
||||
h.module, heard.Stream, doing, heard.Consumer, err, RebindEvery)
|
||||
h.said[heard] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
h.bound[heard] = reading
|
||||
h.made[heard] = created
|
||||
delete(h.said, heard)
|
||||
h.c.Logf("[mesh-tools] %s hears %s on its consumer %s", h.module, heard.Stream, heard.Consumer)
|
||||
// When the bus ends the loop, look again at once rather than at the next tick.
|
||||
go func() {
|
||||
select {
|
||||
case <-reading.Closed():
|
||||
h.nudge()
|
||||
case <-h.done:
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package bus
|
||||
|
||||
import "testing"
|
||||
|
||||
// novox/hq ADR 0363 §2, §3: an event goes to the stream whose pattern carries it, as the bus matches subjects.
|
||||
func TestAnEventFallsUnderAStreamsPatternAsTheBusMatchesIt(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
pattern, event string
|
||||
want bool
|
||||
}{
|
||||
{"mail.*.message.added", "mail.home.message.added", true},
|
||||
{"mail.*.message.added", "mail.home.message.removed", false},
|
||||
{"mail.*.message.added", "mail.home.work.message.added", false},
|
||||
{"mail.>", "mail.home.message.added", true},
|
||||
{"mail.**", "mail.home", true},
|
||||
{"mail.>", "mail", false},
|
||||
{"status", "status", true},
|
||||
{"status", "status.changed", false},
|
||||
{"", "status", false},
|
||||
} {
|
||||
if got := eventMatches(c.pattern, c.event); got != c.want {
|
||||
t.Errorf("%q under %q: %v, want %v", c.event, c.pattern, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A membership naming no stream publishes every event on EVENTS, exactly as before (issue 725).
|
||||
func TestAnEventNoStreamCarriesIsPublishedOnEvents(t *testing.T) {
|
||||
mail := &Membership{Module: "mail", Streams: []StreamIssued{{Stream: "MAIL", Events: []string{"mail.*.message.added"},
|
||||
Subject: "mesh.stream.mail.event.{event}"}}}
|
||||
for _, c := range []struct {
|
||||
m *Membership
|
||||
event string
|
||||
want string
|
||||
}{
|
||||
{nil, "mail.home.message.added", "mesh.mod.mail.event.mail.home.message.added"},
|
||||
{&Membership{Module: "mail"}, "mail.home.message.added", "mesh.mod.mail.event.mail.home.message.added"},
|
||||
{mail, "mail.home.message.added", "mesh.stream.mail.event.mail.home.message.added"},
|
||||
{mail, "status", "mesh.mod.mail.event.status"},
|
||||
} {
|
||||
if got := eventSubject(c.m, "mail", c.event); got != c.want {
|
||||
t.Errorf("%s: %s, want %s", c.event, got, c.want)
|
||||
}
|
||||
}
|
||||
if got := keyOf("mesh.stream.mail.event.mail.home.message.added"); got != "mail.mail.home.message.added" {
|
||||
t.Errorf("a stream's event reaches its consumer as %q, not as on EVENTS", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// streamMesh is a mesh with the mail module's own stream MAIL, and a connection on anchor following the
|
||||
// memberships of the modules named.
|
||||
func streamMesh(t *testing.T, memberships ...bus.Membership) (*mt.Mesh, *bus.Conn) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
mesh.ModuleStream(t, "mail", "MAIL")
|
||||
for _, m := range memberships {
|
||||
mesh.Issue(t, m)
|
||||
}
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
for _, m := range memberships {
|
||||
conn.Follow(m.Module)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
for _, m := range memberships {
|
||||
if conn.Membership(m.Module) == nil {
|
||||
return errors.New(m.Module + " not issued yet")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return mesh, conn
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §2, §8 (issue 725): an event one of the module's streams carries is published on that
|
||||
// stream's subject and kept by that stream alone; any other event of it goes to EVENTS as before.
|
||||
func TestAnEventADeclaredStreamCarriesIsPublishedOnItsSubjectAlone(t *testing.T) {
|
||||
mail := mt.MembershipOf("mail", "anchor", true, nil)
|
||||
mail.Streams = []bus.StreamIssued{{Stream: "MAIL", Events: []string{"mail.*.message.added"},
|
||||
Subject: "mesh.stream.mail.event.{event}"}}
|
||||
mesh, conn := streamMesh(t, mail)
|
||||
|
||||
landed := mesh.NextEvent(t, "mesh.stream.mail.event.>")
|
||||
if err := conn.PublishAs("mail", bus.Envelope{Key: "mail.home.message.added", Body: json.RawMessage(`{"id":1}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case got := <-landed:
|
||||
if got != "mesh.stream.mail.event.mail.home.message.added" {
|
||||
t.Errorf("published on %s", got)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the event did not land on its stream's subject")
|
||||
}
|
||||
if err := conn.PublishAs("mail", bus.Envelope{Key: "status", Body: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := mesh.Messages(t, "MAIL"); n != 1 {
|
||||
t.Errorf("MAIL holds %d messages, want the one event it carries", n)
|
||||
}
|
||||
if n := mesh.Messages(t, "EVENTS"); n != 1 {
|
||||
t.Errorf("EVENTS holds %d messages, want only the event no stream carries", n)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §8 (issue 725): a module hears, beside EVENTS, every module stream its membership names,
|
||||
// one event at a time, with the key it reads on EVENTS; a membership naming none hears EVENTS alone.
|
||||
func TestAModuleHearsEveryStreamItsMembershipNames(t *testing.T) {
|
||||
reader := mt.MembershipOf("reader", "anchor", false, nil)
|
||||
mesh, conn := streamMesh(t, reader)
|
||||
mesh.Consumer(t, "anchor", "reader", []string{"mesh.mod.alpha.event.>"}, 5*time.Second)
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.mail.home.>"}, 5*time.Second)
|
||||
|
||||
var mu sync.Mutex
|
||||
heard := map[string]int{}
|
||||
handling := 0
|
||||
stop, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||
mu.Lock()
|
||||
handling++
|
||||
if handling > 1 {
|
||||
t.Error("two events were handed to the module at once")
|
||||
}
|
||||
mu.Unlock()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
mu.Lock()
|
||||
handling--
|
||||
heard[env.Key]++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
count := func(key string) int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return heard[key]
|
||||
}
|
||||
|
||||
// Its membership names no stream: EVENTS alone, as before.
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 1})
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error {
|
||||
if count("alpha.happened") != 1 {
|
||||
return errors.New("the event on EVENTS was not heard")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if _, notYet := mesh.PendingOn(t, "MAIL", "anchor", "reader"); notYet != 1 || count("mail.mail.home.message.added") != 0 {
|
||||
t.Fatalf("a module whose membership names no stream heard MAIL (%d not yet delivered)", notYet)
|
||||
}
|
||||
|
||||
// Its membership now names its consumer on MAIL: bound, and what waited there is heard.
|
||||
reader.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "anchor_reader"}}
|
||||
mesh.Issue(t, reader)
|
||||
mt.Until(t, func() error {
|
||||
if count("mail.mail.home.message.added") != 1 {
|
||||
return errors.New("the event on MAIL was not heard")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for n := 2; n <= 4; n++ {
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": n})
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": n})
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
if a, m := count("alpha.happened"), count("mail.mail.home.message.added"); a != 4 || m != 4 {
|
||||
return fmt.Errorf("heard %d on EVENTS and %d on MAIL, want 4 and 4", a, m)
|
||||
}
|
||||
for _, stream := range []string{"EVENTS", "MAIL"} {
|
||||
if ack, notYet := mesh.PendingOn(t, stream, "anchor", "reader"); ack != 0 || notYet != 0 {
|
||||
return fmt.Errorf("%s still holds %d unacknowledged, %d undelivered", stream, ack, notYet)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// A membership that stops naming it stops what it reads there.
|
||||
reader.Hears = nil
|
||||
mesh.Issue(t, reader)
|
||||
mt.Until(t, func() error {
|
||||
if mesh.Consumers(t, "MAIL") != 1 {
|
||||
return errors.New("MAIL's consumer is gone")
|
||||
}
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 0})
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if _, notYet := mesh.PendingOn(t, "MAIL", "anchor", "reader"); notYet == 0 {
|
||||
return errors.New("MAIL is still read")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// The membership as the grants part keeps it (ADR 0571 clause 7): read first, applied live, and the assignments
|
||||
// stream's copy passed over once the state has given one.
|
||||
granted := reader
|
||||
granted.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "from-grants"}}
|
||||
mesh.IssueInGrants(t, granted)
|
||||
again, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again.Logf = func(string, ...any) {}
|
||||
t.Cleanup(again.Close)
|
||||
again.Follow("reader")
|
||||
consumerOf := func() string {
|
||||
if m := again.Membership("reader"); m != nil && len(m.Hears) == 1 {
|
||||
return m.Hears[0].Consumer
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if got := consumerOf(); got != "from-grants" {
|
||||
t.Fatalf("the membership read at the follow names %q, want the grants part's", got)
|
||||
}
|
||||
granted.Hears[0].Consumer = "from-grants-2"
|
||||
mesh.IssueInGrants(t, granted)
|
||||
mt.Until(t, func() error {
|
||||
if got := consumerOf(); got != "from-grants-2" {
|
||||
return fmt.Errorf("the membership names %q, want the grants part's change", got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// A later, different copy on the assignments stream changes nothing once the state has given one.
|
||||
arrived := make(chan struct{}, 1)
|
||||
probe, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "probe", Node: "elsewhere"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe.Logf = func(string, ...any) {}
|
||||
t.Cleanup(probe.Close)
|
||||
stopProbe, err := probe.Raw(bus.MembershipSubject("anchor", "reader"), func(string, []byte) []byte {
|
||||
select {
|
||||
case arrived <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopProbe)
|
||||
probe.Flush()
|
||||
streamed := reader
|
||||
streamed.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "from-stream"}}
|
||||
mesh.Issue(t, streamed)
|
||||
select {
|
||||
case <-arrived:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the stream's copy was not published")
|
||||
}
|
||||
again.Flush()
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if got := consumerOf(); got != "from-grants-2" {
|
||||
t.Fatalf("the stream's copy was applied over the grants part's: the membership names %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §6 (issue 725): a consumer the membership names before the controller made it is bound once
|
||||
// it exists, without the module's events on EVENTS waiting for it.
|
||||
func TestAConsumerNamedBeforeItExistsIsBoundOnceItDoes(t *testing.T) {
|
||||
restore := bus.RebindEvery
|
||||
bus.RebindEvery = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.RebindEvery = restore })
|
||||
reader := mt.MembershipOf("reader", "anchor", false, nil)
|
||||
reader.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "anchor_reader"}}
|
||||
mesh, conn := streamMesh(t, reader)
|
||||
mesh.Consumer(t, "anchor", "reader", []string{"mesh.mod.alpha.event.>"}, 5*time.Second)
|
||||
|
||||
var mu sync.Mutex
|
||||
heard := map[string]int{}
|
||||
stop, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||
mu.Lock()
|
||||
heard[env.Key]++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a consumer on a module stream not made yet kept the module from hearing EVENTS: %v", err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{})
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.>"}, 5*time.Second)
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{})
|
||||
mt.Until(t, func() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if heard["alpha.happened"] != 1 || heard["mail.mail.home.message.added"] != 1 {
|
||||
return fmt.Errorf("heard %v", heard)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §6 (issue 725): a consumer the controller deletes and makes again under the same name, its
|
||||
// membership unchanged, is bound again, and what is published after is heard — the module does not go deaf.
|
||||
func TestAConsumerDeletedAndMadeAgainIsHeardAgain(t *testing.T) {
|
||||
restore := bus.RebindEvery
|
||||
bus.RebindEvery = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.RebindEvery = restore })
|
||||
reader := mt.MembershipOf("reader", "anchor", false, nil)
|
||||
reader.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "anchor_reader"}}
|
||||
mesh, conn := streamMesh(t, reader)
|
||||
mesh.Consumer(t, "anchor", "reader", []string{"mesh.mod.alpha.event.>"}, 5*time.Second)
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.>"}, 5*time.Second)
|
||||
logs := &syncLogs{}
|
||||
conn.Logf = logs.Logf
|
||||
|
||||
var mu sync.Mutex
|
||||
heard := 0
|
||||
stop, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||
if env.Key == "mail.mail.home.message.added" {
|
||||
mu.Lock()
|
||||
heard++
|
||||
mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
count := func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return heard
|
||||
}
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error {
|
||||
if count() != 1 {
|
||||
return errors.New("the first event on MAIL was not heard")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
mesh.RemoveConsumer(t, "MAIL", "anchor", "reader")
|
||||
mt.Until(t, func() error {
|
||||
if !logs.Has("stopped hearing MAIL") {
|
||||
return errors.New("the end of the loop was not said")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.>"}, 5*time.Second)
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 2})
|
||||
mt.Until(t, func() error {
|
||||
if count() != 2 {
|
||||
return fmt.Errorf("heard %d events on MAIL after its consumer was made again, want 2", count())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// syncLogs keeps what the runtime says, from any goroutine.
|
||||
type syncLogs struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (l *syncLogs) Logf(format string, args ...any) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.lines = append(l.lines, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *syncLogs) Has(fragment string) bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
for _, line := range l.lines {
|
||||
if strings.Contains(line, fragment) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// Waiting on the bus for one event (novox/hq ADR 0367): what the mesh MCP server's `mesh_wait` does.
|
||||
//
|
||||
// **A subscription for the call, and nothing kept** (§2): no durable consumer, no queue, no stream. It
|
||||
// subscribes the subjects when called and unsubscribes them before it answers, so an event published
|
||||
// between two calls is not heard — a session reads the state from its owner after every wake (§4).
|
||||
|
||||
// WaitIssued is one event family the mesh issued this account to wait on (ADR 0367 §2): the name a session
|
||||
// asks for, and the subject it lands on. Only the tool runner's membership carries any.
|
||||
type WaitIssued struct {
|
||||
Event string `json:"event"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
|
||||
// Heard is one event a wait heard: the subject it arrived on, its body and when it arrived.
|
||||
type Heard struct {
|
||||
Subject string
|
||||
Body json.RawMessage
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// HearFirst subscribes the subjects for this call alone and answers the first message whose body match
|
||||
// accepts (every one, when match is nil), or ok false once `within` has passed. Every subscription is
|
||||
// closed before it returns. A subscription the bus refuses is said as an error naming the subject, at once;
|
||||
// a caller gone — ctx done — ends the wait at once with ctx's error, so nothing waits on for nobody.
|
||||
func (c *Conn) HearFirst(ctx context.Context, subjects []string, within time.Duration, match func(json.RawMessage) bool) (heard Heard, ok bool, err error) {
|
||||
if len(subjects) == 0 {
|
||||
return Heard{}, false, fmt.Errorf("a wait names at least one subject")
|
||||
}
|
||||
got := make(chan Heard, 1)
|
||||
refused := make(chan string, len(subjects))
|
||||
var subs []*nats.Subscription
|
||||
defer func() {
|
||||
for _, s := range subs {
|
||||
_ = s.Unsubscribe()
|
||||
}
|
||||
c.mu.Lock()
|
||||
for _, subject := range subjects {
|
||||
delete(c.waits[subject], refused)
|
||||
if len(c.waits[subject]) == 0 {
|
||||
delete(c.waits, subject)
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}()
|
||||
c.mu.Lock()
|
||||
if c.waits == nil {
|
||||
c.waits = map[string]map[chan string]bool{}
|
||||
}
|
||||
for _, subject := range subjects {
|
||||
if c.waits[subject] == nil {
|
||||
c.waits[subject] = map[chan string]bool{}
|
||||
}
|
||||
c.waits[subject][refused] = true
|
||||
}
|
||||
c.mu.Unlock()
|
||||
for _, subject := range subjects {
|
||||
sub, err := c.nc.Subscribe(subject, func(msg *nats.Msg) {
|
||||
body := json.RawMessage(msg.Data)
|
||||
if !json.Valid(body) {
|
||||
quoted, _ := json.Marshal(string(msg.Data))
|
||||
body = quoted
|
||||
}
|
||||
if match != nil && !match(body) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case got <- Heard{Subject: msg.Subject, Body: body, At: time.Now().UTC()}:
|
||||
default:
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return Heard{}, false, fmt.Errorf("could not wait on %s: %w", subject, err)
|
||||
}
|
||||
subs = append(subs, sub)
|
||||
}
|
||||
// The bus has every subscription — or has said which it refused — once this returns.
|
||||
if err := c.nc.FlushTimeout(5 * time.Second); err != nil {
|
||||
return Heard{}, false, fmt.Errorf("the bus did not take the wait: %w", err)
|
||||
}
|
||||
timer := time.NewTimer(within)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case h := <-got:
|
||||
return h, true, nil
|
||||
case subject := <-refused:
|
||||
return Heard{}, false, fmt.Errorf("the bus refused this account a subscription on %s: it is not in the "+
|
||||
"account's wait grant (novox/hq ADR 0367 §2)", subject)
|
||||
case <-timer.C:
|
||||
return Heard{}, false, nil
|
||||
case <-ctx.Done():
|
||||
return Heard{}, false, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// waitRefused tells every wait on a subject the bus refused that it was refused; true when one was waiting.
|
||||
func (c *Conn) waitRefused(subject string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
told := false
|
||||
for ch := range c.waits[subject] {
|
||||
select {
|
||||
case ch <- subject:
|
||||
default:
|
||||
}
|
||||
told = true
|
||||
}
|
||||
return told
|
||||
}
|
||||
|
||||
// Subscriptions is how many subscriptions this connection holds now: what a wait leaves behind is none.
|
||||
func (c *Conn) Subscriptions() int { return c.nc.NumSubscriptions() }
|
||||
|
||||
// subjectRefused is the subject a subscription refusal names, or "".
|
||||
func subjectRefused(err error) string {
|
||||
m := refusedSubject.FindStringSubmatch(err.Error())
|
||||
if len(m) < 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.Trim(m[1], `"`)
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0367 §2, §5: the bus refuses the tool runner's account any subject its wait grant does not
|
||||
// name — another asker's `decided` among them — and a wait on one says the bus refused it, at once.
|
||||
func TestTheBusRefusesAWaitOutsideTheGrant(t *testing.T) {
|
||||
granted := "mesh.seat.operator-channel.event.decided.claude-code"
|
||||
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, NoLog: true, NoSigs: true,
|
||||
Users: []*server.User{{Username: "desk.node-tools", Password: "pw", Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: []string{"$JS.API.DIRECT.GET.>", "mesh.>"}},
|
||||
Subscribe: &server.SubjectPermission{Allow: []string{granted, "_INBOX.desk.node-tools.>", "mesh.assignment.desk.*"}},
|
||||
}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
c := dial(t, s.ClientURL(), "node-tools", "desk", "desk.node-tools")
|
||||
|
||||
start := time.Now()
|
||||
_, ok, err := c.HearFirst(context.Background(), []string{"mesh.seat.operator-channel.event.decided.shop"}, 5*time.Second, nil)
|
||||
if ok || err == nil || !strings.Contains(err.Error(), "refused") {
|
||||
t.Errorf("a wait on another asker's decided: heard %v, %v", ok, err)
|
||||
}
|
||||
if time.Since(start) > 3*time.Second {
|
||||
t.Errorf("the refusal took %s to be said", time.Since(start))
|
||||
}
|
||||
|
||||
done := make(chan bus.Heard, 1)
|
||||
go func() {
|
||||
h, ok, err := c.HearFirst(context.Background(), []string{granted}, 5*time.Second, func(body json.RawMessage) bool { return true })
|
||||
if ok && err == nil {
|
||||
done <- h
|
||||
}
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if err := c.Say(granted, []byte(`{"ask":"a1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case h := <-done:
|
||||
if h.Subject != granted || string(h.Body) != `{"ask":"a1"}` {
|
||||
t.Errorf("heard %+v", h)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a wait on the granted subject heard nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// A wait whose caller is gone ends at once, its subscription closed (the review of issue 745).
|
||||
func TestAWaitEndsWhenItsCallerIsGone(t *testing.T) {
|
||||
c := dial(t, mt.URL(t), "node-tools", "desk", "")
|
||||
before := c.Subscriptions()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
time.AfterFunc(200*time.Millisecond, cancel)
|
||||
start := time.Now()
|
||||
_, ok, err := c.HearFirst(ctx, []string{"mesh.mod.mesh-knowledge.event.reviewed"}, time.Minute, nil)
|
||||
if ok || !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("a cancelled wait answered %v, %v", ok, err)
|
||||
}
|
||||
if took := time.Since(start); took > 2*time.Second {
|
||||
t.Errorf("a cancelled wait ended after %s", took)
|
||||
}
|
||||
if after := c.Subscriptions(); after != before {
|
||||
t.Errorf("%d subscriptions before, %d after", before, after)
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,8 @@ package console
|
||||
|
||||
// The mesh's tools found by address, not announced whole (novox/hq ADR 0195, to-be 34 §3a).
|
||||
//
|
||||
// The console announces six tools — five to find and call, and mesh_runtimes to say which runtimes
|
||||
// discovery heard. Everything the mesh answers is reached through them by one address per layer:
|
||||
// The console announces seven tools — five to find and call, mesh_runtimes to say which runtimes
|
||||
// discovery heard, and mesh_wait to wait on the bus for an event (novox/hq ADR 0367). Everything the mesh answers is reached through them by one address per layer:
|
||||
//
|
||||
// <seat>.<verb> a seat held once for the mesh — its holder answers
|
||||
// <node>/<seat>.<verb> a seat held once per machine — that machine's holder answers
|
||||
@@ -17,6 +17,10 @@ package console
|
||||
// session — so a tool that arrived a minute ago is found without the client reconnecting.
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
@@ -32,7 +36,7 @@ import (
|
||||
// one agent turn's search, describe and call ask once, short enough that nothing goes stale.
|
||||
var IndexKept = 5 * time.Second
|
||||
|
||||
// The six tools the console announces. Names of the API's kind — letters, digits, `_`, `-`.
|
||||
// The seven tools the console announces. Names of the API's kind — letters, digits, `_`, `-`.
|
||||
const (
|
||||
verbOverview = "mesh_overview"
|
||||
verbMachine = "mesh_machine"
|
||||
@@ -40,6 +44,7 @@ const (
|
||||
verbDescribe = "mesh_describe"
|
||||
verbCall = "mesh_call"
|
||||
verbRuntimes = "mesh_runtimes"
|
||||
verbWait = "mesh_wait"
|
||||
)
|
||||
|
||||
// searchCap is how many matches a search answers before it says how many more there were.
|
||||
@@ -50,7 +55,7 @@ const grammar = "Addresses: `<seat>.<verb>` for a seat held once for the mesh (e
|
||||
"`<node>/<module>.<tool>` for a module on one machine (e.g. `novox/postgres.postgres_list_databases`); " +
|
||||
"and `<module>.<tool>` also for a module whose instances are interchangeable."
|
||||
|
||||
// discovery is the six tools as tools/list announces them.
|
||||
// discovery is the seven tools as tools/list announces them.
|
||||
func discovery() []map[string]any {
|
||||
str := func(desc string) map[string]any { return map[string]any{"type": "string", "description": desc} }
|
||||
obj := func(props map[string]any, required ...string) map[string]any {
|
||||
@@ -84,12 +89,13 @@ func discovery() []map[string]any {
|
||||
"arrive, its size in bytes, how many modules and tools it announced, whether it was shortened to fit the " +
|
||||
"bus, and when it was last heard — and every runtime or machine expected that was not heard. Read-only; " +
|
||||
"for when an address is said to be missing or on another machine."},
|
||||
waitTool(),
|
||||
}
|
||||
}
|
||||
|
||||
func isDiscovery(name string) bool {
|
||||
switch name {
|
||||
case verbOverview, verbMachine, verbSearch, verbDescribe, verbCall, verbRuntimes:
|
||||
case verbOverview, verbMachine, verbSearch, verbDescribe, verbCall, verbRuntimes, verbWait:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -125,15 +131,55 @@ type index struct {
|
||||
NotAnswering []string
|
||||
Listing *Listing // the flat catalogue the call path resolves subjects with
|
||||
Discovery announce.Discovery
|
||||
// Recorded is where the controller's records place each module that declares tools.
|
||||
// Recorded is where placement places each module that declares tools.
|
||||
Recorded map[string][]string
|
||||
// Unheard is every runtime whose answer this index lacks: it said it was there and did not say
|
||||
// what it serves in time, or it answered before and not now. An address it might answer is never
|
||||
// called missing while it is here (2026-10-05).
|
||||
Unheard []unheard
|
||||
// Replaces is what each verb and tool replaces, from the controller's records (novox/hq ADR 0245):
|
||||
// Placed is the state seat-holders as the seat mesh-placement's holder wrote it (novox/hq ADR 0571 clause 4), by its
|
||||
// keys `<seat>` and `<seat>.<node>`, read with delivery.HoldersOf; PlacedWhy says why it could not be read.
|
||||
Placed map[string][]byte
|
||||
PlacedWhy string
|
||||
// Replaces is what each verb and tool replaces (novox/hq ADR 0245):
|
||||
// `seat:<seat>.<verb>` and `<module>.<tool>`.
|
||||
Replaces map[string][]string
|
||||
// Earlier is every runtime not heard this round whose last answer, within RuntimeForgotten, this
|
||||
// index holds instead (novox/hq issue 554), by runtimeKey: when it was heard.
|
||||
Earlier map[string]time.Time
|
||||
// Stale is every module on a machine (`mod:<module>@<node>`) and every seat's holder
|
||||
// (`seat:<seat>@<node>`) this index knows only from an earlier answer: from whom, and when.
|
||||
Stale map[string]staleFrom
|
||||
}
|
||||
|
||||
// staleFrom is where an address known only from an earlier round of discovery was heard.
|
||||
type staleFrom struct {
|
||||
Runtime string
|
||||
Machine string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// say is the sentence a call or a description carries for it.
|
||||
func (f staleFrom) say() string {
|
||||
who := f.Runtime
|
||||
if f.Machine != "" {
|
||||
who += " on " + f.Machine
|
||||
}
|
||||
return fmt.Sprintf("found from what %s announced %s ago: it did not answer the latest round of discovery",
|
||||
who, time.Since(f.At).Round(time.Second))
|
||||
}
|
||||
|
||||
// earlier is a runtime's last answer, used for a round that did not hear it.
|
||||
type earlier struct {
|
||||
Heard announce.Heard
|
||||
At time.Time
|
||||
}
|
||||
|
||||
func endpointKey(e announce.Endpoint) string {
|
||||
if e.Kind == announce.KindSeat {
|
||||
return "seat:" + e.Seat + "@" + e.Node
|
||||
}
|
||||
return "mod:" + e.Module + "@" + e.Node
|
||||
}
|
||||
|
||||
// unheard is one runtime discovery did not hear in full, and why.
|
||||
@@ -185,42 +231,8 @@ func findTool(tools []Tool, name string) *Tool {
|
||||
return nil
|
||||
}
|
||||
|
||||
// controllerOutput is a controller seat verb's answer: the command's printed output.
|
||||
func controllerOutput(conn *bus.Conn, verb string) (string, error) {
|
||||
got, err := conn.Ask("seat:mesh-controller."+verb, map[string]any{}, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var r struct {
|
||||
Output string `json:"output"`
|
||||
OK *bool `json:"ok"`
|
||||
}
|
||||
if json.Unmarshal(got.Result, &r) != nil {
|
||||
return "", fmt.Errorf("mesh-controller.%s answered something that is not its output", verb)
|
||||
}
|
||||
if r.OK != nil && !*r.OK {
|
||||
return "", fmt.Errorf("mesh-controller.%s: %s", verb, strings.TrimSpace(r.Output))
|
||||
}
|
||||
return r.Output, nil
|
||||
}
|
||||
|
||||
// jsonIn is the JSON document a command printed, after any lines it said first: a seat verb runs
|
||||
// the controller's command, and a command may warn before it answers.
|
||||
func jsonIn(output string) string {
|
||||
t := strings.TrimSpace(output)
|
||||
if strings.HasPrefix(t, "{") || strings.HasPrefix(t, "[") {
|
||||
return t
|
||||
}
|
||||
for _, open := range []string{"\n{", "\n["} {
|
||||
if i := strings.Index(output, open); i >= 0 {
|
||||
return strings.TrimSpace(output[i+1:])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// recordedModule is a module as the controller's records hold it (`module list --json`): where the
|
||||
// mesh assigned it, and whether it declares tools — what should announce itself, and where.
|
||||
// recordedModule is a module as the bus holds it: where the placement seat places it, and whether its manifest declares
|
||||
// tools — what should announce itself, and where (novox/hq ADR 0571).
|
||||
type recordedModule struct {
|
||||
Module string `json:"module"`
|
||||
On []string `json:"on"`
|
||||
@@ -229,42 +241,68 @@ type recordedModule struct {
|
||||
Replaces map[string][]string `json:"replaces,omitempty"`
|
||||
}
|
||||
|
||||
// recordedMachine is a machine as the controller's records hold it (`node list --json`).
|
||||
type recordedMachine struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// indexOn asks the mesh what it holds (novox/hq ADR 0197): what answers, from every runtime's own
|
||||
// announcement on the bus — one `$SRV.INFO` request — and what should, from the controller's records
|
||||
// read as JSON. Nothing is inferred from a roster and nothing is parsed from print.
|
||||
func indexOn(conn *bus.Conn) (*index, error) {
|
||||
//
|
||||
// `held` is what the console holds already, so an instance whose announcement is unchanged is not
|
||||
// asked for it again; `before` is the last answer of every runtime this round did not hear, which the
|
||||
// index holds instead, marked with its age (novox/hq issue 554).
|
||||
func indexOn(conn *bus.Conn, held announce.Known, before func(announce.Discovery) []earlier) (*index, error) {
|
||||
var wg sync.WaitGroup
|
||||
var nodesOut, modulesOut string
|
||||
var toolsAnswer json.RawMessage
|
||||
wg.Add(3)
|
||||
go func() { defer wg.Done(); nodesOut, _ = controllerOutput(conn, "nodes") }()
|
||||
go func() { defer wg.Done(); modulesOut, _ = controllerOutput(conn, "modules") }()
|
||||
// The seats' verbs as the records define them, for what each replaces (novox/hq ADR 0245): answered
|
||||
// in-process from the records, never a command.
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if got, err := conn.Ask("seat:mesh-controller.tools", map[string]any{}, ""); err == nil {
|
||||
toolsAnswer = got.Result
|
||||
}
|
||||
}()
|
||||
d, err := announce.Gather(conn)
|
||||
var placed map[string][]byte
|
||||
var placedErr error
|
||||
var recorded []recordedModule
|
||||
var known []string
|
||||
var recordsWhy []string
|
||||
wg.Add(2)
|
||||
// Which modules there are and where they are placed, and which machines the mesh has, from the bus (ADR 0571
|
||||
// clauses 4 and 5): the indexer's modules and manifests and the placement seat's placements, never the controller.
|
||||
go func() { defer wg.Done(); recorded, known, recordsWhy = recordsOn(conn) }()
|
||||
// Who holds which seat on which node, from placement (ADR 0571 clause 4): no call of the controller.
|
||||
go func() { defer wg.Done(); placed, placedErr = seatHoldersOn(conn) }()
|
||||
d, err := announce.Gather(conn, held)
|
||||
wg.Wait()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var endpoints []announce.Endpoint
|
||||
fresh := map[string]bool{}
|
||||
for _, h := range d.Heard {
|
||||
endpoints = append(endpoints, announce.Endpoints(h.Info)...)
|
||||
for _, e := range announce.Endpoints(h.Info) {
|
||||
fresh[endpointKey(e)] = true
|
||||
endpoints = append(endpoints, e)
|
||||
}
|
||||
}
|
||||
// **A runtime not heard this round is not gone** (novox/hq issue 554): a round in which no answer
|
||||
// crossed the laptop's link in time replaced a full index with an empty one, and every seat and
|
||||
// module call was refused. What it announced last, within RuntimeForgotten, stands — said with its age.
|
||||
stale := map[string]staleFrom{}
|
||||
earlierAt := map[string]time.Time{}
|
||||
if before != nil {
|
||||
for _, b := range before(d) {
|
||||
earlierAt[runtimeKey(b.Heard.Info.Name, b.Heard.Info.ID)] = b.At
|
||||
for _, e := range announce.Endpoints(b.Heard.Info) {
|
||||
k := endpointKey(e)
|
||||
if fresh[k] {
|
||||
continue
|
||||
}
|
||||
if _, have := stale[k]; !have {
|
||||
stale[k] = staleFrom{Runtime: b.Heard.Info.Name, Machine: b.Heard.Machine(), At: b.At}
|
||||
}
|
||||
endpoints = append(endpoints, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
x, announced, machines := indexOfEndpoints(endpoints)
|
||||
l := x.Listing
|
||||
x.Discovery = d
|
||||
x.Stale, x.Earlier = stale, earlierAt
|
||||
x.Placed = placed
|
||||
if placedErr != nil {
|
||||
x.PlacedWhy = placedErr.Error()
|
||||
}
|
||||
for _, s := range d.Silent {
|
||||
x.Unheard = append(x.Unheard, unheard{Runtime: s.Name, Machine: s.Machine,
|
||||
Why: fmt.Sprintf("it answered PING; what it serves did not arrive within %s", announce.Patience)})
|
||||
@@ -272,25 +310,24 @@ func indexOn(conn *bus.Conn) (*index, error) {
|
||||
|
||||
// What should have answered: every assignment of a module that declares tools. Silence is named;
|
||||
// a module with no tools is never a name here.
|
||||
var recorded []recordedModule
|
||||
recordedOK := json.Unmarshal([]byte(jsonIn(modulesOut)), &recorded) == nil
|
||||
x.Replaces = replacesOf(toolsAnswer, recorded)
|
||||
if recordedOK {
|
||||
for _, m := range recorded {
|
||||
if !m.Tools {
|
||||
continue
|
||||
}
|
||||
x.Recorded[m.Module] = append([]string{}, m.On...)
|
||||
for _, n := range m.On {
|
||||
// A holder of a seat held once for the mesh announces no machine — the seat is the mesh's,
|
||||
// not a machine's — so what it announced without one answers for wherever it is assigned.
|
||||
if !announced[m.Module][n] && !announced[m.Module][""] {
|
||||
l.NotAnswering = append(l.NotAnswering, m.Module+" on "+n)
|
||||
}
|
||||
x.Replaces = replacesOf(catalogue.DefaultSeats(), recorded)
|
||||
for _, m := range recorded {
|
||||
if !m.Tools {
|
||||
continue
|
||||
}
|
||||
x.Recorded[m.Module] = append([]string{}, m.On...)
|
||||
for _, n := range m.On {
|
||||
// A holder of a seat held once for the mesh announces no machine — the seat is the mesh's,
|
||||
// not a machine's — so what it announced without one answers for wherever it is assigned.
|
||||
if !announced[m.Module][n] && !announced[m.Module][""] {
|
||||
l.NotAnswering = append(l.NotAnswering, m.Module+" on "+n)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
l.NotAnswering = append(l.NotAnswering, "mesh-controller (its records of the modules did not answer, so what is missing cannot be said)")
|
||||
}
|
||||
l.NotAnswering = append(l.NotAnswering, recordsWhy...)
|
||||
if x.PlacedWhy != "" {
|
||||
l.NotAnswering = append(l.NotAnswering, delivery.SeatHoldersRead+" (who holds which seat by placement could not be read: "+
|
||||
x.PlacedWhy+")")
|
||||
}
|
||||
for _, u := range x.Unheard {
|
||||
l.NotAnswering = append(l.NotAnswering, "the runtime "+sayUnheard([]unheard{u}))
|
||||
@@ -298,13 +335,8 @@ func indexOn(conn *bus.Conn) (*index, error) {
|
||||
sort.Strings(l.NotAnswering)
|
||||
x.NotAnswering = l.NotAnswering
|
||||
|
||||
var known []recordedMachine
|
||||
if json.Unmarshal([]byte(jsonIn(nodesOut)), &known) == nil {
|
||||
for _, n := range known {
|
||||
if n.Name != "" {
|
||||
machines[n.Name] = true
|
||||
}
|
||||
}
|
||||
for _, n := range known {
|
||||
machines[n] = true
|
||||
}
|
||||
for n := range machines {
|
||||
x.Machines = append(x.Machines, n)
|
||||
@@ -313,6 +345,138 @@ func indexOn(conn *bus.Conn) (*index, error) {
|
||||
return x, nil
|
||||
}
|
||||
|
||||
// stateOn reads a state the runtime's membership reads, whole: its current values by key, a deleted key absent.
|
||||
func stateOn(conn *bus.Conn, name string) (map[string][]byte, error) {
|
||||
var mu sync.Mutex
|
||||
keys := map[string][]byte{}
|
||||
stop, err := conn.StateWatch(conn.Credential().Module, name, "", func(c bus.StateChange) error {
|
||||
if c.Current && c.Op == "put" {
|
||||
mu.Lock()
|
||||
keys[c.Key] = c.Value
|
||||
mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stop()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// seatHoldersOn reads the state seat-holders whole (novox/hq ADR 0571 clause 4), as delivery.SeatHoldersRead.
|
||||
func seatHoldersOn(conn *bus.Conn) (map[string][]byte, error) {
|
||||
return stateOn(conn, delivery.SeatHoldersRead)
|
||||
}
|
||||
|
||||
// The indexer's states the console reads (novox/hq ADR 0571 clauses 4 and 5): every module, and each manifest by its
|
||||
// digest, by the names a reader's manifest declares.
|
||||
var (
|
||||
modulesRead = delivery.IndexerModule + ".modules"
|
||||
manifestsRead = delivery.IndexerModule + ".manifests"
|
||||
)
|
||||
|
||||
// manifestFacts is what the console takes of a manifest: the tools it declares and what each replaces. A manifest's
|
||||
// digest never names another, so what is read once is kept.
|
||||
type manifestFacts struct {
|
||||
Tools []string `json:"tools"`
|
||||
Replaces map[string][]string `json:"replaces"`
|
||||
}
|
||||
|
||||
var manifestsKept sync.Map // digest → manifestFacts
|
||||
|
||||
// recordsOn is every module the indexer holds, with where the placement seat places it and whether it declares tools,
|
||||
// and every machine placement names (delivery.ModuleNames, delivery.MachineNames). What cannot be read is said, one line
|
||||
// per state, for the listing's not answering.
|
||||
func recordsOn(conn *bus.Conn) (recorded []recordedModule, machines []string, why []string) {
|
||||
placements, err := stateOn(conn, delivery.PlacementRead)
|
||||
if err != nil {
|
||||
why = append(why, delivery.PlacementRead+" (where modules are placed could not be read, so what is missing cannot "+
|
||||
"be said: "+err.Error()+")")
|
||||
}
|
||||
machines = delivery.MachineNames(placements)
|
||||
on := map[string][]string{}
|
||||
// The placement's keys in either form, a node key and its module keys or the form before (research 072 row 37).
|
||||
assembled, err := delivery.PlacementsOf(placements)
|
||||
if err != nil {
|
||||
why = append(why, delivery.PlacementRead+" (not every key is a placement: "+err.Error()+")")
|
||||
}
|
||||
for _, node := range machines {
|
||||
for _, m := range assembled[node].Modules {
|
||||
on[m.Module] = append(on[m.Module], node)
|
||||
}
|
||||
}
|
||||
modules, err := stateOn(conn, modulesRead)
|
||||
if err != nil {
|
||||
return nil, machines, append(why, modulesRead+" (which modules there are could not be read: "+err.Error()+")")
|
||||
}
|
||||
names, err := delivery.ModuleNames(modules)
|
||||
if err != nil {
|
||||
return nil, machines, append(why, modulesRead+" ("+err.Error()+")")
|
||||
}
|
||||
for _, name := range names {
|
||||
r := recordedModule{Module: name, On: on[name]}
|
||||
if m, err := delivery.DecodeModule(modules[name]); err == nil && m.Manifest != "" {
|
||||
facts, err := manifestOf(conn, m.Manifest)
|
||||
if err != nil {
|
||||
why = append(why, name+" (its manifest could not be read, so whether it answers tools is not known: "+err.Error()+")")
|
||||
} else {
|
||||
r.Tools, r.Replaces = len(facts.Tools) > 0, facts.Replaces
|
||||
}
|
||||
}
|
||||
recorded = append(recorded, r)
|
||||
}
|
||||
return recorded, machines, why
|
||||
}
|
||||
|
||||
// manifestOf is what the console takes of the manifest of a digest, read from the indexer's manifests once.
|
||||
func manifestOf(conn *bus.Conn, digest string) (manifestFacts, error) {
|
||||
if kept, ok := manifestsKept.Load(digest); ok {
|
||||
return kept.(manifestFacts), nil
|
||||
}
|
||||
entry, err := conn.StateGet(conn.Credential().Module, manifestsRead, digest)
|
||||
if err != nil {
|
||||
return manifestFacts{}, err
|
||||
}
|
||||
if entry == nil {
|
||||
return manifestFacts{}, fmt.Errorf("the indexer holds no manifest %s", digest)
|
||||
}
|
||||
m, err := delivery.DecodeManifest(entry.Value)
|
||||
if err != nil {
|
||||
return manifestFacts{}, err
|
||||
}
|
||||
var facts manifestFacts
|
||||
if err := json.Unmarshal([]byte(m.JSON), &facts); err != nil {
|
||||
return manifestFacts{}, err
|
||||
}
|
||||
manifestsKept.Store(digest, facts)
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
// placedHolders is who holds seat (on node, for a seat held per machine) by placement, or none.
|
||||
func (x *index) placedHolders(seat, node string) []delivery.SeatHolder {
|
||||
if len(x.Placed) == 0 {
|
||||
return nil
|
||||
}
|
||||
hs, err := delivery.HoldersOf(x.Placed, seat, node)
|
||||
if err != nil || len(hs) > 0 || node != "" {
|
||||
return hs
|
||||
}
|
||||
// A seat held per machine, asked without one: every node's key of it.
|
||||
var all []delivery.SeatHolder
|
||||
for key := range x.Placed {
|
||||
if s, n, ok := strings.Cut(key, "."); ok && s == seat {
|
||||
if more, err := delivery.HoldersOf(x.Placed, seat, n); err == nil {
|
||||
all = append(all, more...)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool { return all[i].Node+"/"+all[i].Module < all[j].Node+"/"+all[j].Module })
|
||||
return all
|
||||
}
|
||||
|
||||
func containsHolder(hs []holder, h holder) bool {
|
||||
for _, x := range hs {
|
||||
if x == h {
|
||||
@@ -325,6 +489,10 @@ func containsHolder(hs []holder, h holder) bool {
|
||||
func (s *Surface) index() (*index, error) { return s.indexAsked(false) }
|
||||
|
||||
// indexAsked is what the mesh answered, asked again when `fresh` or when the kept answer is old.
|
||||
//
|
||||
// **A round that heard nothing is not kept** (novox/hq issue 554): the index it gives holds what every
|
||||
// runtime announced last, marked with its age, and the next question asks again rather than living
|
||||
// with it for IndexKept.
|
||||
func (s *Surface) indexAsked(fresh bool) (*index, error) {
|
||||
s.mu.Lock()
|
||||
if !fresh && s.idx != nil && time.Since(s.idxAt) <= IndexKept {
|
||||
@@ -333,13 +501,17 @@ func (s *Surface) indexAsked(fresh bool) (*index, error) {
|
||||
return x, nil
|
||||
}
|
||||
s.mu.Unlock()
|
||||
x, err := indexOn(s.conn)
|
||||
x, err := indexOn(s.conn, s.held, s.before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.remember(x)
|
||||
s.idx, s.idxAt = x, time.Now()
|
||||
s.sayRound(x)
|
||||
s.idx = x
|
||||
if !x.Discovery.Empty() {
|
||||
s.idxAt = time.Now()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return x, nil
|
||||
}
|
||||
@@ -450,6 +622,26 @@ type target struct {
|
||||
Node string
|
||||
Tool Tool
|
||||
Seat bool
|
||||
// Stale is set when the target is known only from an earlier round of discovery (novox/hq issue 554).
|
||||
Stale *staleFrom
|
||||
}
|
||||
|
||||
// staleOf is where a target was heard, when every place it answers is known only from an earlier round.
|
||||
func (x *index) staleOf(keys []string) *staleFrom {
|
||||
if len(keys) == 0 || len(x.Stale) == 0 {
|
||||
return nil
|
||||
}
|
||||
var first *staleFrom
|
||||
for _, k := range keys {
|
||||
f, ok := x.Stale[k]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if first == nil {
|
||||
first = &f
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// resolve turns an address into exactly one target, or says why it cannot.
|
||||
@@ -487,12 +679,17 @@ func resolve(x *index, address string) (target, error) {
|
||||
return target{}, fmt.Errorf("%s is held once per machine: write <node>/%s — it is held on %s",
|
||||
prefix, rest, orNobody(nodesOf(s.Holders)))
|
||||
}
|
||||
return target{Address: node + "/" + rest, Key: "seat:" + rest + "@" + node, Name: rest, Node: node, Tool: *verb, Seat: true}, nil
|
||||
return target{Address: node + "/" + rest, Key: "seat:" + rest + "@" + node, Name: rest, Node: node, Tool: *verb, Seat: true,
|
||||
Stale: x.staleOf([]string{"seat:" + prefix + "@" + node})}, nil
|
||||
}
|
||||
if node != "" {
|
||||
return target{}, fmt.Errorf("%s is held once for the whole mesh: write %s, without a machine", prefix, rest)
|
||||
}
|
||||
return target{Address: rest, Key: "seat:" + rest, Name: rest, Tool: *verb, Seat: true}, nil
|
||||
var holders []string
|
||||
for _, h := range s.Holders {
|
||||
holders = append(holders, "seat:"+prefix+"@"+h.Node)
|
||||
}
|
||||
return target{Address: rest, Key: "seat:" + rest, Name: rest, Tool: *verb, Seat: true, Stale: x.staleOf(holders)}, nil
|
||||
}
|
||||
|
||||
m := x.Modules[prefix]
|
||||
@@ -505,10 +702,23 @@ func resolve(x *index, address string) (target, error) {
|
||||
return target{}, fmt.Errorf("%s is assigned to %s in the mesh's records, but nothing that answered discovery serves it: %s. "+
|
||||
"Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, strings.Join(on, ", "), why)
|
||||
}
|
||||
if hs := x.placedHolders(prefix, node); len(hs) > 0 {
|
||||
var held []string
|
||||
for _, h := range hs {
|
||||
held = append(held, h.Module+" on "+h.Node)
|
||||
}
|
||||
return target{}, fmt.Errorf("the seat %s is held by %s by placement, but no holder of it announced itself on the bus. "+
|
||||
"Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, strings.Join(held, ", "))
|
||||
}
|
||||
if len(x.Unheard) > 0 {
|
||||
return target{}, fmt.Errorf("nothing that answered discovery is called %s, but not every runtime answered: %s — "+
|
||||
"%s may be theirs. Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, sayUnheard(x.Unheard), prefix)
|
||||
}
|
||||
if x.Discovery.Empty() {
|
||||
return target{}, fmt.Errorf("nothing answered discovery at all within %s, not even PING, and this console holds no "+
|
||||
"earlier answer: whether anything is called %s is not known. Ask again in a moment; mesh_runtimes shows which "+
|
||||
"runtimes answered", x.Discovery.Waited.Round(time.Millisecond), prefix)
|
||||
}
|
||||
return target{}, fmt.Errorf("nothing in the mesh is called %s: no seat, and no module that answers tools. "+
|
||||
"mesh_search finds a tool by words", prefix)
|
||||
}
|
||||
@@ -521,7 +731,11 @@ func resolve(x *index, address string) (target, error) {
|
||||
return target{}, fmt.Errorf("%s keeps state on each machine it runs on, so a call names the machine: "+
|
||||
"write <node>/%s — it runs on %s", prefix, rest, orNobody(m.On))
|
||||
}
|
||||
return target{Address: rest, Key: rest, Name: rest, Tool: *tool}, nil
|
||||
var on []string
|
||||
for _, n := range m.On {
|
||||
on = append(on, "mod:"+prefix+"@"+n)
|
||||
}
|
||||
return target{Address: rest, Key: rest, Name: rest, Tool: *tool, Stale: x.staleOf(on)}, nil
|
||||
}
|
||||
if len(m.On) > 0 && !contains(m.On, node) {
|
||||
if us := x.unheardOn(node); len(us) > 0 || contains(x.Recorded[prefix], node) {
|
||||
@@ -534,7 +748,8 @@ func resolve(x *index, address string) (target, error) {
|
||||
}
|
||||
return target{}, fmt.Errorf("%s does not run on %s; it runs on %s", prefix, node, orNobody(m.On))
|
||||
}
|
||||
return target{Address: node + "/" + rest, Key: rest + "@" + node, Name: rest, Node: node, Tool: *tool}, nil
|
||||
return target{Address: node + "/" + rest, Key: rest + "@" + node, Name: rest, Node: node, Tool: *tool,
|
||||
Stale: x.staleOf([]string{"mod:" + prefix + "@" + node})}, nil
|
||||
}
|
||||
|
||||
// unheardOnAny is what is unheard on any of these machines.
|
||||
@@ -707,10 +922,13 @@ func failure(text string) map[string]any {
|
||||
}
|
||||
|
||||
// discover answers one of the discovery verbs.
|
||||
func (s *Surface) discover(name string, args map[string]any) map[string]any {
|
||||
func (s *Surface) discover(ctx context.Context, name string, args map[string]any) map[string]any {
|
||||
if name == verbRuntimes {
|
||||
return s.runtimesAnswer()
|
||||
}
|
||||
if name == verbWait {
|
||||
return s.waitAnswer(ctx, args)
|
||||
}
|
||||
x, err := s.index()
|
||||
if err != nil {
|
||||
return failure("the mesh's discovery failed: " + err.Error())
|
||||
@@ -861,6 +1079,9 @@ func (s *Surface) discover(name string, args map[string]any) map[string]any {
|
||||
if r := x.Replaces[key]; len(r) > 0 {
|
||||
described["instead_of"] = r
|
||||
}
|
||||
if t.Stale != nil {
|
||||
described["found"] = t.Stale.say()
|
||||
}
|
||||
return answerText(described)
|
||||
|
||||
case verbCall:
|
||||
@@ -880,12 +1101,19 @@ func (s *Surface) discover(name string, args map[string]any) map[string]any {
|
||||
got, err = callTool(s.conn, t.Key, callArgs, nil, x.Listing)
|
||||
}
|
||||
if err != nil {
|
||||
return failure(whyItFailed(t.Address, err))
|
||||
why := whyItFailed(t.Address, err)
|
||||
if t.Stale != nil {
|
||||
why += " (" + t.Stale.say() + ")"
|
||||
}
|
||||
return failure(why)
|
||||
}
|
||||
content := []map[string]any{{"type": "text", "text": pretty(got.Result)}}
|
||||
if got.Node != "" {
|
||||
content = append(content, map[string]any{"type": "text", "text": "answered by " + got.Node})
|
||||
}
|
||||
if t.Stale != nil {
|
||||
content = append(content, map[string]any{"type": "text", "text": t.Stale.say()})
|
||||
}
|
||||
return map[string]any{"content": content}
|
||||
}
|
||||
return failure("no discovery verb " + name)
|
||||
|
||||
@@ -36,9 +36,10 @@ func call(t *testing.T, endpoint, tool string, args map[string]any) (string, boo
|
||||
return text(t, post(t, endpoint, string(body)))
|
||||
}
|
||||
|
||||
// novox/hq ADR 0195: the console announces six tools, and everything the mesh answers is reached
|
||||
// novox/hq ADR 0195 and ADR 0367: the console announces seven tools, and everything the mesh answers is reached
|
||||
// through them by one address per layer.
|
||||
func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
was := IndexKept
|
||||
IndexKept = 0 // every discovery asks the mesh, so a module arriving mid-test is found
|
||||
t.Cleanup(func() { IndexKept = was })
|
||||
@@ -69,29 +70,28 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
|
||||
// The controller: its records as JSON, as its seat verbs answer them, and its own seat's verbs
|
||||
// announced on the bus like every runtime's.
|
||||
// The controller: its own seat's verbs announced on the bus like every runtime's; its records are asked by no one.
|
||||
controller := connect(t, "mesh-controller", "bench")
|
||||
out := func(s string) map[string]any { return map[string]any{"output": s, "ok": true} }
|
||||
serve := func(verb string, answer func() any) {
|
||||
stop, err := controller.HandleSubject("mesh.seat.mesh-controller.tool."+verb, func(json.RawMessage) (any, error) {
|
||||
return answer(), nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
serve("nodes", func() any {
|
||||
return out("the bus's user list leaves out 2 user(s)\n" +
|
||||
`[{"name":"bench","heard":"3m ago","mode":"converged","id":"1f2e"},{"name":"desk","heard":"here","mode":"converged","id":"9a8b"}]`)
|
||||
// Its verb nodes, called through the console as any seat's verb is.
|
||||
stopNodes, err := controller.HandleSubject("mesh.seat.mesh-controller.tool.nodes", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"ok": true, "output": `[{"name":"bench","mode":"converged"},{"name":"desk","mode":"converged"}]`}, nil
|
||||
})
|
||||
var gammaOn atomic.Value
|
||||
gammaOn.Store("[]")
|
||||
serve("modules", func() any {
|
||||
return out(fmt.Sprintf(`[{"module":"alpha","on":["desk"],"tools":true},{"module":"beta","on":["desk"],"tools":true},`+
|
||||
`{"module":"gamma","on":%s,"tools":true},{"module":"delta","on":["desk"],"tools":false},`+
|
||||
`{"module":"epsilon","on":["bench"],"tools":true},{"module":"mesh-controller","on":["bench"],"tools":true}]`, gammaOn.Load().(string)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopNodes)
|
||||
// What the mesh holds, as the indexer and placement write it (ADR 0571 clauses 4, 5): never the controller's records.
|
||||
tools := map[string][]string{"alpha": {"one", "two"}, "beta": {"three"}, "gamma": {"given"}, "delta": nil,
|
||||
"epsilon": {"seven"}, "mesh-controller": {"nodes"}}
|
||||
mesh.Placed(t, tools, map[string][]string{"desk": {"alpha", "beta", "delta"}, "bench": {"epsilon", "mesh-controller"}})
|
||||
console := mt.MembershipOf("node-tools", "desk", false, nil)
|
||||
console.State = mt.ConsoleReads()
|
||||
mesh.Issue(t, console)
|
||||
mt.Until(t, func() error {
|
||||
if m := nodeTools.Membership("node-tools"); m == nil || len(m.State) == 0 {
|
||||
return fmt.Errorf("the console's reads are not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
stopAnn, err := announce.Serve(controller, announce.Service{Name: "mesh-controller", ID: "bench"}, func() []announce.Endpoint {
|
||||
return []announce.Endpoint{{Kind: announce.KindSeat, Module: "mesh-controller", Seat: "mesh-controller", Scope: "mesh",
|
||||
@@ -113,7 +113,7 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
defer up.Close()
|
||||
endpoint := "http://" + up.Address + "/mcp"
|
||||
|
||||
// Six tools, nothing else.
|
||||
// Seven tools, nothing else (ADR 0367).
|
||||
listed := post(t, endpoint, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`)["result"].(map[string]any)
|
||||
var names []string
|
||||
for _, x := range listed["tools"].([]any) {
|
||||
@@ -125,7 +125,7 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if got := strings.Join(names, ","); got != "mesh_overview,mesh_machine,mesh_search,mesh_describe,mesh_call,mesh_runtimes" {
|
||||
if got := strings.Join(names, ","); got != "mesh_overview,mesh_machine,mesh_search,mesh_describe,mesh_call,mesh_runtimes,mesh_wait" {
|
||||
t.Errorf("announced %s", got)
|
||||
}
|
||||
|
||||
@@ -201,7 +201,7 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
t.Fatalf("gamma was found before it served: %s", got)
|
||||
}
|
||||
mesh.Issue(t, mt.MembershipOf("gamma", "desk", false, nil))
|
||||
gammaOn.Store(`["desk"]`)
|
||||
mesh.Placed(t, tools, map[string][]string{"desk": {"alpha", "beta", "delta", "gamma"}, "bench": {"epsilon", "mesh-controller"}})
|
||||
late := connect(t, "node-tools", "desk")
|
||||
stopLate, err := runtime.Run(late, []runtime.Served{{Module: "gamma", Entrypoints: []string{mt.Fixture("env-gamma.serve.mjs")}}},
|
||||
nil, (&mt.Logs{}).Logf)
|
||||
@@ -232,12 +232,3 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
t.Errorf("an old name: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControllersRecordsAreReadAsJSON(t *testing.T) {
|
||||
if got := jsonIn("a warning printed first\n[{\"name\":\"ace\"}]"); got != `[{"name":"ace"}]` {
|
||||
t.Errorf("an array after a warning: %q", got)
|
||||
}
|
||||
if got := jsonIn(`{"seats":[]}`); got != `{"seats":[]}` {
|
||||
t.Errorf("an object: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
@@ -62,7 +64,7 @@ func toolKey(name string, seats Seats) string {
|
||||
// toolsOn is the flat catalogue (MESH_CONSOLE_FLAT=1): what announced itself on the bus, every
|
||||
// tool and seat verb, and the assignments with tools that did not (novox/hq ADR 0197).
|
||||
func toolsOn(conn *bus.Conn) (*Listing, error) {
|
||||
x, err := indexOn(conn)
|
||||
x, err := indexOn(conn, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -109,7 +111,6 @@ func subjectListed(name, node string, l *Listing) string {
|
||||
}
|
||||
|
||||
var (
|
||||
noResponders = regexp.MustCompile(`(?i)no responders|503`)
|
||||
// The bus's own two refusals, by their whole phrase. "authorization" alone also matched a tool's own
|
||||
// answer that merely mentions the word — the runtime refusing a state value with an Authorization
|
||||
// header (novox/hq ADR 0201) read as "this account may not call", which sent the reader the wrong way.
|
||||
@@ -127,7 +128,8 @@ func whyItFailed(key string, err error) string {
|
||||
}
|
||||
msg := err.Error()
|
||||
switch {
|
||||
case noResponders.MatchString(msg):
|
||||
// By type, not text: a tool's own error that mentions 503 is the tool's answer (issue 1529).
|
||||
case errors.Is(err, bus.ErrNoResponders) || errors.Is(err, nats.ErrNoResponders):
|
||||
extra := ""
|
||||
if strings.HasPrefix(key, "seat:") {
|
||||
extra = ", or nothing holds that seat"
|
||||
|
||||
@@ -3,6 +3,7 @@ package console
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -42,6 +43,7 @@ func post(t *testing.T, endpoint string, body string) map[string]any {
|
||||
// As node-tools, the runtime serves the bundles and is the console on loopback: the listing is what
|
||||
// the modules and the mesh's records answered, and a call reaches the module on the machine named.
|
||||
func TestTheConsoleListsAndCallsOverHTTP(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "desk", true, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "desk", false, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
@@ -55,14 +57,19 @@ func TestTheConsoleListsAndCallsOverHTTP(t *testing.T) {
|
||||
}
|
||||
defer stop()
|
||||
|
||||
// The controller's records (ADR 0197): ghost declares tools on desk and nothing announces it.
|
||||
controller := connect(t, "mesh-controller", "")
|
||||
stopSeat, _ := controller.HandleSubject("mesh.seat.mesh-controller.tool.modules", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"ok": true, "output": `[{"module":"alpha","on":["desk"],"tools":true},` +
|
||||
`{"module":"beta","on":["desk"],"tools":true},{"module":"ghost","on":["desk"],"tools":true}]`}, nil
|
||||
// What the mesh holds, as the indexer and placement write it (ADR 0571): ghost declares tools on desk and nothing
|
||||
// announces it.
|
||||
mesh.Placed(t, map[string][]string{"alpha": {"one", "two"}, "beta": {"three"}, "ghost": {"boo"}},
|
||||
map[string][]string{"desk": {"alpha", "beta", "ghost"}})
|
||||
console := mt.MembershipOf("node-tools", "desk", false, nil)
|
||||
console.State = mt.ConsoleReads()
|
||||
mesh.Issue(t, console)
|
||||
mt.Until(t, func() error {
|
||||
if m := nodeTools.Membership("node-tools"); m == nil || len(m.State) == 0 {
|
||||
return fmt.Errorf("the console's reads are not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
defer stopSeat()
|
||||
controller.Flush()
|
||||
|
||||
flat := NewSurface(nodeTools, "desk.node-tools")
|
||||
flat.Flat = true // the whole catalogue, as before ADR 0195: still reachable, no longer announced
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq issue 554, reproduced: the laptop's console heard every runtime, then a round in which
|
||||
// nothing answered discovery in time — and every seat and module call was refused ("it answered
|
||||
// discovery 2s ago, not now") for the five seconds that round was kept, again and again. Now a call
|
||||
// is resolved against what the runtime announced last, says how old that is, and reaches the tool; the
|
||||
// empty round is logged once, is not kept, and the next question asks again.
|
||||
func TestACallAfterARoundThatHeardNothingUsesWhatWasHeardBefore(t *testing.T) {
|
||||
keptWas, patienceWas := IndexKept, announce.Patience
|
||||
IndexKept, announce.Patience = 0, 1200*time.Millisecond
|
||||
t.Cleanup(func() { IndexKept, announce.Patience = keptWas, patienceWas })
|
||||
mt.New(t)
|
||||
|
||||
// The tools answer on their own connection; discovery is answered on another, which goes quiet.
|
||||
tools := connect(t, "node-tools", "laptop")
|
||||
stopTool, err := tools.HandleSubject("mesh.mod.slack.tool.slack_check.laptop", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"ok": true}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopTool)
|
||||
stopSeat, err := tools.HandleSubject("mesh.seat.issue-tracker.tool.list", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"issues": []int{554}}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopSeat)
|
||||
tools.Flush()
|
||||
endpoints := []announce.Endpoint{
|
||||
{Kind: announce.KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Subject: "mesh.mod.slack.tool.slack_check.laptop"},
|
||||
{Kind: announce.KindSeat, Module: "issue-tracker", Seat: "issue-tracker", Tool: "list", Scope: "mesh", Node: "laptop",
|
||||
Schema: json.RawMessage(`{}`), Subject: "mesh.seat.issue-tracker.tool.list"},
|
||||
}
|
||||
announcing := connect(t, "node-tools", "laptop")
|
||||
stopAnnounce, err := announce.Serve(announcing, announce.Service{Name: "node-tools", ID: "laptop",
|
||||
Metadata: map[string]string{"node": "laptop"}}, func() []announce.Endpoint { return endpoints })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
announcing.Flush()
|
||||
|
||||
console := connect(t, "node-tools", "laptop-console")
|
||||
var mu sync.Mutex
|
||||
var logged []string
|
||||
console.Logf = func(f string, a ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
logged = append(logged, fmt.Sprintf(f, a...))
|
||||
}
|
||||
said := func(words string) int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range logged {
|
||||
if strings.Contains(l, words) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
surface := NewSurface(console, "laptop.node-tools")
|
||||
up, err := Serve(surface, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer up.Close()
|
||||
endpoint := "http://" + up.Address + "/mcp"
|
||||
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/slack.slack_check"}); isErr || strings.Contains(got, "announced") {
|
||||
t.Fatalf("while discovery hears the runtime: %s", got)
|
||||
}
|
||||
|
||||
// Discovery goes quiet: no PING arrives at all.
|
||||
stopAnnounce()
|
||||
announcing.Flush()
|
||||
for _, address := range []string{"laptop/slack.slack_check", "issue-tracker.list"} {
|
||||
got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": address})
|
||||
if isErr || !strings.Contains(got, "found from what node-tools on laptop announced") ||
|
||||
!strings.Contains(got, "ago: it did not answer the latest round of discovery") {
|
||||
t.Errorf("%s after a round that heard nothing: %s", address, got)
|
||||
}
|
||||
}
|
||||
if got, isErr := call(t, endpoint, "mesh_describe", map[string]any{"address": "issue-tracker.list"}); isErr || !strings.Contains(got, `"found"`) {
|
||||
t.Errorf("describe after a round that heard nothing: %s", got)
|
||||
}
|
||||
if n := said("discovery heard nothing at all"); n != 1 {
|
||||
t.Errorf("the empty round was logged %d time(s), want once: %v", n, logged)
|
||||
}
|
||||
// A runtime really gone: the call is sent on what it announced, and fails with the bus's own refusal,
|
||||
// saying where the address was found.
|
||||
stopTool()
|
||||
tools.Flush()
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/slack.slack_check"}); !isErr ||
|
||||
!strings.Contains(got, "nothing serves") || !strings.Contains(got, "found from what node-tools on laptop announced") {
|
||||
t.Errorf("a gone runtime's tool: %s", got)
|
||||
}
|
||||
surface.mu.Lock()
|
||||
kept := surface.idx != nil && !surface.idx.Discovery.Empty() && time.Since(surface.idxAt) < time.Second
|
||||
surface.mu.Unlock()
|
||||
if kept {
|
||||
t.Error("a round that heard nothing was kept as the answer")
|
||||
}
|
||||
|
||||
// Discovery answers again: the calls are plain again, and that is logged once.
|
||||
stopAgain, err := announce.Serve(announcing, announce.Service{Name: "node-tools", ID: "laptop",
|
||||
Metadata: map[string]string{"node": "laptop"}}, func() []announce.Endpoint { return endpoints })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopAgain)
|
||||
announcing.Flush()
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "issue-tracker.list"}); isErr || strings.Contains(got, "announced") {
|
||||
t.Errorf("once discovery hears the runtime again: %s", got)
|
||||
}
|
||||
if n := said("discovery hears every runtime again"); n != 1 {
|
||||
t.Errorf("the recovery was logged %d time(s): %v", n, logged)
|
||||
}
|
||||
}
|
||||
|
||||
// An address known only from an earlier answer says how old it is; one heard fresh on its machine is
|
||||
// plain, even where the same module is known only from an earlier answer on another.
|
||||
func TestAnAddressKnownOnlyFromAnEarlierAnswerSaysHowOld(t *testing.T) {
|
||||
at := time.Now().Add(-20 * time.Second)
|
||||
x := &index{Modules: map[string]*moduleInfo{"slack": {Module: "slack", On: []string{"laptop"},
|
||||
Tools: []Tool{{Module: "slack", Name: "slack_check"}}}}, Recorded: map[string][]string{},
|
||||
Stale: map[string]staleFrom{"mod:slack@laptop": {Runtime: "node-tools", Machine: "laptop", At: at}},
|
||||
Discovery: announce.Discovery{Pinged: 0}}
|
||||
got, err := resolve(x, "laptop/slack.slack_check")
|
||||
if err != nil || got.Stale == nil || !strings.Contains(got.Stale.say(), "announced 20s ago") {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
// A module heard fresh on one machine and only earlier on another: the fresh one is plain.
|
||||
x.Modules["slack"].On = []string{"desk", "laptop"}
|
||||
if got, err := resolve(x, "desk/slack.slack_check"); err != nil || got.Stale != nil {
|
||||
t.Errorf("a fresh machine's module marked stale: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/wire"
|
||||
)
|
||||
@@ -45,12 +46,30 @@ func Serve(s *Surface, listen string) (*Listening, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route(w, r, s) })}
|
||||
// Every request's context ends when the console is closed, so a mesh_wait in flight ends at once rather
|
||||
// than holding a stop or a restart for up to its ten minutes (the review of issue 745).
|
||||
base, cancel := context.WithCancel(context.Background())
|
||||
server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route(w, r, s) }),
|
||||
BaseContext: func(net.Listener) context.Context { return base }}
|
||||
go func() { _ = server.Serve(ln) }()
|
||||
bound := ln.Addr().(*net.TCPAddr).Port
|
||||
return &Listening{Address: host + ":" + strconv.Itoa(bound), Close: func() error { return server.Shutdown(context.Background()) }}, nil
|
||||
return &Listening{Address: host + ":" + strconv.Itoa(bound), Close: func() error {
|
||||
cancel()
|
||||
// A backstop: what is still answering after the cancel is cut off, never waited on for long.
|
||||
ctx, done := context.WithTimeout(context.Background(), ShutdownWithin)
|
||||
defer done()
|
||||
err := server.Shutdown(ctx)
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return server.Close()
|
||||
}
|
||||
return err
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// ShutdownWithin is how long closing the console waits for the requests still being answered after their
|
||||
// context was cancelled.
|
||||
var ShutdownWithin = 5 * time.Second
|
||||
|
||||
func route(w http.ResponseWriter, r *http.Request, s *Surface) {
|
||||
switch r.URL.Path {
|
||||
case "/":
|
||||
@@ -89,7 +108,7 @@ func route(w http.ResponseWriter, r *http.Request, s *Surface) {
|
||||
}
|
||||
replies := []*Reply{}
|
||||
for _, req := range batch {
|
||||
if reply := s.Handle(req); reply != nil {
|
||||
if reply := s.HandleContext(r.Context(), req); reply != nil {
|
||||
replies = append(replies, reply)
|
||||
}
|
||||
}
|
||||
@@ -105,7 +124,7 @@ func route(w http.ResponseWriter, r *http.Request, s *Surface) {
|
||||
writeJSON(w, 400, map[string]any{"jsonrpc": "2.0", "id": nil, "error": map[string]any{"code": -32700, "message": "the body is not JSON"}})
|
||||
return
|
||||
}
|
||||
reply := s.Handle(req)
|
||||
reply := s.HandleContext(r.Context(), req)
|
||||
if reply == nil {
|
||||
w.WriteHeader(202)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0367 §5: the tool runner's manifest names the families a session waits on today, one by one,
|
||||
// and the asker's `decided` for the claude-code module alone. Adding one is a reviewed change to this file
|
||||
// (§6), so this list changes with it. The delivery's transitions are waited on under the gate's name alone (ADR 0450 §8, issues 1350
|
||||
// and 1406): today's module is forgotten.
|
||||
func TestTheManifestNamesTheFirstFamiliesOneByOne(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Waits []string `json:"waits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"mesh-knowledge.reviewed", "mesh-delivery-gate.transition", "gitea.forge.status",
|
||||
"mesh-self-check.checked", "operator-channel.decided.claude-code"}
|
||||
if m.Module != "node-tools" || !reflect.DeepEqual(m.Waits, want) {
|
||||
t.Errorf("%s waits on %v, want %v", m.Module, m.Waits, want)
|
||||
}
|
||||
for _, w := range m.Waits {
|
||||
if strings.ContainsAny(w, "*>") {
|
||||
t.Errorf("a family named by a wildcard: %s", w)
|
||||
}
|
||||
}
|
||||
if len(m.Consumes) > 0 {
|
||||
t.Errorf("the tool runner consumes %v: a wait is no consumer (ADR 0367 §2)", m.Consumes)
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,10 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -52,11 +55,41 @@ type Surface struct {
|
||||
idxAt time.Time
|
||||
// runtimes is every runtime discovery has heard, as it last heard it (mesh_runtimes).
|
||||
runtimes map[string]*runtimeSeen
|
||||
// roundSaid is what the last round logged was missing, so a round is logged once (novox/hq issue 554).
|
||||
roundSaid string
|
||||
// Flat announces the whole catalogue, as the console did before ADR 0195: for a person reading
|
||||
// it or a client that wants it. Off by default; MESH_CONSOLE_FLAT=1 turns it on.
|
||||
Flat bool
|
||||
}
|
||||
|
||||
// MethodSeats is the console's JSON-RPC method beside MCP's that answers every seat's verbs and holders, in the shape
|
||||
// the controller's `tools` answered: `{seats: [{seat, scope, tools: [{name, description, input, replaces}], holders:
|
||||
// [{module, node}]}], holders_why}`.
|
||||
const MethodSeats = "mesh/seats"
|
||||
|
||||
// seatsAnswer is MethodSeats' answer: mesh-sdk's seat definitions, with each seat's holders by placement.
|
||||
func (s *Surface) seatsAnswer() map[string]any {
|
||||
placed, err := seatHoldersOn(s.conn)
|
||||
x := &index{Placed: placed}
|
||||
seats := []map[string]any{}
|
||||
for _, st := range catalogue.DefaultSeats() {
|
||||
holders := x.placedHolders(st.Name, "")
|
||||
if holders == nil {
|
||||
holders = []delivery.SeatHolder{}
|
||||
}
|
||||
tools := st.Serves
|
||||
if tools == nil {
|
||||
tools = []catalogue.Verb{}
|
||||
}
|
||||
seats = append(seats, map[string]any{"seat": st.Name, "scope": st.Scope, "tools": tools, "holders": holders})
|
||||
}
|
||||
out := map[string]any{"seats": seats}
|
||||
if err != nil {
|
||||
out["holders_why"] = "who holds which seat by placement could not be read: " + err.Error()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NewSurface is the surface over a connection, as `who`.
|
||||
func NewSurface(conn *bus.Conn, who string) *Surface {
|
||||
return &Surface{conn: conn, who: who, Flat: os.Getenv("MESH_CONSOLE_FLAT") == "1"}
|
||||
@@ -101,7 +134,11 @@ func idOrNull(id json.RawMessage) json.RawMessage {
|
||||
}
|
||||
|
||||
// Handle answers one request; nil for a notification, which expects none.
|
||||
func (s *Surface) Handle(r Request) *Reply {
|
||||
func (s *Surface) Handle(r Request) *Reply { return s.HandleContext(context.Background(), r) }
|
||||
|
||||
// HandleContext answers one request for as long as ctx lasts: a wait whose client hung up or cancelled
|
||||
// ends at once, its subscription closed (the review of issue 745).
|
||||
func (s *Surface) HandleContext(ctx context.Context, r Request) *Reply {
|
||||
notification := isNotification(r.ID)
|
||||
switch r.Method {
|
||||
case "initialize":
|
||||
@@ -118,6 +155,11 @@ func (s *Surface) Handle(r Request) *Reply {
|
||||
return nil
|
||||
}
|
||||
return answer(r.ID, map[string]any{})
|
||||
case MethodSeats:
|
||||
// The seats' contract and who holds each, for a client on this machine that has no seat definitions of its own
|
||||
// (the TypeScript `mesh`): read in-process from mesh-sdk and from placement, never asked of the controller
|
||||
// (novox/hq ADR 0571 clauses 4 and 5).
|
||||
return answer(r.ID, s.seatsAnswer())
|
||||
case "tools/list":
|
||||
if !s.Flat {
|
||||
return answer(r.ID, map[string]any{"tools": discovery()})
|
||||
@@ -162,7 +204,7 @@ func (s *Surface) Handle(r Request) *Reply {
|
||||
if p.Arguments == nil {
|
||||
p.Arguments = map[string]any{}
|
||||
}
|
||||
return answer(r.ID, s.discover(p.Name, p.Arguments))
|
||||
return answer(r.ID, s.discover(ctx, p.Name, p.Arguments))
|
||||
}
|
||||
args := map[string]any{}
|
||||
for k, v := range p.Arguments {
|
||||
@@ -303,7 +345,8 @@ func (s *Surface) instructions() string {
|
||||
"whole (novox/hq ADR 0195). mesh_overview shows the mesh's seats and machines; mesh_machine one " +
|
||||
"machine's seats and modules; mesh_search finds a tool by words; mesh_describe gives one tool's " +
|
||||
"arguments; mesh_call calls it; mesh_runtimes says which machines' runtimes answered discovery, how fast " +
|
||||
"and how much — ask it when an address is said to be missing or on another machine. " + grammar + " What may be called was fixed when this account " +
|
||||
"and how much — ask it when an address is said to be missing or on another machine; mesh_wait waits on the " +
|
||||
"bus for an event instead of polling (novox/hq ADR 0367). " + grammar + " What may be called was fixed when this account " +
|
||||
"was issued, so a refusal means the account, not the tool."
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// A tool's own error that mentions 503 is the tool's answer, not a routing fault (issue 1529).
|
||||
func TestAToolsOwn503IsNotNothingServes(t *testing.T) {
|
||||
got := whyItFailed("scanner.scan", errors.New("the scanner refused the scan job: 503 Service Unavailable"))
|
||||
if strings.Contains(got, "nothing serves") || !strings.HasPrefix(got, "scanner.scan failed: ") || !strings.Contains(got, "503") {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARealNoRespondersStillSaysNothingServes(t *testing.T) {
|
||||
for _, err := range []error{bus.ErrNoResponders, fmt.Errorf("ask: %w", bus.ErrNoResponders), nats.ErrNoResponders} {
|
||||
if got := whyItFailed("a.b", err); !strings.HasPrefix(got, "nothing serves a.b") {
|
||||
t.Fatalf("%v: got %q", err, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,9 @@ package console
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
@@ -31,6 +33,11 @@ type runtimeSeen struct {
|
||||
Shortened string
|
||||
Answered bool // in the latest round
|
||||
Why string // why not, when it did not
|
||||
// Reused is true when its PING said its announcement was unchanged, so it was not fetched again.
|
||||
Reused bool
|
||||
// Last is its last whole answer: what a round that does not hear it uses instead, and what an
|
||||
// unchanged digest stands for (novox/hq issue 554).
|
||||
Last announce.Heard
|
||||
}
|
||||
|
||||
func runtimeKey(name, id string) string { return name + "/" + id }
|
||||
@@ -50,7 +57,108 @@ func seenOf(h announce.Heard, at time.Time) *runtimeSeen {
|
||||
}
|
||||
}
|
||||
return &runtimeSeen{Name: h.Info.Name, ID: h.Info.ID, Machine: h.Machine(), LastHeard: at, Took: h.Took,
|
||||
Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true}
|
||||
Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true,
|
||||
Reused: h.Reused, Last: h}
|
||||
}
|
||||
|
||||
// held is the answer this console holds for an instance at this digest (announce.Known): an instance
|
||||
// whose announcement is unchanged is not asked for it again (novox/hq issue 554).
|
||||
func (s *Surface) held(name, id, digest string) (announce.Heard, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
r := s.runtimes[runtimeKey(name, id)]
|
||||
if r == nil || r.Last.Digest == "" || r.Last.Digest != digest || r.Last.Info.Name == "" {
|
||||
return announce.Heard{}, false
|
||||
}
|
||||
h := r.Last
|
||||
h.Took, h.Reused = 0, false
|
||||
return h, true
|
||||
}
|
||||
|
||||
// before is the last answer of every runtime this round did not hear, heard within RuntimeForgotten
|
||||
// and not since restarted under another instance (novox/hq issue 554).
|
||||
func (s *Surface) before(d announce.Discovery) []earlier {
|
||||
heard := map[string]bool{}
|
||||
running := map[string]bool{}
|
||||
for _, h := range d.Heard {
|
||||
heard[runtimeKey(h.Info.Name, h.Info.ID)] = true
|
||||
if m := h.Machine(); m != "" {
|
||||
running[h.Info.Name+"@"+m] = true
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []earlier
|
||||
for k, r := range s.runtimes {
|
||||
if heard[k] || r.Last.Info.Name == "" || r.LastHeard.IsZero() || time.Since(r.LastHeard) > RuntimeForgotten {
|
||||
continue
|
||||
}
|
||||
if r.Machine != "" && running[r.Name+"@"+r.Machine] {
|
||||
continue // restarted: the new instance answered
|
||||
}
|
||||
out = append(out, earlier{Heard: r.Last, At: r.LastHeard})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return runtimeKey(out[i].Heard.Info.Name, out[i].Heard.Info.ID) < runtimeKey(out[j].Heard.Info.Name, out[j].Heard.Info.ID)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// sayRound logs a round of discovery that did not hear every runtime — once, when what it missed
|
||||
// changes, and once when every runtime is heard again — so the next case leaves evidence in the
|
||||
// journal (novox/hq issue 554). Called with s.mu held, after remember.
|
||||
func (s *Surface) sayRound(x *index) {
|
||||
d := x.Discovery
|
||||
var missed []string
|
||||
for _, u := range x.Unheard {
|
||||
who := u.Runtime
|
||||
if u.Machine != "" {
|
||||
who += " on " + u.Machine
|
||||
}
|
||||
missed = append(missed, who)
|
||||
}
|
||||
sort.Strings(missed)
|
||||
shape := strings.Join(missed, ", ")
|
||||
if d.Empty() {
|
||||
shape = "empty|" + shape
|
||||
}
|
||||
if shape == s.roundSaid {
|
||||
return
|
||||
}
|
||||
s.roundSaid = shape
|
||||
reused := 0
|
||||
for _, h := range d.Heard {
|
||||
if h.Reused {
|
||||
reused++
|
||||
}
|
||||
}
|
||||
logf := log.Printf
|
||||
if s.conn != nil && s.conn.Logf != nil {
|
||||
logf = s.conn.Logf
|
||||
}
|
||||
window := fmt.Sprintf("waited %s (window %s, patience %s)", d.Waited.Round(time.Millisecond), announce.Window, announce.Patience)
|
||||
held := ""
|
||||
if len(x.Earlier) > 0 {
|
||||
held = fmt.Sprintf("; what %d of them announced earlier is used, with its age", len(x.Earlier))
|
||||
}
|
||||
switch {
|
||||
case d.Empty():
|
||||
logf("[mesh-tools] discovery heard nothing at all: no PING arrived; %s; not heard: %s%s; asking again at the next question",
|
||||
window, orNone(missed), held)
|
||||
case len(missed) > 0:
|
||||
logf("[mesh-tools] discovery did not hear every runtime: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s; not heard: %s%s",
|
||||
d.Pinged, len(d.Heard), reused, window, shape, held)
|
||||
default:
|
||||
logf("[mesh-tools] discovery hears every runtime again: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s",
|
||||
d.Pinged, len(d.Heard), reused, window)
|
||||
}
|
||||
}
|
||||
|
||||
func orNone(xs []string) string {
|
||||
if len(xs) == 0 {
|
||||
return "none heard before either"
|
||||
}
|
||||
return strings.Join(xs, ", ")
|
||||
}
|
||||
|
||||
// remember records a round of discovery and adds to the index every runtime heard lately that did not
|
||||
@@ -101,14 +209,25 @@ func (s *Surface) remember(x *index) {
|
||||
continue
|
||||
}
|
||||
r.Answered = false
|
||||
used := ""
|
||||
if at, ok := x.Earlier[k]; ok {
|
||||
used = fmt.Sprintf("; what it announced %s ago is used", time.Since(at).Round(time.Second))
|
||||
}
|
||||
if why, ok := silent[k]; ok {
|
||||
r.Why = why
|
||||
if used != "" {
|
||||
for i := range x.Unheard {
|
||||
if x.Unheard[i].Runtime == r.Name && x.Unheard[i].Machine == r.Machine && x.Unheard[i].Why == why {
|
||||
x.Unheard[i].Why += used
|
||||
}
|
||||
}
|
||||
}
|
||||
continue // already in the index's unheard
|
||||
}
|
||||
r.Why = "it did not answer this round of discovery"
|
||||
if time.Since(r.LastHeard) <= RuntimeForgotten {
|
||||
x.Unheard = append(x.Unheard, unheard{Runtime: r.Name, Machine: r.Machine,
|
||||
Why: fmt.Sprintf("it answered discovery %s ago, not now", time.Since(r.LastHeard).Round(time.Second))})
|
||||
Why: fmt.Sprintf("it answered discovery %s ago, not now%s", time.Since(r.LastHeard).Round(time.Second), used)})
|
||||
}
|
||||
}
|
||||
for _, u := range x.Discovery.Silent {
|
||||
@@ -157,6 +276,9 @@ func (s *Surface) runtimesAnswer() map[string]any {
|
||||
if r.Shortened != "" {
|
||||
entry["shortened, how"] = r.Shortened
|
||||
}
|
||||
if r.Reused && r.Answered {
|
||||
entry["announcement"] = "unchanged (its PING's digest), so not fetched again; took is its PING's"
|
||||
}
|
||||
}
|
||||
if r.Answered {
|
||||
onMachine[r.Machine] = true
|
||||
@@ -184,7 +306,7 @@ func (s *Surface) runtimesAnswer() map[string]any {
|
||||
return answerText(map[string]any{
|
||||
"asked": x.Discovery.At.Format(time.RFC3339),
|
||||
"waited": x.Discovery.Waited.Round(time.Millisecond).String(),
|
||||
"window": fmt.Sprintf("at least %s; up to %s for a runtime that answered PING", announce.Window, announce.Patience),
|
||||
"window": fmt.Sprintf("at least %s; up to %s for a runtime that answered PING, or until one does", announce.Window, announce.Patience),
|
||||
"answered": answered,
|
||||
"not heard": notHeard,
|
||||
})
|
||||
|
||||
@@ -2,10 +2,13 @@ package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
@@ -23,18 +26,22 @@ func slowRuntime(t *testing.T, conn *bus.Conn, node string, lag time.Duration, e
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
stop, err = conn.Raw("$SRV.INFO", func(string, []byte) []byte {
|
||||
// Asked of every service, and of this instance by name and id — as discovery asks it (issue 554).
|
||||
info := func(string, []byte) []byte {
|
||||
if lag < 0 {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(lag)
|
||||
body, _ := json.Marshal(announce.Info(announce.Service{Name: id.Name, ID: id.ID, Metadata: id.Metadata}, endpoints))
|
||||
return body
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO." + id.Name + "." + id.ID} {
|
||||
stop, err := conn.Raw(subject, info)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
conn.Flush()
|
||||
}
|
||||
|
||||
@@ -48,7 +55,23 @@ func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T)
|
||||
// Kept for the test: one round of discovery answers every question but mesh_runtimes, which asks anew.
|
||||
IndexKept, announce.Patience = time.Minute, 2500*time.Millisecond
|
||||
t.Cleanup(func() { IndexKept, announce.Patience = keptWas, patienceWas })
|
||||
mt.New(t)
|
||||
mesh := mt.New(t)
|
||||
// What the mesh holds, as the indexer and placement write it (ADR 0571): never the controller's records.
|
||||
mesh.Placed(t, map[string][]string{"slack": {"slack_check"}, "nap": {"doze"}, "nextcloud-client": {"nextcloud_check"}},
|
||||
map[string][]string{"laptop": {"slack", "nextcloud-client"}, "sleeper": {"nap", "nextcloud-client"},
|
||||
"desktop": {"nextcloud-client"}, "bench": nil})
|
||||
// Who holds which seat, by placement (ADR 0571 clause 4), as the seat mesh-placement's holder composes it.
|
||||
placedKV := mesh.Bucket(t, delivery.SeatHoldersBucket)
|
||||
for key, sh := range delivery.SeatHoldersOf(map[string]delivery.Placement{"bench": {Node: "bench",
|
||||
Holds: []delivery.Holding{{Claim: "mesh-wants", Scope: "mesh", Module: "mesh-delivery-assigner"}}}}) {
|
||||
raw, _ := json.Marshal(sh)
|
||||
if _, err := placedKV.Put(key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
consoleMembership := mt.MembershipOf("node-tools", "laptop-console", false, nil)
|
||||
consoleMembership.State = mt.ConsoleReads()
|
||||
mesh.Issue(t, consoleMembership)
|
||||
|
||||
laptop := connect(t, "node-tools", "laptop")
|
||||
stopTool, err := laptop.HandleSubject("mesh.mod.slack.tool.slack_check.laptop", func(json.RawMessage) (any, error) {
|
||||
@@ -73,13 +96,16 @@ func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T)
|
||||
})
|
||||
|
||||
controller := connect(t, "mesh-controller", "bench")
|
||||
var toolsAsked atomic.Int32
|
||||
for verb, output := range map[string]string{
|
||||
"nodes": `[{"name":"laptop"},{"name":"sleeper"},{"name":"desktop"},{"name":"bench"}]`,
|
||||
"modules": `[{"module":"slack","on":["laptop"],"tools":true},{"module":"nap","on":["sleeper"],"tools":true},` +
|
||||
`{"module":"nextcloud-client","on":["laptop","desktop","sleeper"],"tools":true}]`,
|
||||
// The controller's records are served here only to see that none is asked (ADR 0571 clauses 4 and 5).
|
||||
"nodes": `[{"name":"laptop"}]`,
|
||||
"modules": `[]`,
|
||||
"tools": `{"seats":[]}`,
|
||||
} {
|
||||
output := output
|
||||
output, verb := output, verb
|
||||
stop, err := controller.HandleSubject("mesh.seat.mesh-controller.tool."+verb, func(json.RawMessage) (any, error) {
|
||||
toolsAsked.Add(1)
|
||||
return map[string]any{"output": output, "ok": true}, nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -90,6 +116,13 @@ func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T)
|
||||
controller.Flush()
|
||||
|
||||
console := connect(t, "node-tools", "laptop-console")
|
||||
console.Follow("node-tools")
|
||||
mt.Until(t, func() error {
|
||||
if console.Membership("node-tools") == nil {
|
||||
return fmt.Errorf("the console's membership is not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
up, err := Serve(NewSurface(console, "laptop.node-tools"), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -109,8 +142,20 @@ func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T)
|
||||
t.Errorf("the late runtime's machine lists nothing: %s", machine)
|
||||
}
|
||||
|
||||
// The seats' contract and holders, for a client on this machine with no seat definitions (the TypeScript `mesh`).
|
||||
seats := post(t, endpoint, `{"jsonrpc":"2.0","id":9,"method":"mesh/seats"}`)
|
||||
if raw, _ := json.Marshal(seats["result"]); !strings.Contains(string(raw), `"holders":[{"module":"mesh-delivery-assigner","node":"bench"}],"scope":"mesh","seat":"mesh-wants"`) ||
|
||||
!strings.Contains(string(raw), `"replaces":["journalctl"]`) {
|
||||
t.Errorf("mesh/seats answered %s", raw)
|
||||
}
|
||||
if n := toolsAsked.Load(); n != 0 {
|
||||
t.Errorf("the console asked the controller's records %d times", n)
|
||||
}
|
||||
|
||||
// The silent one is named, never called missing.
|
||||
for address, want := range map[string]string{
|
||||
// A seat whose holder announced nothing is named with its holder by placement, never called missing.
|
||||
"mesh-wants.read": "the seat mesh-wants is held by mesh-delivery-assigner on bench by placement",
|
||||
"sleeper/nap.doze": "nap is assigned to sleeper in the mesh's records, but nothing that answered discovery serves it: discovery did not hear in full from node-tools on sleeper (it answered PING",
|
||||
"sleeper/nextcloud-client.nextcloud_check": "nextcloud-client on sleeper did not answer discovery: discovery did not hear in full from node-tools on sleeper",
|
||||
} {
|
||||
@@ -175,7 +220,7 @@ func TestARuntimeHeardBeforeAndNotNowIsNamed(t *testing.T) {
|
||||
s.remember(before)
|
||||
now := &index{Modules: map[string]*moduleInfo{"nextcloud-client": {Module: "nextcloud-client", On: []string{"desktop"},
|
||||
Tools: []Tool{{Module: "nextcloud-client", Name: "nextcloud_check"}}}}, Recorded: map[string][]string{},
|
||||
Discovery: announce.Discovery{At: time.Now().UTC()}}
|
||||
Discovery: announce.Discovery{At: time.Now().UTC(), Pinged: 1}} // desktop's runtime answered
|
||||
s.remember(now)
|
||||
if len(now.Unheard) != 1 || now.Unheard[0].Machine != "laptop" {
|
||||
t.Fatalf("unheard: %+v", now.Unheard)
|
||||
@@ -190,7 +235,7 @@ func TestARuntimeHeardBeforeAndNotNowIsNamed(t *testing.T) {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
// Nobody unheard: the plain answers stand.
|
||||
quiet := &index{Modules: now.Modules, Recorded: map[string][]string{}}
|
||||
quiet := &index{Modules: now.Modules, Recorded: map[string][]string{}, Discovery: now.Discovery}
|
||||
if _, err := resolve(quiet, "laptop/nextcloud-client.nextcloud_check"); err == nil || !strings.Contains(err.Error(), "does not run on laptop; it runs on desktop") {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
|
||||
@@ -19,7 +19,8 @@ package console
|
||||
// Ties go to a seat — the place tools belong — then to the shorter name, then by address.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -44,6 +45,31 @@ var synonyms = map[string][]string{
|
||||
"hosts": {"hostname"},
|
||||
}
|
||||
|
||||
// filler are the function words a question carries and no tool is named for: a query's words must all
|
||||
// match, so "did our weekly upgrade run on all our nodes" would otherwise find nothing for "did", "our",
|
||||
// "on" and "all". Kept short and here only; a word that names something (`every`, `list`, `not`) is not
|
||||
// in it. A query of nothing but these matches nothing.
|
||||
var filler = map[string]bool{}
|
||||
|
||||
func init() {
|
||||
for _, w := range strings.Fields("a an the did do does our my your we i is are was on in at of for to all any and or with it its there what which how has have been") {
|
||||
filler[w] = true
|
||||
}
|
||||
}
|
||||
|
||||
// queryWords are a query's words to match: split at spaces, the punctuation of a sentence trimmed from
|
||||
// each end (`nodes?` is `nodes`), filler left out.
|
||||
func queryWords(query string) []string {
|
||||
var out []string
|
||||
for _, w := range strings.Fields(query) {
|
||||
w = strings.Trim(w, "?!.,;:\"'()")
|
||||
if w != "" && !filler[w] {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hit is one match as mesh_search answers it.
|
||||
type hit struct {
|
||||
Address string `json:"address"`
|
||||
@@ -169,8 +195,11 @@ func scoreOf(query string, qwords []string, name, description string, replaces [
|
||||
// search is mesh_search's answer: every match, best first.
|
||||
func (x *index) search(query string) []hit {
|
||||
query = strings.ToLower(strings.TrimSpace(query))
|
||||
qwords := strings.Fields(query)
|
||||
qwords := queryWords(query)
|
||||
var hits []hit
|
||||
if len(qwords) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, st := range x.Seats {
|
||||
for _, v := range st.Verbs {
|
||||
replaces := x.Replaces["seat:"+st.Seat+"."+v.Name]
|
||||
@@ -237,27 +266,16 @@ func (x *index) search(query string) []hit {
|
||||
return hits
|
||||
}
|
||||
|
||||
// replacesOf is what every verb and tool replaces, from the controller's records (novox/hq ADR 0245):
|
||||
// a seat's verbs from its `tools` answer, keyed `seat:<seat>.<verb>`, and a module's own tools from
|
||||
// its `modules` answer, keyed `<module>.<tool>`. A controller older than the field answers neither,
|
||||
// and search matches names and descriptions as before.
|
||||
func replacesOf(toolsAnswer json.RawMessage, modules []recordedModule) map[string][]string {
|
||||
// replacesOf is what every verb and tool replaces (novox/hq ADR 0245): a seat's verbs from the seat's contract in
|
||||
// mesh-sdk, keyed `seat:<seat>.<verb>`, read in-process and asked of no part (ADR 0571 clause 5: no part keeps a
|
||||
// central record of tools); and a module's own tools from its records, keyed `<module>.<tool>`. A verb or a tool with
|
||||
// no `replaces` is matched by its name and description as before.
|
||||
func replacesOf(seats []catalogue.Seat, modules []recordedModule) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
var seats struct {
|
||||
Seats []struct {
|
||||
Seat string `json:"seat"`
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
Replaces []string `json:"replaces"`
|
||||
} `json:"tools"`
|
||||
} `json:"seats"`
|
||||
}
|
||||
if json.Unmarshal(toolsAnswer, &seats) == nil {
|
||||
for _, s := range seats.Seats {
|
||||
for _, t := range s.Tools {
|
||||
if len(t.Replaces) > 0 {
|
||||
out["seat:"+s.Seat+"."+t.Name] = t.Replaces
|
||||
}
|
||||
for _, s := range seats {
|
||||
for _, v := range s.Serves {
|
||||
if len(v.Replaces) > 0 {
|
||||
out["seat:"+s.Name+"."+v.Name] = append([]string(nil), v.Replaces...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -19,6 +20,7 @@ func meshToSearch(t *testing.T) *index {
|
||||
"status": "One unit as the service manager sees it now: its states, whether it starts at boot.",
|
||||
"restart": "Restart one unit.",
|
||||
"journal": "The last lines of one unit's journal (at most 2000).",
|
||||
"failed": "The units that failed.",
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
for verb, does := range verbs {
|
||||
@@ -44,13 +46,7 @@ func meshToSearch(t *testing.T) *index {
|
||||
Description: does, Subject: "mesh.mod.docker.tool." + tool + ".anchor"})
|
||||
}
|
||||
x, _, _ := indexOfEndpoints(endpoints)
|
||||
tools := json.RawMessage(`{"seats":[{"seat":"node-service-manager","scope":"node","tools":[
|
||||
{"name":"units","replaces":["systemctl list-units","systemctl --failed"]},
|
||||
{"name":"status","replaces":["systemctl status","systemctl is-active"]},
|
||||
{"name":"restart","replaces":["systemctl restart"]},
|
||||
{"name":"journal","replaces":["journalctl"]}]},
|
||||
{"seat":"mesh-controller","scope":"mesh","tools":[{"name":"node","replaces":["hostnamectl","uptime"]},{"name":"nodes"}]}]}`)
|
||||
x.Replaces = replacesOf(tools, []recordedModule{{Module: "docker", On: []string{"anchor"}, Tools: true,
|
||||
x.Replaces = replacesOf(catalogue.DefaultSeats(), []recordedModule{{Module: "docker", On: []string{"anchor"}, Tools: true,
|
||||
Replaces: map[string][]string{"docker_list": {"docker ps"}, "docker_logs": {"docker logs"}, "docker_restart": {"docker restart"}}}})
|
||||
return x
|
||||
}
|
||||
@@ -68,7 +64,7 @@ func TestSearchFindsTheToolThatReplacesACommandFirst(t *testing.T) {
|
||||
"systemctl status": "anchor/node-service-manager.status",
|
||||
"systemctl status nats.service": "anchor/node-service-manager.status",
|
||||
"systemctl restart mesh-controller": "anchor/node-service-manager.restart",
|
||||
"systemctl --failed": "anchor/node-service-manager.units",
|
||||
"systemctl --failed": "anchor/node-service-manager.failed",
|
||||
"docker ps": "anchor/docker.docker_list",
|
||||
"docker ps -a": "anchor/docker.docker_list",
|
||||
"docker logs": "anchor/docker.docker_logs",
|
||||
@@ -134,10 +130,10 @@ func TestSearchStillMatchesEveryWord(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A controller older than `replaces` answers without it: search matches names and descriptions as before.
|
||||
// A seat's contract and a module's records without `replaces`: search matches names and descriptions as before.
|
||||
func TestSearchWithoutReplacesMatchesAsBefore(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
x.Replaces = replacesOf(json.RawMessage(`{"seats":[{"seat":"node-service-manager","tools":[{"name":"journal"}]}]}`),
|
||||
x.Replaces = replacesOf([]catalogue.Seat{{Name: "node-service-manager", Serves: []catalogue.Verb{{Name: "journal"}}}},
|
||||
[]recordedModule{{Module: "docker"}})
|
||||
if len(x.Replaces) != 0 {
|
||||
t.Fatalf("replaces from records that have none: %v", x.Replaces)
|
||||
@@ -149,3 +145,31 @@ func TestSearchWithoutReplacesMatchesAsBefore(t *testing.T) {
|
||||
t.Errorf("a tool's own name: %+v", hits)
|
||||
}
|
||||
}
|
||||
|
||||
// A question's filler words do not have to match (every other word still does): the operator's own words
|
||||
// find the tool whose description says it in other words; a query of filler alone finds nothing.
|
||||
func TestSearchIgnoresFillerWords(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
x.Modules["pacman"] = &moduleInfo{Module: "pacman", On: []string{"anchor"}, Tools: []Tool{{Module: "pacman", Name: "pacman_round",
|
||||
Description: "Whether this week's system upgrade ran on every node (the weekly upgrade round, last run of all nodes)."}}}
|
||||
for _, c := range []struct {
|
||||
query string
|
||||
want string // the first hit's address, "" for none
|
||||
}{
|
||||
{"did our weekly system upgrade run on all our nodes?", "anchor/pacman.pacman_round"},
|
||||
{"the weekly upgrade", "anchor/pacman.pacman_round"},
|
||||
{"did our", ""},
|
||||
{"what is the", ""},
|
||||
{"", ""},
|
||||
{"did our weekly kubernetes upgrade run?", ""},
|
||||
} {
|
||||
hits := x.search(c.query)
|
||||
got := ""
|
||||
if len(hits) > 0 {
|
||||
got = hits[0].Address
|
||||
}
|
||||
if got != c.want || (c.want == "" && len(hits) != 0) {
|
||||
t.Errorf("%q: first hit %q, want %q (%+v)", c.query, got, c.want, hits)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
package console
|
||||
|
||||
// mesh_wait: a session waits on the bus for one event instead of polling (novox/hq ADR 0367).
|
||||
//
|
||||
// **Only the families the tool runner was issued** (§2, §6): its membership names them, composed by the
|
||||
// controller from the tool runner's manifest together with the grant that lets this account subscribe them.
|
||||
// A name not listed there is refused here, before the bus would refuse it; no argument or setting adds one.
|
||||
// **An event only wakes the session** (§4): what it says is read again from the verb that owns it.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WaitDefault is how long a wait lasts when no timeout is given, and WaitLongest the most it may last
|
||||
// (ADR 0367 §1): four minutes and ten.
|
||||
var (
|
||||
WaitDefault = 4 * time.Minute
|
||||
WaitLongest = 10 * time.Minute
|
||||
)
|
||||
|
||||
func waitTool() map[string]any {
|
||||
return map[string]any{"name": verbWait, "inputSchema": map[string]any{"type": "object", "properties": map[string]any{
|
||||
"subjects": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "one or more event families to wait on, by name, e.g. `mesh-knowledge.reviewed`; only " +
|
||||
"those the mesh granted this machine's tool runner — a refusal names them"},
|
||||
"filter": map[string]any{"type": "object",
|
||||
"description": "fields of the event's body that must equal the given values, e.g. {\"kind\": \"decision\", \"number\": 367} (optional)"},
|
||||
"timeout": map[string]any{"type": "number",
|
||||
"description": fmt.Sprintf("seconds to wait: %d when absent, %d at most", int(WaitDefault.Seconds()), int(WaitLongest.Seconds()))},
|
||||
}, "required": []string{"subjects"}},
|
||||
"description": "Wait for the first event of the named families that matches the filter, and answer it with its " +
|
||||
"family, body and time — or say it timed out. Call it in a loop from a background command, and after every " +
|
||||
"answer read the state from the verb that owns it: an event only wakes you, and one published between two " +
|
||||
"waits is not heard (novox/hq ADR 0367)."}
|
||||
}
|
||||
|
||||
// waitAnswer answers one mesh_wait call.
|
||||
func (s *Surface) waitAnswer(ctx context.Context, args map[string]any) map[string]any {
|
||||
names, err := waitNames(args["subjects"])
|
||||
if err != nil {
|
||||
return failure(err.Error())
|
||||
}
|
||||
timeout, err := waitTimeout(args["timeout"])
|
||||
if err != nil {
|
||||
return failure(err.Error())
|
||||
}
|
||||
filter, _ := args["filter"].(map[string]any)
|
||||
if args["filter"] != nil && filter == nil {
|
||||
return failure("mesh_wait's filter is an object of body fields to the values they must equal")
|
||||
}
|
||||
|
||||
issued := map[string]string{} // name → subject
|
||||
if m := s.conn.Membership(s.conn.Module()); m != nil {
|
||||
for _, w := range m.Waits {
|
||||
issued[w.Event] = w.Subject
|
||||
}
|
||||
}
|
||||
if len(issued) == 0 {
|
||||
return failure("this machine's tool runner was issued no event families to wait on: the mesh grants them from " +
|
||||
"the tool runner's manifest (novox/hq ADR 0367 §2), and its membership names none yet")
|
||||
}
|
||||
byName := make([]string, 0, len(issued))
|
||||
for name := range issued {
|
||||
byName = append(byName, name)
|
||||
}
|
||||
sort.Strings(byName)
|
||||
family := map[string]string{} // subject → name
|
||||
var subjects []string
|
||||
for _, name := range names {
|
||||
subject, granted := issued[name]
|
||||
if !granted {
|
||||
return failure(fmt.Sprintf("mesh_wait may not wait on %q: this machine's tool runner is granted %s alone. "+
|
||||
"A family is added only by a reviewed change to the tool runner's manifest (node-tools' module.json, "+
|
||||
"novox/hq ADR 0367 §6); no argument, call or setting adds one", name, strings.Join(byName, ", ")))
|
||||
}
|
||||
if _, twice := family[subject]; !twice {
|
||||
subjects = append(subjects, subject)
|
||||
}
|
||||
family[subject] = name
|
||||
}
|
||||
|
||||
heard, ok, err := s.conn.HearFirst(ctx, subjects, timeout, func(body json.RawMessage) bool { return bodyMatches(body, filter) })
|
||||
if err != nil {
|
||||
return failure("mesh_wait could not wait: " + err.Error())
|
||||
}
|
||||
if !ok {
|
||||
return answerText(map[string]any{"timed-out": true, "waited": timeout.String(), "subjects": names,
|
||||
"read": "nothing matching was heard; read the state once from the verb that owns it, then wait again"})
|
||||
}
|
||||
return answerText(map[string]any{
|
||||
"heard": map[string]any{"subject": family[heard.Subject], "body": heard.Body, "at": heard.At.Format(time.RFC3339Nano)},
|
||||
"read": "an event only wakes you: read the state from the verb that owns it before acting (novox/hq ADR 0367 §4)",
|
||||
})
|
||||
}
|
||||
|
||||
// waitNames is the families a call names: a list of names, or one name.
|
||||
func waitNames(given any) ([]string, error) {
|
||||
var names []string
|
||||
switch v := given.(type) {
|
||||
case string:
|
||||
names = []string{v}
|
||||
case []any:
|
||||
for _, x := range v {
|
||||
name, ok := x.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("mesh_wait's subjects are event names, such as mesh-knowledge.reviewed; %v is not one", x)
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
for i := range names {
|
||||
names[i] = strings.TrimSpace(names[i])
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return nil, fmt.Errorf("mesh_wait names at least one event family in `subjects`, such as mesh-knowledge.reviewed")
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// waitTimeout is the call's timeout: WaitDefault when absent, refused beyond WaitLongest or below a second.
|
||||
func waitTimeout(given any) (time.Duration, error) {
|
||||
if given == nil {
|
||||
return WaitDefault, nil
|
||||
}
|
||||
longest := int(WaitLongest.Seconds())
|
||||
seconds, ok := given.(float64)
|
||||
if !ok {
|
||||
if n, isInt := given.(int); isInt {
|
||||
seconds, ok = float64(n), true
|
||||
}
|
||||
}
|
||||
if !ok || math.IsNaN(seconds) || seconds < 1 || seconds > float64(longest) {
|
||||
return 0, fmt.Errorf("mesh_wait's timeout is a number of seconds from 1 to %d (novox/hq ADR 0367 §1), not %v", longest, given)
|
||||
}
|
||||
return time.Duration(seconds * float64(time.Second)), nil
|
||||
}
|
||||
|
||||
// bodyMatches says whether every field the filter names equals, in the event's body, the value given.
|
||||
func bodyMatches(body json.RawMessage, filter map[string]any) bool {
|
||||
if len(filter) == 0 {
|
||||
return true
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(body, &fields) != nil {
|
||||
return false
|
||||
}
|
||||
for k, want := range filter {
|
||||
got, has := fields[k]
|
||||
if !has {
|
||||
return false
|
||||
}
|
||||
wanted, _ := json.Marshal(want)
|
||||
if !sameJSON(got, wanted) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// sameJSON compares two JSON values as values: 367 and 367.0 are one number, key order does not matter.
|
||||
func sameJSON(a, b []byte) bool {
|
||||
var x, y any
|
||||
da, db := json.NewDecoder(bytes.NewReader(a)), json.NewDecoder(bytes.NewReader(b))
|
||||
if da.Decode(&x) != nil || db.Decode(&y) != nil {
|
||||
return false
|
||||
}
|
||||
ja, _ := json.Marshal(x)
|
||||
jb, _ := json.Marshal(y)
|
||||
return bytes.Equal(ja, jb)
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// waitingRuntime is the tool runner on a machine, issued the families ADR 0367 §5 names, as the controller
|
||||
// composes them from its manifest: the surface over its connection, and the mesh to publish on.
|
||||
func waitingRuntime(t *testing.T) (*Surface, *mt.Mesh) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
m := mt.MembershipOf("node-tools", "desk", false, nil)
|
||||
m.Waits = []bus.WaitIssued{
|
||||
{Event: "mesh-knowledge.reviewed", Subject: "mesh.mod.mesh-knowledge.event.reviewed"},
|
||||
{Event: "operator-channel.decided.claude-code", Subject: "mesh.seat.operator-channel.event.decided.claude-code"},
|
||||
}
|
||||
mesh.Issue(t, m)
|
||||
conn := connect(t, "node-tools", "desk")
|
||||
mt.Until(t, func() error {
|
||||
if got := conn.Membership("node-tools"); got == nil || len(got.Waits) == 0 {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return NewSurface(conn, "desk.node-tools"), mesh
|
||||
}
|
||||
|
||||
// callWait is one tools/call of mesh_wait, answered as an agent host reads it.
|
||||
func callWait(s *Surface, args map[string]any) (text string, isError bool) {
|
||||
raw, _ := json.Marshal(map[string]any{"name": verbWait, "arguments": args})
|
||||
r := s.Handle(Request{JSONRPC: "2.0", ID: json.RawMessage("1"), Method: "tools/call", Params: raw})
|
||||
result, _ := r.Result.(map[string]any)
|
||||
content, _ := result["content"].([]map[string]any)
|
||||
var parts []string
|
||||
for _, c := range content {
|
||||
parts = append(parts, c["text"].(string))
|
||||
}
|
||||
isErr, _ := result["isError"].(bool)
|
||||
return strings.Join(parts, "\n"), isErr
|
||||
}
|
||||
|
||||
// novox/hq ADR 0367 §1: mesh_wait answers the first event that matches — by its family and its filter — with
|
||||
// its subject, body and time, within a second of its publication, and ignores the others.
|
||||
func TestMeshWaitAnswersTheFirstMatchingEvent(t *testing.T) {
|
||||
s, mesh := waitingRuntime(t)
|
||||
type answer struct {
|
||||
text string
|
||||
isErr bool
|
||||
at time.Time
|
||||
}
|
||||
done := make(chan answer, 1)
|
||||
go func() {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"},
|
||||
"filter": map[string]any{"kind": "decision", "number": 367}, "timeout": 20})
|
||||
done <- answer{text, isErr, time.Now()}
|
||||
}()
|
||||
// Once it listens: another record's verdict, another family's event, then the one it waits for.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
mesh.Emit(t, "mesh.mod.mesh-knowledge.event.reviewed", map[string]any{"kind": "decision", "number": 366, "what": "approved"})
|
||||
mesh.Emit(t, "mesh.mod.mesh-knowledge.event.proposed", map[string]any{"kind": "decision", "number": 367})
|
||||
published := time.Now()
|
||||
mesh.Emit(t, "mesh.mod.mesh-knowledge.event.reviewed", map[string]any{"kind": "decision", "number": 367, "what": "approved"})
|
||||
var got answer
|
||||
select {
|
||||
case got = <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("mesh_wait did not answer the matching event")
|
||||
}
|
||||
if got.isErr {
|
||||
t.Fatalf("mesh_wait failed: %s", got.text)
|
||||
}
|
||||
if got.at.Sub(published) > time.Second {
|
||||
t.Errorf("answered %s after the event was published, want within a second", got.at.Sub(published))
|
||||
}
|
||||
var heard struct {
|
||||
Heard struct {
|
||||
Subject string `json:"subject"`
|
||||
Body map[string]any `json:"body"`
|
||||
At time.Time `json:"at"`
|
||||
} `json:"heard"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(got.text), &heard); err != nil {
|
||||
t.Fatalf("not an answer: %s", got.text)
|
||||
}
|
||||
if heard.Heard.Subject != "mesh-knowledge.reviewed" || heard.Heard.Body["number"] != float64(367) || heard.Heard.At.IsZero() {
|
||||
t.Errorf("answered %+v, want record 367's verdict with its time", heard.Heard)
|
||||
}
|
||||
// And the subject it is granted on the asker's own `decided`, by the name a session asks for.
|
||||
go func() {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": "operator-channel.decided.claude-code", "timeout": 20})
|
||||
done <- answer{text, isErr, time.Now()}
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
// A seat's event, said by the router on core NATS here: no stream of the test's bus carries the seat.
|
||||
if err := connect(t, "messenger", "").Say("mesh.seat.operator-channel.event.decided.claude-code", []byte(`{"ask":"a1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case got = <-done:
|
||||
if got.isErr || !strings.Contains(got.text, `"subject": "operator-channel.decided.claude-code"`) {
|
||||
t.Errorf("the asker's decided: %s", got.text)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("mesh_wait did not answer the asker's decided")
|
||||
}
|
||||
}
|
||||
|
||||
// It says it timed out at its timeout, and leaves no subscription behind (ADR 0367 §1, §2).
|
||||
func TestMeshWaitSaysItTimedOutAndKeepsNothing(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
before := s.conn.Subscriptions()
|
||||
start := time.Now()
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"}, "timeout": 1})
|
||||
if took := time.Since(start); took < time.Second || took > 3*time.Second {
|
||||
t.Errorf("a one-second wait took %s", took)
|
||||
}
|
||||
if isErr || !strings.Contains(text, `"timed-out": true`) {
|
||||
t.Errorf("a wait that heard nothing: %s", text)
|
||||
}
|
||||
if after := s.conn.Subscriptions(); after != before {
|
||||
t.Errorf("%d subscriptions before the wait, %d after it", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// Its timeout is four minutes when none is given and ten at most: a longer one, or none at all, is refused
|
||||
// with the bound, never shortened in silence.
|
||||
func TestMeshWaitsTimeoutIsBounded(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
if WaitDefault != 4*time.Minute || WaitLongest != 10*time.Minute {
|
||||
t.Errorf("the bounds are %s and %s, want 4m and 10m (ADR 0367 §1)", WaitDefault, WaitLongest)
|
||||
}
|
||||
for _, timeout := range []any{601, 0, -5, "soon"} {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"}, "timeout": timeout})
|
||||
if !isErr || !strings.Contains(text, "600") {
|
||||
t.Errorf("a timeout of %v: %s", timeout, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0367 §2 and §6: an event outside the granted families is refused, naming the families and how
|
||||
// one is added; no argument — a wildcard, another asker's `decided`, a subject spelled out — widens it.
|
||||
func TestMeshWaitRefusesAnEventOutsideTheGrantedFamilies(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
for _, subjects := range []any{
|
||||
[]any{"mesh-knowledge.proposed"},
|
||||
[]any{"mesh-knowledge.reviewed", "gitea.forge.push"},
|
||||
[]any{"mesh-knowledge.*"},
|
||||
[]any{"operator-channel.decided.shop"},
|
||||
[]any{"operator-channel.decided.*"},
|
||||
[]any{"mesh.mod.mesh-knowledge.event.reviewed"},
|
||||
[]any{">"},
|
||||
[]any{},
|
||||
} {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": subjects, "timeout": 1})
|
||||
if !isErr {
|
||||
t.Errorf("%v was waited on: %s", subjects, text)
|
||||
continue
|
||||
}
|
||||
if len(subjects.([]any)) > 0 && (!strings.Contains(text, "mesh-knowledge.reviewed") || !strings.Contains(text, "manifest")) {
|
||||
t.Errorf("%v refused without naming the families and how one is added: %s", subjects, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A tool runner issued no families waits on nothing, and says so.
|
||||
func TestMeshWaitWithNoFamiliesIssuedWaitsOnNothing(t *testing.T) {
|
||||
mt.New(t)
|
||||
s := NewSurface(connect(t, "node-tools", "desk"), "desk.node-tools")
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"}, "timeout": 1})
|
||||
if !isErr || !strings.Contains(text, "no event families") {
|
||||
t.Errorf("a runtime issued no families: %s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// The mesh MCP server announces seven tools (ADR 0367 v13): ADR 0195's five, mesh_runtimes and mesh_wait, its timeout bounded.
|
||||
func TestMeshWaitIsAnnouncedAmongSeven(t *testing.T) {
|
||||
var names []string
|
||||
var wait map[string]any
|
||||
for _, tool := range discovery() {
|
||||
names = append(names, tool["name"].(string))
|
||||
if tool["name"] == verbWait {
|
||||
wait = tool
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
want := []string{"mesh_call", "mesh_describe", "mesh_machine", "mesh_overview", "mesh_runtimes", "mesh_search", "mesh_wait"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("announced %v, want %v", names, want)
|
||||
}
|
||||
if wait == nil || !isDiscovery(verbWait) {
|
||||
t.Fatal("mesh_wait is not announced")
|
||||
}
|
||||
schema, _ := json.Marshal(wait["inputSchema"])
|
||||
for _, word := range []string{`"subjects"`, `"filter"`, `"timeout"`} {
|
||||
if !strings.Contains(string(schema), word) {
|
||||
t.Errorf("mesh_wait's schema has no %s: %s", word, schema)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A client that hangs up or cancels ends its wait at once: the subscription is closed and nothing is left
|
||||
// waiting for the timeout (the review of issue 745). Over the console's own HTTP, as an agent host calls it.
|
||||
func TestMeshWaitEndsWhenItsClientHangsUp(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
up, err := Serve(s, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = up.Close() })
|
||||
before := s.conn.Subscriptions()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
body := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"mesh_wait",` +
|
||||
`"arguments":{"subjects":["mesh-knowledge.reviewed"],"timeout":600}}}`
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "http://"+up.Address+"/mcp", strings.NewReader(body))
|
||||
req.Header.Set("content-type", "application/json")
|
||||
go func() { _, _ = http.DefaultClient.Do(req) }()
|
||||
mt.Until(t, func() error {
|
||||
if s.conn.Subscriptions() <= before {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
cancel()
|
||||
start := time.Now()
|
||||
for s.conn.Subscriptions() > before {
|
||||
if time.Since(start) > 3*time.Second {
|
||||
t.Fatalf("%d subscriptions 3s after the client hung up, %d before the wait", s.conn.Subscriptions(), before)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// The tool runner stopping ends every wait in flight at once (the second review of issue 745): Close cancels
|
||||
// the requests' context rather than waiting up to ten minutes for a wait to time out.
|
||||
func TestMeshWaitEndsWhenTheConsoleCloses(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
up, err := Serve(s, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := s.conn.Subscriptions()
|
||||
body := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"mesh_wait",` +
|
||||
`"arguments":{"subjects":["mesh-knowledge.reviewed"],"timeout":600}}}`
|
||||
go func() { _, _ = http.Post("http://"+up.Address+"/mcp", "application/json", strings.NewReader(body)) }()
|
||||
mt.Until(t, func() error {
|
||||
if s.conn.Subscriptions() <= before {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
closed := make(chan error, 1)
|
||||
start := time.Now()
|
||||
go func() { closed <- up.Close() }()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("closing the console waited on a mesh_wait in flight")
|
||||
}
|
||||
if took := time.Since(start); took > 3*time.Second {
|
||||
t.Errorf("closing took %s", took)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
if s.conn.Subscriptions() > before {
|
||||
return os.ErrExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -3,7 +3,12 @@ package launch
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type lostWrite struct{}
|
||||
@@ -28,3 +33,84 @@ func TestALostWriteIsAnsweredWithItsOwnCodeAndNothingElseIs(t *testing.T) {
|
||||
t.Errorf("a plain refusal is answered %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A bundle that answers the handshake and an empty tool list, and notes each start and its pid in $OUT.
|
||||
const notingBundle = `#!/bin/sh
|
||||
printf 'started %s\n' "$$" >> "$OUT"
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id" ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
// The service manager stops the runtime by signalling its whole group, every bundle with it (issue 1126, ace's
|
||||
// reboot of 2026-10-28: every bundle "exited (SIGTERM); started again in 500ms" for the 15 s the runtime took
|
||||
// to stop, the power bundle among them dialling the system bus again, which restarted the bus during the
|
||||
// shutdown and held it 29 minutes). A bundle that exits while it serves is started again; once the runtime
|
||||
// itself is stopping, none is, by a restart or by a call.
|
||||
func TestABundleIsStartedAgainUnlessTheRuntimeIsStopping(t *testing.T) {
|
||||
saved := RestartFirst
|
||||
RestartFirst = 10 * time.Millisecond
|
||||
t.Cleanup(func() { RestartFirst = saved; shuttingDown.Store(false) })
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
stopping bool
|
||||
starts int
|
||||
}{
|
||||
{"a bundle that dies while the runtime serves", false, 2},
|
||||
{"a bundle signalled with the stopping runtime", true, 1},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
shuttingDown.Store(false)
|
||||
dir := t.TempDir()
|
||||
entry, out := filepath.Join(dir, "bundle"), filepath.Join(dir, "out")
|
||||
if err := os.WriteFile(entry, []byte(notingBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l, err := Start("noting", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
said, _ := os.ReadFile(out)
|
||||
var pid int
|
||||
if _, err := fmt.Sscanf(string(said), "started %d", &pid); err != nil {
|
||||
t.Fatalf("the bundle noted %q", said)
|
||||
}
|
||||
if row.stopping {
|
||||
ShutDown()
|
||||
}
|
||||
if err := syscall.Kill(pid, syscall.SIGTERM); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The restart row ends at its second start; the stopping row watches a window many restart pauses long.
|
||||
deadline := time.Now().Add(250 * time.Millisecond)
|
||||
for {
|
||||
said, _ = os.ReadFile(out)
|
||||
n := strings.Count(string(said), "started")
|
||||
if n > row.starts {
|
||||
t.Fatalf("started %d times, want %d", n, row.starts)
|
||||
}
|
||||
if n == row.starts && !row.stopping {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
if n != row.starts {
|
||||
t.Fatalf("started %d times, want %d", n, row.starts)
|
||||
}
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if row.stopping {
|
||||
// And a bundle first loaded now (a reload) is not started either.
|
||||
if _, err := Start("late", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf); err == nil {
|
||||
t.Error("a bundle was started after the runtime began stopping")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,12 +37,17 @@ type subscribing struct {
|
||||
func (b *subscribing) Publish(json.RawMessage) error { return nil }
|
||||
func (b *subscribing) Ask(json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) State(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) Log(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) Watch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
func (b *subscribing) Seat(string, json.RawMessage, func(string, any) (json.RawMessage, error)) (json.RawMessage, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (b *subscribing) Provision(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) ProvisionWatch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
func (b *subscribing) Subscribe(d func(json.RawMessage) error) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
@@ -90,3 +95,7 @@ func TestAnEventIsTakenByAnyAnswerThatIsNotAnError(t *testing.T) {
|
||||
t.Fatalf("an error answer was not the bundle's refusal of mesh/event: %#v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *subscribing) LogWatch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -77,10 +78,19 @@ type Bus interface {
|
||||
Subscribe(deliver func(envelope json.RawMessage) error) error
|
||||
State(verb string, params json.RawMessage) (json.RawMessage, error)
|
||||
Watch(params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error)
|
||||
// Log answers a bundle's `append`, `read`, `last` and `info` on its module's log (novox/hq ADR 0297).
|
||||
Log(verb string, params json.RawMessage) (json.RawMessage, error)
|
||||
// LogWatch hands deliver each entry of a log from where the bundle asked, one at a time (novox/hq ADR 0571).
|
||||
LogWatch(params json.RawMessage, deliver func(entry json.RawMessage) error) (stop func(), err error)
|
||||
// Seat answers a bundle's seat traffic (novox/hq ADR 0259 §3): `publish`, `prove` and `record` at once;
|
||||
// `take` and `answer` bind once and hand each message or proof on through handOn, to whichever child
|
||||
// of the module asked last — a restarted child asks again as its code runs again.
|
||||
Seat(verb string, params json.RawMessage, handOn func(method string, params any) (json.RawMessage, error)) (json.RawMessage, error)
|
||||
// Provision answers a bundle's `get` of one of its module's values at runtime, and ProvisionWatch hands deliver
|
||||
// that value now and each change after, as the module whose bundle asks and never a module the bundle names
|
||||
// (novox/hq ADR 0349 §14, ADR 0370 §4): its config, and its bindings' values that are not secret.
|
||||
Provision(verb string, params json.RawMessage) (json.RawMessage, error)
|
||||
ProvisionWatch(params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error)
|
||||
}
|
||||
|
||||
// EventTimeout bounds how long a bundle has to handle one event before it is offered again.
|
||||
@@ -105,6 +115,21 @@ var (
|
||||
RestartSettle = time.Minute
|
||||
)
|
||||
|
||||
// shuttingDown is set once the runtime itself is stopping (ShutDown): from then on no bundle is started again.
|
||||
var shuttingDown atomic.Bool
|
||||
|
||||
// ShutDown says the runtime itself is stopping, before it stops its bundles one by one. The service manager stops
|
||||
// the runtime by signalling its whole group, so every bundle receives SIGTERM with it, and a bundle's exit must not
|
||||
// be taken for a fault: started again, it would run on through the machine's shutdown with nothing left to stop it
|
||||
// (issue 1126: on ace's reboot of 2026-10-28 the restarted bundles dialled the system bus again and started it
|
||||
// during the shutdown, which then waited 29 minutes). After ShutDown no bundle is started again, by a restart or a
|
||||
// call.
|
||||
func ShutDown() { shuttingDown.Store(true) }
|
||||
|
||||
// StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that
|
||||
// ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476).
|
||||
var StopWait = 5 * time.Second
|
||||
|
||||
// Launched is a running bundle: what it registered, and how to stop it.
|
||||
type Launched struct {
|
||||
Registrations []Registration
|
||||
@@ -141,6 +166,9 @@ type child struct {
|
||||
pending map[int64]chan message
|
||||
next int64
|
||||
dead chan struct{}
|
||||
// handled is closed once the child's exit has been handled whole — its state cleared and its
|
||||
// restart decided — which Stop waits for (novox/hq issue 476).
|
||||
handled chan struct{}
|
||||
why error
|
||||
// watches are the state watches this child asked for, stopped when it exits (ADR 0201): a child
|
||||
// that starts again watches again, as its code runs again.
|
||||
@@ -160,9 +188,17 @@ func (c *child) write(m any) error {
|
||||
|
||||
var stackLine = regexp.MustCompile(`^\s+at\s`)
|
||||
|
||||
// colour is a terminal's colour or style sequence, which node writes into a stack trace when its
|
||||
// environment asks for colour (FORCE_COLOR): taken out before a line is judged or kept as a bundle's
|
||||
// reason, so the reason reads the same in every environment.
|
||||
var colour = regexp.MustCompile("\x1b\\[[0-9;]*m")
|
||||
|
||||
// Start launches one bundle and learns its tools. It fails when the child cannot be started or does
|
||||
// not complete the handshake. A child that exits later is started again on its next call.
|
||||
func Start(module, entry string, env []string, mesh Bus, logf func(string, ...any)) (*Launched, error) {
|
||||
if shuttingDown.Load() {
|
||||
return nil, fmt.Errorf("%s's bundle is not started: the runtime is stopping", module)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var current *child
|
||||
// The child that last subscribed is the one events are handed to: a restarted child subscribes
|
||||
@@ -193,7 +229,10 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
}
|
||||
var restart func(after time.Duration)
|
||||
var startedAt time.Time
|
||||
pause := RestartFirst
|
||||
// The restart pauses are read once, here: the exit handling runs on its own goroutine and must not
|
||||
// read the package's settings while a caller (a test) changes them (novox/hq issue 476).
|
||||
pauseFirst, pauseMost, settle := RestartFirst, RestartMost, RestartSettle
|
||||
pause := pauseFirst
|
||||
|
||||
start := func() (*child, error) {
|
||||
cmd := exec.Command(entry)
|
||||
@@ -213,7 +252,8 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &child{cmd: cmd, stdin: stdin, pending: map[int64]chan message{}, next: 1, dead: make(chan struct{})}
|
||||
c := &child{cmd: cmd, stdin: stdin, pending: map[int64]chan message{}, next: 1, dead: make(chan struct{}),
|
||||
handled: make(chan struct{})}
|
||||
var lastSaid string
|
||||
var saidMu sync.Mutex
|
||||
stderrDone := make(chan struct{})
|
||||
@@ -227,14 +267,16 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
continue
|
||||
}
|
||||
logf("[%s] %s", module, line)
|
||||
if !stackLine.MatchString(line) && !strings.HasPrefix(line, "Node.js v") {
|
||||
plain := colour.ReplaceAllString(line, "")
|
||||
if !stackLine.MatchString(plain) && !strings.HasPrefix(plain, "Node.js v") {
|
||||
saidMu.Lock()
|
||||
lastSaid = strings.TrimSpace(line)
|
||||
lastSaid = strings.TrimSpace(plain)
|
||||
saidMu.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer close(c.handled)
|
||||
scan := bufio.NewScanner(stdout)
|
||||
scan.Buffer(make([]byte, 64*1024), 16<<20)
|
||||
for scan.Scan() {
|
||||
@@ -269,7 +311,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
subscriber = c
|
||||
mu.Unlock()
|
||||
err = mesh.Subscribe(deliver)
|
||||
case "mesh/seat.publish", "mesh/seat.prove", "mesh/seat.record", "mesh/seat.kinds":
|
||||
case "mesh/seat.publish", "mesh/seat.prove", "mesh/seat.record", "mesh/seat.kinds", "mesh/seat.pull", "mesh/seat.settle":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Seat(strings.TrimPrefix(m.Method, "mesh/seat."), m.Params, nil); err == nil {
|
||||
result = answered
|
||||
@@ -283,11 +325,59 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
result = answered
|
||||
}
|
||||
case "mesh/state.get", "mesh/state.put", "mesh/state.delete", "mesh/state.keys",
|
||||
"mesh/state.create", "mesh/state.update":
|
||||
"mesh/state.create", "mesh/state.update", "mesh/state.history", "mesh/state.info":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.State(strings.TrimPrefix(m.Method, "mesh/state."), m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/log.append", "mesh/log.read", "mesh/log.newest", "mesh/log.last", "mesh/log.info":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Log(strings.TrimPrefix(m.Method, "mesh/log."), m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/provision.get":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Provision("get", m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/provision.watch":
|
||||
// As a state's watch: each change asked of this child in order, naming the watch by the
|
||||
// id the child asked it with, and answered once the current value is handed over.
|
||||
var stop func()
|
||||
watch := m.ID
|
||||
stop, err = mesh.ProvisionWatch(m.Params, func(change json.RawMessage) error {
|
||||
_, err := c.ask(module, "mesh/provision", map[string]any{"watch": watch, "change": change}, EventTimeout)
|
||||
return err
|
||||
})
|
||||
if err == nil {
|
||||
c.mu.Lock()
|
||||
if c.why != nil {
|
||||
c.mu.Unlock()
|
||||
stop()
|
||||
} else {
|
||||
c.watches = append(c.watches, stop)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
case "mesh/log.watch":
|
||||
// As a state's watch: each entry asked of this child in order, `mesh/log {watch, entry}`,
|
||||
// naming the watch by the id the child asked it with (novox/hq ADR 0571).
|
||||
var stop func()
|
||||
watch := m.ID
|
||||
stop, err = mesh.LogWatch(m.Params, func(entry json.RawMessage) error {
|
||||
_, err := c.ask(module, "mesh/log", map[string]any{"watch": watch, "entry": entry}, EventTimeout)
|
||||
return err
|
||||
})
|
||||
if err == nil {
|
||||
c.mu.Lock()
|
||||
if c.why != nil {
|
||||
c.mu.Unlock()
|
||||
stop()
|
||||
} else {
|
||||
c.watches = append(c.watches, stop)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
case "mesh/state.watch":
|
||||
// Each change is asked of this child, in order, naming the watch it is for by
|
||||
// the id the child asked it with — two watches of one state are two handlers;
|
||||
@@ -380,17 +470,20 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
}
|
||||
wasStopped := stopped
|
||||
// Started again at once if it ran a while; after a growing pause while it keeps exiting.
|
||||
wait := RestartFirst
|
||||
if time.Since(startedAt) < RestartSettle {
|
||||
wait := pauseFirst
|
||||
if time.Since(startedAt) < settle {
|
||||
wait = pause
|
||||
if pause *= 2; pause > RestartMost {
|
||||
pause = RestartMost
|
||||
if pause *= 2; pause > pauseMost {
|
||||
pause = pauseMost
|
||||
}
|
||||
} else {
|
||||
pause = RestartFirst
|
||||
pause = pauseFirst
|
||||
}
|
||||
mu.Unlock()
|
||||
if !wasStopped && wasServing {
|
||||
if wasServing && !wasStopped && shuttingDown.Load() {
|
||||
logf("[mesh-tools] %s; not started again: the runtime is stopping", why)
|
||||
}
|
||||
if !wasStopped && wasServing && !shuttingDown.Load() {
|
||||
// Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner —
|
||||
// is not waiting for a call to bring it back, and a tool bundle back early costs nothing.
|
||||
logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond))
|
||||
@@ -408,14 +501,27 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
|
||||
asking := func(method string, params any, timeout time.Duration) (json.RawMessage, error) {
|
||||
mu.Lock()
|
||||
c := current
|
||||
c, gone := current, stopped
|
||||
mu.Unlock()
|
||||
if gone {
|
||||
// A stopped bundle is never started again by a call: a process started now would be one
|
||||
// nothing stops (issue 1945).
|
||||
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
|
||||
}
|
||||
if c == nil {
|
||||
if shuttingDown.Load() {
|
||||
return nil, fmt.Errorf("%s's bundle is not started again: the runtime is stopping", module)
|
||||
}
|
||||
fresh, err := start()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mu.Lock()
|
||||
if stopped {
|
||||
mu.Unlock()
|
||||
_ = fresh.cmd.Process.Signal(syscall.SIGTERM)
|
||||
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
|
||||
}
|
||||
if current == nil {
|
||||
current = fresh
|
||||
startedAt = time.Now()
|
||||
@@ -433,7 +539,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
go func() {
|
||||
time.Sleep(after)
|
||||
mu.Lock()
|
||||
if stopped || current != nil {
|
||||
if stopped || current != nil || shuttingDown.Load() {
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
@@ -442,10 +548,10 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
if !stopped {
|
||||
if !stopped && !shuttingDown.Load() {
|
||||
wait := pause
|
||||
if pause *= 2; pause > RestartMost {
|
||||
pause = RestartMost
|
||||
if pause *= 2; pause > pauseMost {
|
||||
pause = pauseMost
|
||||
}
|
||||
logf("[mesh-tools] %s's bundle did not start again: %v; trying in %s", module, err, wait)
|
||||
go restart(wait)
|
||||
@@ -550,6 +656,12 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
mu.Unlock()
|
||||
if c != nil && c.cmd.Process != nil {
|
||||
_ = c.cmd.Process.Signal(syscall.SIGTERM)
|
||||
// Returns once the bundle's exit is handled, so nothing of it runs on after Stop
|
||||
// (novox/hq issue 476); bounded, for a bundle that does not exit on SIGTERM.
|
||||
select {
|
||||
case <-c.handled:
|
||||
case <-time.After(StopWait):
|
||||
}
|
||||
}
|
||||
}}
|
||||
for _, under := range order {
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package launch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A bundle that asks for one of its values and watches it, and writes each change it is handed to $OUT.
|
||||
const provisionBundle = `#!/bin/sh
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*)
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
||||
printf '{"jsonrpc":"2.0","id":"g1","method":"mesh/provision.get","params":{"kind":"config","key":"role"}}\n'
|
||||
printf '{"jsonrpc":"2.0","id":"w1","method":"mesh/provision.watch","params":{"kind":"config","key":"role"}}\n' ;;
|
||||
*'"method":"mesh/provision"'*)
|
||||
printf '%s\n' "$line" >> "$OUT"
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"id":"g1"'*) printf '%s\n' "$line" >> "$OUT" ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
type provisioning struct {
|
||||
subscribing
|
||||
mu sync.Mutex
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (b *provisioning) Provision(verb string, params json.RawMessage) (json.RawMessage, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.asked = append(b.asked, verb+" "+string(params))
|
||||
return json.RawMessage(`"the build machine"`), nil
|
||||
}
|
||||
|
||||
func (b *provisioning) ProvisionWatch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
b.mu.Lock()
|
||||
b.asked = append(b.asked, "watch "+string(params))
|
||||
b.mu.Unlock()
|
||||
return func() {}, deliver(json.RawMessage(`{"kind":"config","key":"role","value":"the build machine"}`))
|
||||
}
|
||||
|
||||
// novox/hq ADR 0349 §14, ADR 0370 §4: a bundle's ask for its values reaches the runtime's bus from its own channel,
|
||||
// answered there, and a watch's change is handed to it naming the watch it asked.
|
||||
func TestABundleAsksForItsValuesAndIsHandedItsWatchsChanges(t *testing.T) {
|
||||
entry := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.WriteFile(entry, []byte(provisionBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := filepath.Join(t.TempDir(), "out")
|
||||
b := &provisioning{}
|
||||
l, err := Start("claude-code", entry, append(os.Environ(), "OUT="+out), b, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
said, _ := os.ReadFile(out)
|
||||
if strings.Count(string(said), "\n") >= 2 {
|
||||
// Each line as the bundle read it, by what it is: the get's answer, and the watch's change.
|
||||
got := map[string]string{}
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(said)), "\n") {
|
||||
var m struct {
|
||||
ID any `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(line), &m); err != nil {
|
||||
t.Fatalf("the bundle read %s", line)
|
||||
}
|
||||
if m.Method != "" {
|
||||
got[m.Method] = string(m.Params)
|
||||
} else {
|
||||
got["answer"] = string(m.Result)
|
||||
}
|
||||
}
|
||||
if got["answer"] != `"the build machine"` {
|
||||
t.Errorf("the get was answered %s", got["answer"])
|
||||
}
|
||||
if got["mesh/provision"] != `{"change":{"kind":"config","key":"role","value":"the build machine"},"watch":"w1"}` {
|
||||
t.Errorf("the watch was handed %s", got["mesh/provision"])
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the bundle was handed %q", said)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
slices.Sort(b.asked)
|
||||
if len(b.asked) != 2 || b.asked[0] != `get {"kind":"config","key":"role"}` || b.asked[1] != `watch {"kind":"config","key":"role"}` {
|
||||
t.Errorf("the runtime's bus was asked %q", b.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *provisioning) LogWatch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
@@ -69,3 +69,45 @@ func TestABundleTakesASeatsWorkThroughItsOwnChannel(t *testing.T) {
|
||||
t.Fatalf("the work was not handed to the bundle: %s %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A bundle that pulls one ask from its worker when it is listed.
|
||||
const pullingBundle = `#!/bin/sh
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*)
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
||||
printf '{"jsonrpc":"2.0","id":"p1","method":"mesh/seat.pull","params":{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker","wait_ms":0}}\n' ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
// 1887/A11: a bundle's pull reaches the runtime's bus from its own channel, as its other seat traffic does.
|
||||
func TestABundlePullsASeatsWorkThroughItsOwnChannel(t *testing.T) {
|
||||
entry := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.WriteFile(entry, []byte(pullingBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := &seating{}
|
||||
l, err := Start("mesh-build-go", entry, os.Environ(), b, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
for i := 0; ; i++ {
|
||||
b.mu.Lock()
|
||||
verb, params := b.verb, b.params
|
||||
b.mu.Unlock()
|
||||
if verb == "pull" {
|
||||
if params != `{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker","wait_ms":0}` {
|
||||
t.Fatalf("pulled with %s", params)
|
||||
}
|
||||
return
|
||||
}
|
||||
if i > 100 {
|
||||
t.Fatal("the bundle's pull never reached the runtime")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package meshtest
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// bundlesWhy returns the reason bundles cannot be launched from a checkout rooted at root, or "" when they can: the
|
||||
// TypeScript and Python bundles the runtime and the console launch need node and the package
|
||||
// @novox/mesh-sdk, which comes from the package registry and is installed beside node-tools/package.json
|
||||
// (novox/hq ADR 0529 clause 1: what merge-check.sh said "NOT TESTED HERE" about is a skip with its reason).
|
||||
func bundlesWhy(root string, haveNode bool) string {
|
||||
if !haveNode {
|
||||
return "node is not installed"
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "node_modules", "@novox", "mesh-sdk")); err != nil {
|
||||
return "@novox/mesh-sdk is not installed in " + filepath.Join(root, "node_modules")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// NeedBundles skips the test, saying why, where the bundles it launches cannot run here.
|
||||
func NeedBundles(t *testing.T) {
|
||||
t.Helper()
|
||||
_, err := exec.LookPath("node")
|
||||
if why := bundlesWhy(moduleRoot(), err == nil); why != "" {
|
||||
t.Skip("NOT TESTED HERE: this test launches bundles, and " + why)
|
||||
}
|
||||
}
|
||||
|
||||
// moduleRoot is node-tools/, where package.json and node_modules are.
|
||||
func moduleRoot() string {
|
||||
_, here, _, _ := runtime.Caller(0)
|
||||
return filepath.Join(filepath.Dir(here), "..", "..")
|
||||
}
|
||||
@@ -11,7 +11,11 @@
|
||||
package meshtest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
@@ -22,6 +26,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
@@ -104,6 +109,111 @@ func (m *Mesh) Issue(t *testing.T, mem bus.Membership) {
|
||||
}
|
||||
}
|
||||
|
||||
// IssueInGrants writes a membership where the grants part keeps it, its state memberships, key `<node>.<module>`
|
||||
// (novox/hq ADR 0571 clause 7; mesh-sdk buscompose.GrantsMembershipsBucket), making the bucket when it is not there.
|
||||
// It writes with mesh-sdk's encoder, the grants part's own, never json of this package's type: a value built here by
|
||||
// hand agreed with the reader's mistake and hid that the part wrote another shape (issue 774).
|
||||
func (m *Mesh) IssueInGrants(t *testing.T, mem bus.Membership) {
|
||||
t.Helper()
|
||||
m.putGrants(t, mem, membership.EncodeGrantsMembership(sdkMembership(t, mem)))
|
||||
}
|
||||
|
||||
// IssueInGrantsWrapped writes a membership as the grants part wrote it before mesh-sdk's codec (until 2026-10-30, what
|
||||
// the live mesh holds until the part writes again): its pass record, the membership inside, published saying what
|
||||
// became of it ("published", "unchanged", or why it was held back).
|
||||
func (m *Mesh) IssueInGrantsWrapped(t *testing.T, mem bus.Membership, published string) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(map[string]any{"node": mem.Node, "module": mem.Module, "digest": "sha256:0",
|
||||
"membership": sdkMembership(t, mem), "published": published})
|
||||
m.putGrants(t, mem, body)
|
||||
}
|
||||
|
||||
func (m *Mesh) putGrants(t *testing.T, mem bus.Membership, body []byte) {
|
||||
t.Helper()
|
||||
kv, err := m.js.KeyValue(buscompose.GrantsMembershipsBucket)
|
||||
if err != nil {
|
||||
if kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: buscompose.GrantsMembershipsBucket, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := kv.Put(buscompose.MembershipKey(mem.Node, mem.Module), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// sdkMembership is a membership of this package's type as mesh-sdk's, which the grants part composes and writes.
|
||||
func sdkMembership(t *testing.T, mem bus.Membership) membership.Membership {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(mem)
|
||||
var out membership.Membership
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ConsoleReads are the states a console's runtime reads to say what the mesh holds (novox/hq ADR 0571 clauses 4, 5):
|
||||
// placement, the indexer's modules and manifests, and the seat holders.
|
||||
func ConsoleReads() []bus.StateIssued {
|
||||
return []bus.StateIssued{
|
||||
{Name: delivery.PlacementRead, Bucket: delivery.PlacementBucket},
|
||||
{Name: delivery.IndexerModule + ".modules", Bucket: delivery.ModulesBucket},
|
||||
{Name: delivery.IndexerModule + ".manifests", Bucket: delivery.ManifestsBucket},
|
||||
{Name: delivery.SeatHoldersRead, Bucket: delivery.SeatHoldersBucket},
|
||||
}
|
||||
}
|
||||
|
||||
// Placed writes what the mesh holds as its parts write it, with mesh-sdk's encoders: each module the indexer holds with
|
||||
// its manifest (tools, the tools it declares; none for a module without), and each node's placement of modules.
|
||||
func (m *Mesh) Placed(t *testing.T, tools map[string][]string, on map[string][]string) {
|
||||
t.Helper()
|
||||
kv := func(bucket string) nats.KeyValue {
|
||||
_ = m.js.DeleteKeyValue(bucket)
|
||||
b, err := m.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 1, MaxValueSize: 256 * 1024})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
modules, manifests, placement := kv(delivery.ModulesBucket), kv(delivery.ManifestsBucket), kv(delivery.PlacementBucket)
|
||||
kv(delivery.SeatHoldersBucket)
|
||||
for module, declared := range tools {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": module, "tools": declared})
|
||||
digest := delivery.ManifestDigest(manifest)
|
||||
raw, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: string(manifest)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := manifests.Put(digest, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err = delivery.EncodeModule(delivery.Module{None: "not built here", Manifest: digest, Repository: "novox/" + module,
|
||||
Commit: strings.Repeat("a", 40)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := modules.Put(module, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for node, placed := range on {
|
||||
p := delivery.Placement{V: delivery.Version, Node: node}
|
||||
for _, module := range placed {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: module})
|
||||
}
|
||||
// As the seat writes it: its node key and a key per module (research 072 row 37).
|
||||
values, err := delivery.PlacementValues(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
if _, err := placement.Put(key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NextEvent is the subject the next event under a pattern lands on.
|
||||
func (m *Mesh) NextEvent(t *testing.T, pattern string) <-chan string {
|
||||
t.Helper()
|
||||
@@ -220,6 +330,17 @@ func (m *Mesh) Pending(t *testing.T, node, module string) (ackPending, notDelive
|
||||
return uint64(info.NumAckPending), info.NumPending
|
||||
}
|
||||
|
||||
// BucketKeeping makes a module's state afresh as Bucket does, keeping up to history values a key.
|
||||
func (m *Mesh) BucketKeeping(t *testing.T, name string, history uint8) nats.KeyValue {
|
||||
t.Helper()
|
||||
_ = m.js.DeleteKeyValue(name)
|
||||
kv, err := m.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: name, History: history, MaxValueSize: 256 * 1024})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// Bucket makes a module's state afresh as the controller does from the catalogue (novox/hq ADR 0201),
|
||||
// and answers it for writing what is there before a bundle starts.
|
||||
func (m *Mesh) Bucket(t *testing.T, name string) nats.KeyValue {
|
||||
@@ -231,3 +352,104 @@ func (m *Mesh) Bucket(t *testing.T, name string) nats.KeyValue {
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// BucketKeyTTL makes a module's state afresh as the controller does for one declared `key-ttl` (mesh-sdk buscompose:
|
||||
// a time to live per key allowed, its end leaving a marker kept a minute; issue 1952).
|
||||
func (m *Mesh) BucketKeyTTL(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
_ = m.js.DeleteKeyValue(name)
|
||||
js, err := jetstream.New(m.nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1, MaxValueSize: 256 * 1024,
|
||||
LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Log makes a module's log afresh as the controller does from the catalogue (novox/hq ADR 0297): a file
|
||||
// stream named LOG_<module>_<name> on mesh.log.<module>.<name>.>, limits retention, no age, no cap per
|
||||
// key, the declared cap, 260 KiB a message (a 256 KiB value and its headers), refusing new entries when full, direct get, no delete or
|
||||
// purge. It answers the membership entry the controller issues for it.
|
||||
func (m *Mesh) Log(t *testing.T, module, name string, maxBytes int64) bus.LogIssued {
|
||||
t.Helper()
|
||||
stream := "LOG_" + module + "_" + name
|
||||
subject := "mesh.log." + module + "." + name
|
||||
_ = m.js.DeleteStream(stream)
|
||||
if _, err := m.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{subject + ".>"},
|
||||
Storage: nats.FileStorage, Retention: nats.LimitsPolicy, MaxAge: 0, MaxMsgsPerSubject: -1,
|
||||
MaxBytes: maxBytes, MaxMsgSize: 260 * 1024, Discard: nats.DiscardNew, AllowDirect: true,
|
||||
DenyDelete: true, DenyPurge: true, Replicas: 1,
|
||||
Description: module + "'s log " + name + ", kept as its record (novox/hq ADR 0297)"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return bus.LogIssued{Name: name, Stream: stream, Subject: subject, Writes: true}
|
||||
}
|
||||
|
||||
// Consumers is how many consumers a stream has.
|
||||
func (m *Mesh) Consumers(t *testing.T, stream string) int {
|
||||
t.Helper()
|
||||
info, err := m.js.StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.State.Consumers
|
||||
}
|
||||
|
||||
// ModuleStream makes a module's own stream afresh as the controller does from the catalogue (novox/hq ADR
|
||||
// 0363 §1): named in capitals, taking `mesh.stream.<module>.event.>`, a subject space EVENTS does not take.
|
||||
func (m *Mesh) ModuleStream(t *testing.T, module, name string) {
|
||||
t.Helper()
|
||||
_ = m.js.DeleteStream(name)
|
||||
if _, err := m.js.AddStream(&nats.StreamConfig{Name: name, Subjects: []string{"mesh.stream." + module + ".event.>"},
|
||||
Retention: nats.LimitsPolicy, Discard: nats.DiscardOld}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ConsumerOn makes a module's durable consumer on a machine on any stream, as Consumer does on EVENTS.
|
||||
func (m *Mesh) ConsumerOn(t *testing.T, stream, node, module string, filters []string, ackWait time.Duration) {
|
||||
t.Helper()
|
||||
cfg := &nats.ConsumerConfig{Durable: node + "_" + module, AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: ackWait, MaxDeliver: 10, DeliverPolicy: nats.DeliverNewPolicy}
|
||||
if len(filters) == 1 {
|
||||
cfg.FilterSubject = filters[0]
|
||||
} else {
|
||||
cfg.FilterSubjects = filters
|
||||
}
|
||||
if _, err := m.js.AddConsumer(stream, cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// PendingOn is what a module's consumer on a stream still holds, as Pending is on EVENTS.
|
||||
func (m *Mesh) PendingOn(t *testing.T, stream, node, module string) (ackPending, notDelivered uint64) {
|
||||
t.Helper()
|
||||
info, err := m.js.ConsumerInfo(stream, node+"_"+module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return uint64(info.NumAckPending), info.NumPending
|
||||
}
|
||||
|
||||
// Messages is how many messages a stream holds.
|
||||
func (m *Mesh) Messages(t *testing.T, stream string) uint64 {
|
||||
t.Helper()
|
||||
info, err := m.js.StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.State.Msgs
|
||||
}
|
||||
|
||||
// RemoveConsumer deletes a module's consumer on a machine on a stream, as the controller does when a grant
|
||||
// it cannot narrow is withdrawn (novox/hq ADR 0363 §6).
|
||||
func (m *Mesh) RemoveConsumer(t *testing.T, stream, node, module string) {
|
||||
t.Helper()
|
||||
if err := m.js.DeleteConsumer(stream, node+"_"+module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,3 +73,49 @@ func TestEachTestHasABusOfItsOwn(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// What merge-check.sh said was "NOT TESTED HERE" is a skip with its reason: no node, or no package, each says so.
|
||||
func TestBundlesCannotBeLaunchedWithoutNodeOrTheSDKAndTheSkipSaysWhy(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if why := bundlesWhy(root, false); !strings.Contains(why, "node is not installed") {
|
||||
t.Errorf("no node said %q", why)
|
||||
}
|
||||
if why := bundlesWhy(root, true); !strings.Contains(why, "@novox/mesh-sdk") {
|
||||
t.Errorf("no package said %q", why)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "node_modules", "@novox", "mesh-sdk"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := bundlesWhy(root, true); why != "" {
|
||||
t.Errorf("node and the package present said %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// The checker runs node-tools' Go level by Go's convention and its Node level by package.json's test script
|
||||
// (novox/hq ADR 0529): no script of the repository's own runs beside them, and no module but node-tools is built from
|
||||
// here (ADR 0524 §5, the toolchain image retired).
|
||||
func TestTheRepositoryHoldsNoCheckScriptAndNodeToolsIsItsOnlyModule(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "..")
|
||||
if _, err := os.Stat(filepath.Join(root, "merge-check.sh")); err == nil {
|
||||
t.Errorf("merge-check.sh is back: the checker's levels run the tests")
|
||||
}
|
||||
var found []string
|
||||
err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && (d.Name() == "node_modules" || d.Name() == ".git") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == "module.json" {
|
||||
found = append(found, filepath.ToSlash(p))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || !strings.HasSuffix(found[0], "/node-tools/module.json") {
|
||||
t.Errorf("the module manifests are %v, want node-tools' alone", found)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// Package npmlock says whether a package's lock file still describes its package.json (novox/hq issue 624).
|
||||
//
|
||||
// `npm ci` installs only from a lock file in step with package.json, and refuses the whole install when it
|
||||
// is not. node-tools' lock fell behind twice — the SDK's range moved and the toolchain's development
|
||||
// dependencies were added to package.json alone — and nothing here noticed, because the image's build runs
|
||||
// `npm install` and never reads the lock; the lab's runs, which run `npm ci`, stopped at setup instead.
|
||||
// The check is offline and needs no node: it reads the two files the way `npm ci` compares them first.
|
||||
package npmlock
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
DevDependencies map[string]string `json:"devDependencies"`
|
||||
OptionalDependencies map[string]string `json:"optionalDependencies"`
|
||||
PeerDependencies map[string]string `json:"peerDependencies"`
|
||||
}
|
||||
|
||||
type lock struct {
|
||||
LockfileVersion int `json:"lockfileVersion"`
|
||||
Packages map[string]manifest `json:"packages"`
|
||||
}
|
||||
|
||||
// Check reads dir's package.json and package-lock.json and names every way the lock is out of step: a
|
||||
// declared range the lock's root entry records differently or not at all, a range the lock records that
|
||||
// package.json no longer declares, and a declared dependency the lock resolved no package for. None means
|
||||
// in step.
|
||||
func Check(dir string) ([]string, error) {
|
||||
var pkg manifest
|
||||
if err := readJSON(filepath.Join(dir, "package.json"), &pkg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lk lock
|
||||
if err := readJSON(filepath.Join(dir, "package-lock.json"), &lk); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lk.LockfileVersion < 2 {
|
||||
return []string{fmt.Sprintf("package-lock.json is lockfileVersion %d; npm ci needs 2 or later here", lk.LockfileVersion)}, nil
|
||||
}
|
||||
root, ok := lk.Packages[""]
|
||||
if !ok {
|
||||
return []string{`package-lock.json has no root entry (packages[""])`}, nil
|
||||
}
|
||||
var out []string
|
||||
for _, kind := range []struct {
|
||||
name string
|
||||
want, have map[string]string
|
||||
}{
|
||||
{"dependencies", pkg.Dependencies, root.Dependencies},
|
||||
{"devDependencies", pkg.DevDependencies, root.DevDependencies},
|
||||
{"optionalDependencies", pkg.OptionalDependencies, root.OptionalDependencies},
|
||||
{"peerDependencies", pkg.PeerDependencies, root.PeerDependencies},
|
||||
} {
|
||||
for _, name := range sortedKeys(kind.want) {
|
||||
have, ok := kind.have[name]
|
||||
switch {
|
||||
case !ok:
|
||||
out = append(out, fmt.Sprintf("%s %s %q is in package.json and not in the lock", kind.name, name, kind.want[name]))
|
||||
case have != kind.want[name]:
|
||||
out = append(out, fmt.Sprintf("%s %s is %q in package.json and %q in the lock", kind.name, name, kind.want[name], have))
|
||||
}
|
||||
if kind.name == "peerDependencies" {
|
||||
continue
|
||||
}
|
||||
if _, ok := lk.Packages["node_modules/"+name]; !ok {
|
||||
out = append(out, fmt.Sprintf("%s %s has no resolved package in the lock", kind.name, name))
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(kind.have) {
|
||||
if _, ok := kind.want[name]; !ok {
|
||||
out = append(out, fmt.Sprintf("%s %s %q is in the lock and no longer in package.json", kind.name, name, kind.have[name]))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func readJSON(path string, v any) error {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(b, v); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]string) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package npmlock
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestNodeToolsLockIsInStepWithItsPackage: the lab installs node-tools with `npm ci`, which refuses a lock
|
||||
// out of step with package.json, so a change to package.json without its lock fails here and not at a
|
||||
// lab run's setup (novox/hq issue 624).
|
||||
func TestNodeToolsLockIsInStepWithItsPackage(t *testing.T) {
|
||||
out, err := Check(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(out) > 0 {
|
||||
t.Fatalf("node-tools/package-lock.json is out of step with package.json; run `npm install --package-lock-only` in node-tools:\n%s", strings.Join(out, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestALockBehindItsPackageIsNamed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, dir, "package.json", `{"dependencies":{"@novox/mesh-sdk":"^0.1.6","nats":"^2.29.0"},"devDependencies":{"pg":"^8"}}`)
|
||||
write(t, dir, "package-lock.json", `{"lockfileVersion":3,"packages":{
|
||||
"":{"dependencies":{"@novox/mesh-sdk":"^0.1.0","nats":"^2.29.0","left":"^1"}},
|
||||
"node_modules/@novox/mesh-sdk":{"version":"0.1.1"},
|
||||
"node_modules/nats":{"version":"2.29.3"}}}`)
|
||||
out, err := Check(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(out, "\n")
|
||||
for _, want := range []string{
|
||||
`dependencies @novox/mesh-sdk is "^0.1.6" in package.json and "^0.1.0" in the lock`,
|
||||
`dependencies left "^1" is in the lock and no longer in package.json`,
|
||||
`devDependencies pg "^8" is in package.json and not in the lock`,
|
||||
`devDependencies pg has no resolved package in the lock`,
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if len(out) != 4 {
|
||||
t.Errorf("want 4 findings, got %d:\n%s", len(out), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALockInStepIsQuiet(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, dir, "package.json", `{"dependencies":{"nats":"^2.29.0"}}`)
|
||||
write(t, dir, "package-lock.json", `{"lockfileVersion":3,"packages":{"":{"dependencies":{"nats":"^2.29.0"}},"node_modules/nats":{"version":"2.29.3"}}}`)
|
||||
out, err := Check(dir)
|
||||
if err != nil || len(out) != 0 {
|
||||
t.Fatalf("want quiet, got %v %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOldLockfileVersionIsNamed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, dir, "package.json", `{}`)
|
||||
write(t, dir, "package-lock.json", `{"lockfileVersion":1,"dependencies":{}}`)
|
||||
out, err := Check(dir)
|
||||
if err != nil || len(out) != 1 {
|
||||
t.Fatalf("want one finding, got %v %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func write(t *testing.T, dir, name, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
// Package provide is the runtime's half of node-tools being a provider of `mcp-endpoint`: it runs the mesh SDK's
|
||||
// provider helper, so a consumer handed to the provisioner (claude-code's `${bound:mcp-endpoint:port}`) is answered
|
||||
// from an entry node-tools writes in its own bucket, not from the controller's binding file (novox/hq ADR 0349
|
||||
// clause 12, issue 791).
|
||||
//
|
||||
// The offer carries no credential: the console's authority is the machine's login (ADR 0152, 0175), so the helper
|
||||
// reads nothing from the vault, needs no consumers' file and the entry names none. node-tools is not marked moved:
|
||||
// its manifest says `"answered-by": "helper"` for the provision, and the helper runs whenever the runtime does
|
||||
// (issue 791), as route-proxy's does (issue 792).
|
||||
package provide
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
const (
|
||||
// Module is the provider: the runtime's own module.
|
||||
Module = "node-tools"
|
||||
// Provision is what node-tools provides, as its manifest says.
|
||||
Provision = "mcp-endpoint"
|
||||
// ServedPort is the port its manifest serves (`serves.mcp-endpoint.port`): where the console listens.
|
||||
ServedPort = 4270
|
||||
)
|
||||
|
||||
// Offers is what node-tools serves, per provision.
|
||||
func Offers() map[string]provider.Offer {
|
||||
return map[string]provider.Offer{Provision: {
|
||||
Credential: provider.NoCredential,
|
||||
Serves: map[string]any{"port": ServedPort},
|
||||
}}
|
||||
}
|
||||
|
||||
// busOf is the helper's bus over the runtime's own state: the provider's bucket is its state `bound`, and the
|
||||
// provisions and plans of the seat mesh-provisioner are the states it reads, by mesh-sdk's names (novox/hq ADR 0570,
|
||||
// issue 1954 L6-T1). The controller's assignments are read no more (mesh-sdk 692dbf9, L6-P3).
|
||||
type busOf struct{ conn *bus.Conn }
|
||||
|
||||
func (b busOf) state(bucket string) (string, error) {
|
||||
switch bucket {
|
||||
case provider.BucketOf(Module):
|
||||
return "bound", nil
|
||||
case provisions.ProvisionsBucket:
|
||||
return provisions.ProvisionsRead, nil
|
||||
case provisions.PlansBucket:
|
||||
return provisions.PlansRead, nil
|
||||
}
|
||||
return "", fmt.Errorf("%s is not a bucket this provider keeps or reads", bucket)
|
||||
}
|
||||
|
||||
func (b busOf) Watch(ctx context.Context, bucket string, handle func(key string, value []byte)) error {
|
||||
name, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stop, err := b.conn.StateWatch(Module, name, "", func(ch bus.StateChange) error {
|
||||
if ch.Op != "put" {
|
||||
// A delete, or an expiry (issue 1952): the key is gone.
|
||||
handle(ch.Key, nil)
|
||||
} else {
|
||||
handle(ch.Key, ch.Value)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { <-ctx.Done(); stop() }()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b busOf) Put(_ context.Context, bucket, key string, value []byte) error {
|
||||
name, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = b.conn.StatePut(Module, name, key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busOf) Delete(_ context.Context, bucket, key string) error {
|
||||
name, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.conn.StateDelete(Module, name, key)
|
||||
}
|
||||
|
||||
// New is the helper answering node-tools's consumers on node.
|
||||
func New(conn *bus.Conn, node string, every time.Duration, logf func(string, ...any)) *provider.Provider {
|
||||
return provider.New(provider.Options{
|
||||
Node: node,
|
||||
Module: Module,
|
||||
Offers: Offers(),
|
||||
Bus: busOf{conn: conn},
|
||||
Every: every,
|
||||
Logf: logf,
|
||||
})
|
||||
}
|
||||
|
||||
// Run answers the consumers until ctx ends. The state is not reachable until the mesh has issued the runtime its
|
||||
// membership and made the buckets, so a start that cannot watch is tried again, not given up.
|
||||
func Run(ctx context.Context, conn *bus.Conn, node string, every time.Duration, logf func(string, ...any)) {
|
||||
for delay := 2 * time.Second; ctx.Err() == nil; delay = min(delay*2, time.Minute) {
|
||||
attempt, cancel := context.WithCancel(ctx)
|
||||
err := New(conn, node, every, logf).Run(attempt)
|
||||
cancel()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logf("[mesh-tools] the provider helper stopped: %v; trying again in %s", err, delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start runs the helper when this process is the machine's runtime (the node-tools credential, on a machine's
|
||||
// name), with no condition on the environment, and returns what ends it. A runtime on another credential, a module's
|
||||
// own in a test, is not the provider and answers nobody.
|
||||
func Start(conn *bus.Conn, module, node string, every time.Duration, logf func(string, ...any)) (stop func()) {
|
||||
if module != Module || node == "" {
|
||||
return func() {}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
logf("[mesh-tools] answering %s's consumers of %s through the provider helper", Module, Provision)
|
||||
go Run(ctx, conn, node, every, logf)
|
||||
return cancel
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package provide
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// claude-code's facts.json template, as the catalogue keeps it (modules/claude-code/files/facts.json).
|
||||
const factsTemplate = `{
|
||||
"node": "${machine:name}",
|
||||
"console": "http://127.0.0.1:${bound:mcp-endpoint:port}/mcp"
|
||||
}
|
||||
`
|
||||
|
||||
// factsToday is that file as the controller composes it today, from the port node-tools serves.
|
||||
const factsToday = `{
|
||||
"node": "anchor",
|
||||
"console": "http://127.0.0.1:4270/mcp"
|
||||
}
|
||||
`
|
||||
|
||||
// fill is what the consumer's client does with the template: the machine's name, and each ${bound:<provision>:<key>}
|
||||
// from the provider's entry.
|
||||
func fill(t *testing.T, template string, entry map[string]any) string {
|
||||
t.Helper()
|
||||
values, _ := entry["values"].(map[string]any)
|
||||
out := strings.ReplaceAll(template, "${machine:name}", "anchor")
|
||||
for key, v := range values {
|
||||
raw, _ := json.Marshal(v)
|
||||
out = strings.ReplaceAll(out, "${bound:mcp-endpoint:"+key+"}", string(raw))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTheRuntimeAnswersAHandedConsumersMcpEndpointThroughTheHelper(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
bound := mesh.Bucket(t, "node-tools_bound")
|
||||
plans := mesh.Bucket(t, provisions.PlansBucket)
|
||||
provs := mesh.Bucket(t, provisions.ProvisionsBucket)
|
||||
|
||||
mem := mt.MembershipOf("node-tools", "anchor", false, nil)
|
||||
mem.State = []bus.StateIssued{
|
||||
{Name: "bound", Bucket: "node-tools_bound", Writes: true},
|
||||
{Name: provisions.PlansRead, Bucket: provisions.PlansBucket},
|
||||
{Name: provisions.ProvisionsRead, Bucket: provisions.ProvisionsBucket},
|
||||
}
|
||||
mesh.Issue(t, mem)
|
||||
|
||||
// claude-code, handed on this machine, is bound to node-tools here for mcp-endpoint: its plan and its provision
|
||||
// key, written as mesh-provisioner writes them, with mesh-sdk's encoders (ADR 0570 clauses 1-3, issue 1954).
|
||||
plan, err := provisions.EncodePlan(provisions.Plan{Node: "anchor", Module: "claude-code", Plan: catalogue.Plan{
|
||||
Provisions: []catalogue.PlanProvision{{Provision: "mcp-endpoint", Provider: "anchor", ProviderModule: "node-tools",
|
||||
At: "127.0.0.1", From: "anchor", As: "mesh_anchor_claude_code"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := plans.Put("anchor.claude-code", plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := provisions.Key("anchor", "claude-code", "mcp-endpoint", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, err := provisions.Encode(provisions.Provision{Provision: "mcp-endpoint", State: provisions.Asked,
|
||||
Consumer: provisions.End{Node: "anchor", Module: "claude-code"}, Provider: provisions.End{Node: "anchor", Module: "node-tools"},
|
||||
At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := provs.Put(key, asked); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("node-tools")
|
||||
|
||||
said := &mt.Logs{}
|
||||
t.Cleanup(Start(conn, "node-tools", "anchor", 20*time.Millisecond, said.Logf))
|
||||
|
||||
var entry map[string]any
|
||||
mt.Until(t, func() error {
|
||||
kept, err := bound.Get("anchor.claude-code.mcp-endpoint")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(kept.Value(), &entry)
|
||||
})
|
||||
values, _ := entry["values"].(map[string]any)
|
||||
for key, want := range map[string]any{"port": float64(4270), "at": "127.0.0.1", "from": "anchor"} {
|
||||
if values[key] != want {
|
||||
t.Errorf("the entry's %s is %v, want %v (entry %v)", key, values[key], want, entry)
|
||||
}
|
||||
}
|
||||
if entry["as"] != "mesh_anchor_claude_code" {
|
||||
t.Errorf("the entry's as is %v", entry["as"])
|
||||
}
|
||||
if c, has := entry["credential"]; has {
|
||||
t.Errorf("an offer with no credential names one: %v", c)
|
||||
}
|
||||
if got := fill(t, factsTemplate, entry); got != factsToday {
|
||||
t.Errorf("claude-code's facts.json fills to\n%s\nwant today's\n%s", got, factsToday)
|
||||
}
|
||||
}
|
||||
|
||||
// The port the offer serves is the port the manifest serves and the console listens on.
|
||||
func TestTheOfferServesThePortTheManifestServes(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Serves map[string]map[string]any `json:"serves"`
|
||||
Moved bool `json:"moved"`
|
||||
State []string `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Provides []struct {
|
||||
Name string `json:"name"`
|
||||
AnsweredBy string `json:"answered-by"`
|
||||
} `json:"provides"`
|
||||
Resources []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Env map[string]string `json:"env"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := m.Serves[Provision]["port"]; got != float64(ServedPort) {
|
||||
t.Errorf("the manifest serves %v for %s, the offer %d", got, Provision, ServedPort)
|
||||
}
|
||||
if m.Moved {
|
||||
t.Error("node-tools is marked moved: that also hands the runtime's bus account to the vault (issue 791 round 1)")
|
||||
}
|
||||
if len(m.Provides) != 1 || m.Provides[0].Name != Provision || m.Provides[0].AnsweredBy != "helper" {
|
||||
t.Errorf("node-tools does not say its %s is answered by the helper: %+v", Provision, m.Provides)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r.ID == "consumers" {
|
||||
t.Error("the manifest still carries a consumers file the helper does not need")
|
||||
}
|
||||
}
|
||||
for _, r := range m.Reads {
|
||||
if r == "mesh-controller.assignments" {
|
||||
t.Error("the manifest still reads mesh-controller.assignments, which the sdk's helper no longer reads (issue 1954)")
|
||||
}
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Name == "files" || len(a.Env) > 0 {
|
||||
t.Errorf("the build still carries %+v, which only a moved module needs", a)
|
||||
}
|
||||
}
|
||||
// Every bucket the sdk's helper reaches is a state the manifest keeps or reads, by the sdk's names: the provider's
|
||||
// own and the seat mesh-provisioner's provisions and plans (issue 1954 L6-T1); the controller's assignments no more.
|
||||
declared := map[string]bool{}
|
||||
for _, n := range append(m.State, m.Reads...) {
|
||||
declared[n] = true
|
||||
}
|
||||
for bucket, want := range map[string]string{
|
||||
provider.BucketOf(Module): "bound",
|
||||
provisions.ProvisionsBucket: "mesh-provisioner.provisions",
|
||||
provisions.PlansBucket: "mesh-provisioner.plans",
|
||||
} {
|
||||
got, err := (busOf{}).state(bucket)
|
||||
if err != nil || got != want || !declared[got] {
|
||||
t.Errorf("the helper's bucket %s reaches state %q (%v), want %q declared in the manifest", bucket, got, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only the machine's runtime is the provider: a process on another credential, or with no machine, answers nobody.
|
||||
func TestOnlyTheMachinesRuntimeStartsTheHelper(t *testing.T) {
|
||||
said := &mt.Logs{}
|
||||
for _, c := range []struct{ module, node string }{{"claude-code", "anchor"}, {"node-tools", ""}} {
|
||||
Start(nil, c.module, c.node, time.Second, said.Logf)()
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
// read here with NATS's own types, each endpoint a subject actually served — and the announcement
|
||||
// follows a re-issued membership.
|
||||
func TestTheRuntimeAnnouncesWhatItServesInTheServicesProtocol(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", false, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "anchor", false, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestACallOnTheCallerNamedSubjectReachesTheModuleWithItsCaller(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer settled(stop)
|
||||
defer stop() // returns once the bundle's exit is handled (novox/hq issue 476)
|
||||
asker, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "shanks",
|
||||
User: "shanks.node-tools", Password: "unused"})
|
||||
if err != nil {
|
||||
@@ -65,7 +65,7 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer settled(stop)
|
||||
defer stop() // returns once the bundle's exit is handled (novox/hq issue 476)
|
||||
old, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -89,11 +89,3 @@ func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// settled stops a runtime and waits for its bundle's exit to be handled: the launcher reads its restart pauses
|
||||
// (launch.RestartFirst) as a bundle exits, and a later test in this package sets them, so a test that leaves its
|
||||
// bundle exiting behind it races the next one.
|
||||
func settled(stop func()) {
|
||||
stop()
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@ func countLines(t *testing.T, path, want string) int {
|
||||
// is its bus — it binds the module's own consumer, hands each event to the bundle, and acknowledges it
|
||||
// only when the bundle has taken it; one the bundle failed or died on is offered again.
|
||||
func TestTheRuntimeHandsAModulesEventsToItsBundleAndAcknowledgesThemOnlyWhenTaken(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
restore := bus.NakDelay
|
||||
bus.NakDelay = 300 * time.Millisecond
|
||||
@@ -128,6 +129,7 @@ func TestTheRuntimeHandsAModulesEventsToItsBundleAndAcknowledgesThemOnlyWhenTake
|
||||
|
||||
// A long-running bundle that exits is started again, without waiting for a call (ADR 0198).
|
||||
func TestALongRunningBundleThatExitsIsStartedAgain(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
firstPause := launch.RestartFirst
|
||||
launch.RestartFirst = 100 * time.Millisecond
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// logBuses is a module, issues, that keeps a log `changes` on a real bus, and a module, other, on the
|
||||
// same runtime that was issued no log — each reached as a bundle reaches it: mesh/log.<verb> with its
|
||||
// JSON, through its module's bus (novox/hq ADR 0297).
|
||||
func logBuses(t *testing.T, maxBytes int64) (issues, other launch.Bus, mesh *mt.Mesh) {
|
||||
t.Helper()
|
||||
mesh = mt.New(t)
|
||||
m := mt.MembershipOf("issues", "anchor", false, nil)
|
||||
changes := mesh.Log(t, "issues", "changes", maxBytes)
|
||||
m.Logs = []bus.LogIssued{changes}
|
||||
mesh.Issue(t, m)
|
||||
// other reads issues' log, as its manifest's reads declare it (ADR 0571): `<module>.<log>`, without writing it.
|
||||
o := mt.MembershipOf("other", "anchor", false, nil)
|
||||
o.Logs = []bus.LogIssued{{Name: "issues.changes", Stream: changes.Stream, Subject: changes.Subject}}
|
||||
mesh.Issue(t, o)
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("issues")
|
||||
conn.Follow("other")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("issues") == nil || conn.Membership("other") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
return all.forModule("issues"), all.forModule("other"), mesh
|
||||
}
|
||||
|
||||
// logAsk is a bundle's mesh/log.<verb>, with its params exactly as they go over the wire.
|
||||
func logAsk(b launch.Bus, verb, params string) (json.RawMessage, error) {
|
||||
return b.Log(verb, json.RawMessage(params))
|
||||
}
|
||||
|
||||
func appendOf(t *testing.T, b launch.Bus, params string) uint64 {
|
||||
t.Helper()
|
||||
raw, err := logAsk(b, "append", params)
|
||||
if err != nil {
|
||||
t.Fatalf("append %s: %v", params, err)
|
||||
}
|
||||
var r struct {
|
||||
Sequence *uint64 `json:"sequence"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &r); err != nil || r.Sequence == nil {
|
||||
t.Fatalf("append answered %s, not {sequence}", raw)
|
||||
}
|
||||
return *r.Sequence
|
||||
}
|
||||
|
||||
// wirePage is a read's answer with exactly the contract's field names.
|
||||
type wirePage struct {
|
||||
Entries []struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
At string `json:"at"`
|
||||
} `json:"entries"`
|
||||
Next uint64 `json:"next"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
func readOf(t *testing.T, b launch.Bus, params string) wirePage {
|
||||
t.Helper()
|
||||
raw, err := logAsk(b, "read", params)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", params, err)
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
_ = json.Unmarshal(raw, &fields)
|
||||
for _, f := range []string{"entries", "next", "last"} {
|
||||
if _, ok := fields[f]; !ok {
|
||||
t.Fatalf("read answered %s, without %q", raw, f)
|
||||
}
|
||||
}
|
||||
var p wirePage
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func infoOf(t *testing.T, b launch.Bus) map[string]uint64 {
|
||||
t.Helper()
|
||||
raw, err := logAsk(b, "info", `{"log":"changes"}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var info map[string]uint64
|
||||
if err := json.Unmarshal(raw, &info); err != nil {
|
||||
t.Fatalf("info answered %s: %v", raw, err)
|
||||
}
|
||||
for _, f := range []string{"entries", "bytes", "max_bytes", "first", "last"} {
|
||||
if _, ok := info[f]; !ok {
|
||||
t.Fatalf("info answered %s, without %q", raw, f)
|
||||
}
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// Each verb against a real bus: append answers the sequence, read pages from a sequence on, of a key
|
||||
// pattern, with the sequence to continue from (0 at the end) and the log's last, each entry's time in
|
||||
// RFC 3339; last is a key's last entry or null; info says entries, bytes, cap, first and last — and no
|
||||
// consumer is made for any of it.
|
||||
func TestALogAnswersEachVerbOnTheBus(t *testing.T) {
|
||||
b, other, mesh := logBuses(t, 1<<20)
|
||||
before := time.Now().Add(-time.Second)
|
||||
|
||||
var seqs []uint64
|
||||
for i, key := range []string{"12", "13", "12", "14.notes", "12"} {
|
||||
seqs = append(seqs, appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":%q,"value":{"op":"move","n":%d}}`, key, i)))
|
||||
}
|
||||
for i := 1; i < len(seqs); i++ {
|
||||
if seqs[i] != seqs[i-1]+1 {
|
||||
t.Fatalf("sequences %v are not the log's order", seqs)
|
||||
}
|
||||
}
|
||||
|
||||
// Everything, no keys and no limit: the default page holds all five.
|
||||
all := readOf(t, b, `{"log":"changes","from":0}`)
|
||||
if len(all.Entries) != 5 || all.Next != 0 || all.Last != seqs[4] {
|
||||
t.Fatalf("a read of everything: %d entries, next %d, last %d", len(all.Entries), all.Next, all.Last)
|
||||
}
|
||||
for i, e := range all.Entries {
|
||||
at, err := time.Parse(time.RFC3339, e.At)
|
||||
if err != nil || at.Before(before) || at.After(time.Now().Add(time.Second)) {
|
||||
t.Errorf("entry %d's time %q is not RFC 3339 and now: %v", e.Sequence, e.At, err)
|
||||
}
|
||||
if e.Sequence != seqs[i] || string(e.Value) != fmt.Sprintf(`{"op":"move","n":%d}`, i) {
|
||||
t.Errorf("entry %d: %d %s %s", i, e.Sequence, e.Key, e.Value)
|
||||
}
|
||||
}
|
||||
if all.Entries[3].Key != "14.notes" {
|
||||
t.Errorf("a key of two names read back as %q", all.Entries[3].Key)
|
||||
}
|
||||
|
||||
// One key's entries, a page of two at a time.
|
||||
first := readOf(t, b, `{"log":"changes","from":1,"keys":"12","limit":2}`)
|
||||
if len(first.Entries) != 2 || first.Entries[0].Sequence != seqs[0] || first.Entries[1].Sequence != seqs[2] ||
|
||||
first.Next != seqs[2]+1 {
|
||||
t.Fatalf("the first page of key 12: %+v", first)
|
||||
}
|
||||
second := readOf(t, b, fmt.Sprintf(`{"log":"changes","from":%d,"keys":"12","limit":2}`, first.Next))
|
||||
if len(second.Entries) != 1 || second.Entries[0].Sequence != seqs[4] || second.Next != 0 {
|
||||
t.Fatalf("the second page of key 12: %+v", second)
|
||||
}
|
||||
// A pattern: `*` one name, `>` the rest.
|
||||
if p := readOf(t, b, `{"log":"changes","from":1,"keys":"14.*"}`); len(p.Entries) != 1 || p.Entries[0].Key != "14.notes" {
|
||||
t.Fatalf("14.*: %+v", p)
|
||||
}
|
||||
if p := readOf(t, b, `{"log":"changes","from":1,"keys":"14.>"}`); len(p.Entries) != 1 {
|
||||
t.Fatalf("14.>: %+v", p)
|
||||
}
|
||||
// Past the end: nothing, nothing follows.
|
||||
if p := readOf(t, b, fmt.Sprintf(`{"log":"changes","from":%d}`, seqs[4]+1)); len(p.Entries) != 0 || p.Next != 0 || p.Last != seqs[4] {
|
||||
t.Fatalf("a read past the end: %+v", p)
|
||||
}
|
||||
if p := readOf(t, b, `{"log":"changes","from":1,"keys":"99"}`); len(p.Entries) != 0 || p.Next != 0 {
|
||||
t.Fatalf("a key with no entries: %+v", p)
|
||||
}
|
||||
|
||||
raw, err := logAsk(b, "last", `{"log":"changes","key":"12"}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var last map[string]json.RawMessage
|
||||
_ = json.Unmarshal(raw, &last)
|
||||
if string(last["sequence"]) != fmt.Sprint(seqs[4]) || string(last["key"]) != `"12"` ||
|
||||
string(last["value"]) != `{"op":"move","n":4}` || len(last["at"]) == 0 {
|
||||
t.Fatalf("the last entry of 12: %s", raw)
|
||||
}
|
||||
if raw, err := logAsk(b, "last", `{"log":"changes","key":"99"}`); err != nil || string(raw) != "null" {
|
||||
t.Fatalf("the last entry of a key with none: %s %v", raw, err)
|
||||
}
|
||||
|
||||
info := infoOf(t, b)
|
||||
if info["entries"] != 5 || info["first"] != seqs[0] || info["last"] != seqs[4] || info["max_bytes"] != 1<<20 ||
|
||||
info["bytes"] == 0 {
|
||||
t.Fatalf("info: %v", info)
|
||||
}
|
||||
if n := mesh.Consumers(t, "LOG_issues_changes"); n != 0 {
|
||||
t.Fatalf("the runtime made %d consumers of the log; it uses none", n)
|
||||
}
|
||||
|
||||
// From a time (ADR 0571): `since` in place of `from`, RFC 3339; both at once are refused.
|
||||
cut := all.Entries[2].At
|
||||
if p := readOf(t, b, fmt.Sprintf(`{"log":"changes","since":%q}`, cut)); len(p.Entries) == 0 || p.Entries[0].Sequence > seqs[2] ||
|
||||
p.Entries[len(p.Entries)-1].Sequence != seqs[4] {
|
||||
t.Fatalf("a read since %s: %+v", cut, p)
|
||||
}
|
||||
if p := readOf(t, b, fmt.Sprintf(`{"log":"changes","since":%q}`, time.Now().Add(time.Hour).Format(time.RFC3339))); len(p.Entries) != 0 {
|
||||
t.Fatalf("a read since an hour ahead: %+v", p)
|
||||
}
|
||||
if _, err := logAsk(b, "read", `{"log":"changes","from":2,"since":"2026-10-29T00:00:00Z"}`); err == nil {
|
||||
t.Fatal("a read from a sequence and a time was not refused")
|
||||
}
|
||||
// The newest of each key, in the log's order; `**` for the rest, as mesh-sdk says it.
|
||||
raw, err = logAsk(b, "newest", `{"log":"changes"}`)
|
||||
var newest []struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
if err != nil || json.Unmarshal(raw, &newest) != nil || len(newest) != 3 || newest[0].Key != "13" || newest[1].Key != "14.notes" ||
|
||||
newest[2].Sequence != seqs[4] {
|
||||
t.Fatalf("newest: %s %v", raw, err)
|
||||
}
|
||||
if raw, err := logAsk(b, "newest", `{"log":"changes","keys":"14.**"}`); err != nil || !strings.Contains(string(raw), `"14.notes"`) ||
|
||||
strings.Contains(string(raw), `"12"`) {
|
||||
t.Fatalf("newest of 14.**: %s %v", raw, err)
|
||||
}
|
||||
// Another module's log, read by the name its reads declare; never written.
|
||||
if p := readOf(t, other, `{"log":"issues.changes","from":1}`); len(p.Entries) != 5 {
|
||||
t.Fatalf("other's read of issues.changes: %+v", p)
|
||||
}
|
||||
if _, err := logAsk(other, "append", `{"log":"issues.changes","key":"1","value":{}}`); err == nil ||
|
||||
!strings.Contains(err.Error(), "without writing it") {
|
||||
t.Fatalf("other's append to the log it reads: %v", err)
|
||||
}
|
||||
// A watch hands each entry from where it is asked, then each new one, one at a time.
|
||||
heard := make(chan bus.LogEntry, 16)
|
||||
stop, err := b.LogWatch(json.RawMessage(fmt.Sprintf(`{"log":"changes","keys":"12","from":%d}`, seqs[2])), func(raw json.RawMessage) error {
|
||||
var e bus.LogEntry
|
||||
_ = json.Unmarshal(raw, &e)
|
||||
heard <- e
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
next := appendOf(t, b, `{"log":"changes","key":"12","value":{"op":"late"}}`)
|
||||
for _, want := range []uint64{seqs[2], seqs[4], next} {
|
||||
select {
|
||||
case e := <-heard:
|
||||
if e.Sequence != want || e.Key != "12" {
|
||||
t.Fatalf("the watch handed %d %s, want %d", e.Sequence, e.Key, want)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the watch did not hand entry %d", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A page holds about 4 MiB at most, whatever its limit, and says where to continue.
|
||||
func TestALogPageIsBoundedByItsBytes(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 64<<20)
|
||||
big := `"` + strings.Repeat("x", 250*1024) + `"`
|
||||
for i := 0; i < 20; i++ {
|
||||
appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":"k%d","value":%s}`, i, big))
|
||||
}
|
||||
p := readOf(t, b, `{"log":"changes","from":1,"limit":1000}`)
|
||||
if len(p.Entries) == 0 || len(p.Entries) >= 20 || p.Next != p.Entries[len(p.Entries)-1].Sequence+1 {
|
||||
t.Fatalf("a page of 20 entries of 250 KiB: %d entries, next %d", len(p.Entries), p.Next)
|
||||
}
|
||||
got := len(p.Entries)
|
||||
for p.Next != 0 {
|
||||
p = readOf(t, b, fmt.Sprintf(`{"log":"changes","from":%d,"limit":1000}`, p.Next))
|
||||
got += len(p.Entries)
|
||||
}
|
||||
if got != 20 {
|
||||
t.Fatalf("paging read %d of 20 entries", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An append with `expect` is written only when the key's last entry is the one expected — 0 when it
|
||||
// has none — and is otherwise refused with the error a lost mesh/state.update answers: the same code,
|
||||
// the same error, and nothing written. An append without `expect` is not checked.
|
||||
func TestAnAppendAtAStaleExpectIsRefusedAsALostUpdate(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 1<<20)
|
||||
isLost := func(err error) bool {
|
||||
var coded interface{ ErrorCode() int }
|
||||
return errors.As(err, &coded) && coded.ErrorCode() == launch.CodeChanged && errors.Is(err, bus.ErrStateChanged)
|
||||
}
|
||||
|
||||
first := appendOf(t, b, `{"log":"changes","key":"7","value":{"op":"open"},"expect":0}`)
|
||||
if _, err := logAsk(b, "append", `{"log":"changes","key":"7","value":{"op":"open"},"expect":0}`); !isLost(err) {
|
||||
t.Fatalf("a second open of one key at expect 0: %v", err)
|
||||
}
|
||||
second := appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"move"},"expect":%d}`, first))
|
||||
if _, err := logAsk(b, "append", fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"note"},"expect":%d}`, first)); !isLost(err) {
|
||||
t.Fatalf("an append at a stale expect: %v", err)
|
||||
}
|
||||
if _, err := logAsk(b, "append", fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"note"},"expect":%d}`, second+100)); !isLost(err) {
|
||||
t.Fatalf("an append at an expect not yet written: %v", err)
|
||||
}
|
||||
// Another key's entries do not make this key's expect stale.
|
||||
appendOf(t, b, `{"log":"changes","key":"8","value":{"op":"open"}}`)
|
||||
third := appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"note"},"expect":%d}`, second))
|
||||
// No expect: no check.
|
||||
appendOf(t, b, `{"log":"changes","key":"7","value":{"op":"set"}}`)
|
||||
|
||||
if info := infoOf(t, b); info["entries"] != 5 {
|
||||
t.Fatalf("refused appends wrote: %d entries, not 5", info["entries"])
|
||||
}
|
||||
p := readOf(t, b, `{"log":"changes","from":1,"keys":"7"}`)
|
||||
if len(p.Entries) != 4 || p.Entries[2].Sequence != third {
|
||||
t.Fatalf("key 7 holds %+v", p.Entries)
|
||||
}
|
||||
|
||||
// The refusal is the very error a lost state update is: its code reaches a bundle as CodeChanged.
|
||||
_, err := logAsk(b, "append", `{"log":"changes","key":"7","value":{},"expect":0}`)
|
||||
var coded interface{ ErrorCode() int }
|
||||
if !errors.As(err, &coded) || coded.ErrorCode() != -32010 {
|
||||
t.Fatalf("the refusal carries no lost-update code: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A log is its owner's alone: another module on the same runtime is refused it with the reason, and so
|
||||
// is a log of a name the owner was not issued — before anything is sent.
|
||||
func TestALogNotIssuedIsRefusedWithTheReason(t *testing.T) {
|
||||
issues, other, _ := logBuses(t, 1<<20)
|
||||
for verb, params := range map[string]string{
|
||||
"append": `{"log":"changes","key":"1","value":{}}`,
|
||||
"read": `{"log":"changes","from":1}`,
|
||||
"last": `{"log":"changes","key":"1"}`,
|
||||
"info": `{"log":"changes"}`,
|
||||
} {
|
||||
_, err := logAsk(other, verb, params)
|
||||
if err == nil || !strings.Contains(err.Error(), `other keeps no log called "changes"`) ||
|
||||
!strings.Contains(err.Error(), "issued: issues.changes") {
|
||||
t.Errorf("%s of a log not issued: %v", verb, err)
|
||||
}
|
||||
}
|
||||
_, err := logAsk(issues, "append", `{"log":"moves","key":"1","value":{}}`)
|
||||
if err == nil || !strings.Contains(err.Error(), `issues keeps no log called "moves"`) ||
|
||||
!strings.Contains(err.Error(), "issued: changes") {
|
||||
t.Errorf("an append to a log of a name not issued: %v", err)
|
||||
}
|
||||
if info := infoOf(t, issues); info["entries"] != 0 {
|
||||
t.Fatalf("a refused append wrote: %v", info)
|
||||
}
|
||||
}
|
||||
|
||||
// What the bus cannot hold, or the contract does not allow, is refused with the reason and not sent:
|
||||
// a value that is not JSON or over 256 KiB, a key that is not names joined by dots or over 255 bytes, a
|
||||
// key pattern with `>` not last, a limit over 1000.
|
||||
func TestWhatALogCannotHoldIsRefusedWithTheReason(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 1<<20)
|
||||
over := `"` + strings.Repeat("x", 256*1024) + `"`
|
||||
for params, reason := range map[string]string{
|
||||
`{"log":"changes","key":"1","value":nope}`: "names no log",
|
||||
`{"log":"changes","key":"1"}`: "is JSON",
|
||||
`{"log":"changes","key":"1","value":` + over + `}`: "at most 262144 bytes",
|
||||
`{"log":"changes","key":"a b","value":1}`: "is not a log key",
|
||||
`{"log":"changes","key":"a..b","value":1}`: "is not a log key",
|
||||
`{"log":"changes","key":"a.*","value":1}`: "is not a log key",
|
||||
`{"log":"changes","key":"` + strings.Repeat("k", 256) + `","value":1}`: "is not a log key",
|
||||
`{"key":"1","value":1}`: "names no log",
|
||||
} {
|
||||
if _, err := logAsk(b, "append", params); err == nil || !strings.Contains(err.Error(), reason) {
|
||||
t.Errorf("append %.80s: %v, want %q", params, err, reason)
|
||||
}
|
||||
}
|
||||
for params, reason := range map[string]string{
|
||||
`{"log":"changes","from":1,"keys":"a.>.b"}`: "is not a key pattern",
|
||||
`{"log":"changes","from":1,"limit":1001}`: "1 to 1000",
|
||||
`{"log":"changes","from":1,"limit":-1}`: "1 to 1000",
|
||||
} {
|
||||
if _, err := logAsk(b, "read", params); err == nil || !strings.Contains(err.Error(), reason) {
|
||||
t.Errorf("read %s: %v, want %q", params, err, reason)
|
||||
}
|
||||
}
|
||||
if info := infoOf(t, b); info["entries"] != 0 {
|
||||
t.Fatalf("a refused append wrote: %v", info)
|
||||
}
|
||||
// The largest value allowed is taken, also with the header an expect carries.
|
||||
largest := `"` + strings.Repeat("x", 256*1024-2) + `"`
|
||||
appendOf(t, b, `{"log":"changes","key":"1","value":`+largest+`,"expect":0}`)
|
||||
appendOf(t, b, `{"log":"changes","key":"2","value":`+largest+`}`)
|
||||
}
|
||||
|
||||
// A value naming a credential anywhere in it is refused, as mesh/state.put refuses one (novox/hq ADR
|
||||
// 0201 §13): a log keeps what it is given for good. A value that only counts tokens is taken.
|
||||
func TestALogRefusesAValueNamingACredential(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 1<<20)
|
||||
for _, value := range []string{`{"op":"set","api_key":"s"}`, `{"op":"note","with":{"headers":{"Authorization":"Bearer s"}}}`} {
|
||||
_, err := logAsk(b, "append", `{"log":"changes","key":"1","value":`+value+`}`)
|
||||
if err == nil || !strings.Contains(err.Error(), "names a credential") {
|
||||
t.Errorf("append of %s: %v", value, err)
|
||||
}
|
||||
}
|
||||
if info := infoOf(t, b); info["entries"] != 0 {
|
||||
t.Fatalf("a refused append wrote: %v", info)
|
||||
}
|
||||
appendOf(t, b, `{"log":"changes","key":"1","value":{"op":"note","maxTokens":4096}}`)
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// A module's code reads its values at runtime (novox/hq ADR 0349 clause 14 as ADR 0370 changes it, issue 685).
|
||||
//
|
||||
// A bundle asks `mesh/provision.get` and `mesh/provision.watch` with `{kind, key}`; the runtime asks its machine's
|
||||
// provisioner client **as the module whose bundle asks** — the module this bus was made for, from the bundle it
|
||||
// launched, never a name the bundle sent (ADR 0370 §4) — and hands the answer back. A watch hands the current value
|
||||
// first and then each change: the client says which value moved, and the runtime asks again. A secret is refused
|
||||
// here, with the reason, and never asked: the client serves no secret on the tool runner's word (§2), and a module
|
||||
// under an account of its own asks the vault itself, which waits on issue 569.
|
||||
|
||||
// The kinds a bundle may ask.
|
||||
const (
|
||||
provisionConfig = "config"
|
||||
provisionBound = "bound"
|
||||
provisionSecret = "secret"
|
||||
)
|
||||
|
||||
// provisionAsked is what a bundle names. Anything else it sends, a module's name among it, is not read.
|
||||
type provisionAsked struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// ProvisionChange is what a watch hands a bundle: the value, or, when it is no longer answered, why.
|
||||
type ProvisionChange struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
// Refused is why the value is no longer answered: the module taken off the machine, the key no longer declared.
|
||||
Refused string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
func readProvisionAsked(verb string, params json.RawMessage) (provisionAsked, error) {
|
||||
var asked provisionAsked
|
||||
if err := json.Unmarshal(params, &asked); err != nil || asked.Kind == "" || asked.Key == "" {
|
||||
return asked, fmt.Errorf("mesh/provision.%s names no value: {kind, key}", verb)
|
||||
}
|
||||
switch asked.Kind {
|
||||
case provisionConfig, provisionBound:
|
||||
return asked, nil
|
||||
case provisionSecret:
|
||||
return asked, fmt.Errorf("the secret %s is not asked for at runtime while the module's code runs in the "+
|
||||
"tool runner: the vault hands a secret only to a caller the bus proves, and the bus proves only the tool "+
|
||||
"runner here (novox/hq ADR 0370 §2, ADR 0349 §11); the module's secrets reach it through its files, "+
|
||||
"until it runs under a bus account of its own (issue 569)", asked.Key)
|
||||
}
|
||||
return asked, fmt.Errorf("mesh/provision.%s: %q is not a kind of value a module reads at runtime: config or bound",
|
||||
verb, asked.Kind)
|
||||
}
|
||||
|
||||
// Provision answers a bundle's `get` of one of its module's values.
|
||||
func (b *moduleBus) Provision(verb string, params json.RawMessage) (json.RawMessage, error) {
|
||||
if verb != "get" {
|
||||
return nil, fmt.Errorf("the runtime answers no mesh/provision.%s", verb)
|
||||
}
|
||||
asked, err := readProvisionAsked(verb, params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.all.conn.AskProvisioner(b.module, asked.Kind, asked.Key)
|
||||
}
|
||||
|
||||
// ProvisionWatch hands deliver the value now and each change after, until stopped. A value not answered now is
|
||||
// refused now, with the client's reason.
|
||||
func (b *moduleBus) ProvisionWatch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
asked, err := readProvisionAsked("watch", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := b.all.conn.AskProvisioner(b.module, asked.Kind, asked.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w := &provisionWatch{bus: b, asked: asked, deliver: deliver, last: value}
|
||||
if err := w.hand(ProvisionChange{Kind: asked.Kind, Key: asked.Key, Value: value}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stop, err := b.all.provisioned.add(b.all, b.module, w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stop, nil
|
||||
}
|
||||
|
||||
// provisionWatch is one watch a bundle holds: what it asked, and what it was handed last.
|
||||
type provisionWatch struct {
|
||||
bus *moduleBus
|
||||
asked provisionAsked
|
||||
deliver func(json.RawMessage) error
|
||||
mu sync.Mutex // one change at a time, in order
|
||||
last json.RawMessage
|
||||
refused string
|
||||
stopped bool
|
||||
}
|
||||
|
||||
func (w *provisionWatch) hand(change ProvisionChange) error {
|
||||
raw, err := json.Marshal(change)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return w.deliver(raw)
|
||||
}
|
||||
|
||||
// again asks once more after the client said something moved, and hands the bundle what changed.
|
||||
func (w *provisionWatch) again() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped {
|
||||
return
|
||||
}
|
||||
value, err := w.bus.all.conn.AskProvisioner(w.bus.module, w.asked.Kind, w.asked.Key)
|
||||
change := ProvisionChange{Kind: w.asked.Kind, Key: w.asked.Key}
|
||||
var refused *bus.ProvisionRefused
|
||||
switch {
|
||||
case errors.As(err, &refused):
|
||||
if w.refused == refused.Reason {
|
||||
return
|
||||
}
|
||||
change.Refused = refused.Reason
|
||||
case err != nil:
|
||||
// The client not reached: asked again on its next word, which it says whenever it comes up.
|
||||
w.bus.all.logf("[mesh-tools] %s's watch of its %s %s could not ask again: %v", w.bus.module, w.asked.Kind,
|
||||
w.asked.Key, err)
|
||||
return
|
||||
default:
|
||||
if w.refused == "" && bytes.Equal(value, w.last) {
|
||||
return
|
||||
}
|
||||
change.Value = value
|
||||
}
|
||||
if err := w.hand(change); err != nil {
|
||||
w.bus.all.logf("[mesh-tools] %s did not take a change of its %s %s: %v", w.bus.module, w.asked.Kind,
|
||||
w.asked.Key, err)
|
||||
return
|
||||
}
|
||||
w.last, w.refused = value, change.Refused
|
||||
}
|
||||
|
||||
// provisionWatches is every watch on this runtime, by module, and the one subscription that hears the client.
|
||||
type provisionWatches struct {
|
||||
mu sync.Mutex
|
||||
of map[string]map[*provisionWatch]bool
|
||||
stop func()
|
||||
}
|
||||
|
||||
func (p *provisionWatches) add(all *consumers, module string, w *provisionWatch) (func(), error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.stop == nil {
|
||||
stop, err := all.conn.OnProvisionChanged(func(module string, changed bus.ProvisionChanged) {
|
||||
for _, w := range p.told(module, changed) {
|
||||
go w.again()
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the runtime cannot hear its machine's provisioner client: %w", err)
|
||||
}
|
||||
p.stop = stop
|
||||
}
|
||||
if p.of == nil {
|
||||
p.of = map[string]map[*provisionWatch]bool{}
|
||||
}
|
||||
if p.of[module] == nil {
|
||||
p.of[module] = map[*provisionWatch]bool{}
|
||||
}
|
||||
p.of[module][w] = true
|
||||
return func() {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
delete(p.of[module], w)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// told is the watches of module that what moved concerns.
|
||||
func (p *provisionWatches) told(module string, changed bus.ProvisionChanged) []*provisionWatch {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var out []*provisionWatch
|
||||
for w := range p.of[module] {
|
||||
if w.asked.Kind == changed.Kind && (changed.Key == "" || changed.Key == w.asked.Key) {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stopModule stops every watch of one module, whose bundle is loaded again.
|
||||
func (p *provisionWatches) stopModule(module string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for w := range p.of[module] {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
}
|
||||
delete(p.of, module)
|
||||
}
|
||||
|
||||
func (p *provisionWatches) stopAll() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for _, ws := range p.of {
|
||||
for w := range ws {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
}
|
||||
}
|
||||
p.of = nil
|
||||
if p.stop != nil {
|
||||
p.stop()
|
||||
p.stop = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A module's code reads its values at runtime through moduleBus.Provision and ProvisionWatch (novox/hq ADR 0349
|
||||
// §14, ADR 0370, issue 685), against a real bus with a stand-in for the machine's provisioner client: the runtime
|
||||
// asks as the module whose bundle asks, whatever the bundle names; a watch hands the current value, then each change
|
||||
// the client says moved, and a refusal once; a secret is refused without asking.
|
||||
|
||||
// standInClient answers prov.ask.<node> from its values, keyed `<module>.<kind>.<key>`, and keeps every ask.
|
||||
type standInClient struct {
|
||||
mu sync.Mutex
|
||||
values map[string]string
|
||||
asks []string
|
||||
nc *nats.Conn
|
||||
}
|
||||
|
||||
func aStandInClient(t *testing.T, node string, values map[string]string) *standInClient {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
c := &standInClient{values: values, nc: nc}
|
||||
if _, err := nc.Subscribe("prov.ask."+node, func(m *nats.Msg) {
|
||||
var a struct{ Module, Kind, Key string }
|
||||
_ = json.Unmarshal(m.Data, &a)
|
||||
c.mu.Lock()
|
||||
c.asks = append(c.asks, string(m.Data))
|
||||
v, ok := c.values[a.Module+"."+a.Kind+"."+a.Key]
|
||||
c.mu.Unlock()
|
||||
if !ok {
|
||||
_ = m.Respond([]byte(`{"error":"` + a.Module + ` declares no config value ` + a.Key + `"}`))
|
||||
return
|
||||
}
|
||||
_ = m.Respond([]byte(`{"value":` + v + `}`))
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = nc.Flush()
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *standInClient) set(key, value string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if value == "" {
|
||||
delete(c.values, key)
|
||||
return
|
||||
}
|
||||
c.values[key] = value
|
||||
}
|
||||
|
||||
func (c *standInClient) asked() []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return append([]string(nil), c.asks...)
|
||||
}
|
||||
|
||||
func (c *standInClient) say(t *testing.T, node, module, body string) {
|
||||
t.Helper()
|
||||
if err := c.nc.Publish("prov.changed."+node+"."+module, []byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundleAsksForItsValuesAsItsOwnModule(t *testing.T) {
|
||||
client := aStandInClient(t, "anchor", map[string]string{
|
||||
"claude-code.config.managed_settings": `{"spinnerTipsEnabled":false}`,
|
||||
"dunst.config.managed_settings": `"dunst's"`,
|
||||
})
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
b := all.forModule("claude-code").(*moduleBus)
|
||||
|
||||
// Its own value, though the bundle names another module: the runtime names the module from the bundle.
|
||||
got, err := b.Provision("get", json.RawMessage(`{"kind":"config","key":"managed_settings","module":"dunst"}`))
|
||||
if err != nil || string(got) != `{"spinnerTipsEnabled":false}` {
|
||||
t.Fatalf("answered %s, %v", got, err)
|
||||
}
|
||||
if asks := client.asked(); len(asks) != 1 || asks[0] != `{"key":"managed_settings","kind":"config","module":"claude-code"}` {
|
||||
t.Fatalf("the client was asked %v", asks)
|
||||
}
|
||||
// The client's refusal, in its words.
|
||||
if _, err := b.Provision("get", json.RawMessage(`{"kind":"config","key":"nothing"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "declares no config value nothing") {
|
||||
t.Errorf("an undeclared key: %v", err)
|
||||
}
|
||||
// A secret, refused here and never asked; and what is not an ask.
|
||||
before := len(client.asked())
|
||||
for params, want := range map[string]string{
|
||||
`{"kind":"secret","key":"gitea-read"}`: "the vault hands a secret only to a caller the bus proves",
|
||||
`{"kind":"setting","key":"role"}`: "not a kind of value",
|
||||
`{"key":"role"}`: "names no value",
|
||||
} {
|
||||
if _, err := b.Provision("get", json.RawMessage(params)); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%s: %v, want %q", params, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := b.ProvisionWatch(json.RawMessage(`{"kind":"secret","key":"gitea-read"}`), func(json.RawMessage) error {
|
||||
return nil
|
||||
}); err == nil {
|
||||
t.Error("a secret was watched")
|
||||
}
|
||||
if n := len(client.asked()); n != before {
|
||||
t.Errorf("the client was asked for what the runtime refuses: %v", client.asked()[before:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundlesWatchIsHandedTheValueAndEachChange(t *testing.T) {
|
||||
client := aStandInClient(t, "anchor", map[string]string{"claude-code.config.managed_settings": `{"a":1}`})
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
t.Cleanup(all.stopAll)
|
||||
b := all.forModule("claude-code").(*moduleBus)
|
||||
|
||||
handed := make(chan ProvisionChange, 16)
|
||||
stop, err := b.ProvisionWatch(json.RawMessage(`{"kind":"config","key":"managed_settings","module":"dunst"}`),
|
||||
func(raw json.RawMessage) error {
|
||||
var c ProvisionChange
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
t.Errorf("handed %s", raw)
|
||||
}
|
||||
handed <- c
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := func(what string) ProvisionChange {
|
||||
t.Helper()
|
||||
select {
|
||||
case c := <-handed:
|
||||
return c
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("nothing handed: %s", what)
|
||||
}
|
||||
return ProvisionChange{}
|
||||
}
|
||||
nothing := func(what string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case c := <-handed:
|
||||
t.Errorf("%s: handed %+v", what, c)
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
if c := next("the current value"); string(c.Value) != `{"a":1}` || c.Kind != "config" || c.Key != "managed_settings" {
|
||||
t.Fatalf("first handed %+v", c)
|
||||
}
|
||||
|
||||
// A change, said by the client: asked again, handed.
|
||||
client.set("claude-code.config.managed_settings", `{"a":2}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config","key":"managed_settings"}`)
|
||||
if c := next("the change"); string(c.Value) != `{"a":2}` {
|
||||
t.Fatalf("after the change, handed %+v", c)
|
||||
}
|
||||
// Every key of the kind said to have moved, the value the same: nothing handed. Another module's or another
|
||||
// key's word: not even asked.
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
nothing("the same value again")
|
||||
before := len(client.asked())
|
||||
client.say(t, "anchor", "dunst", `{"kind":"config","key":"managed_settings"}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config","key":"role"}`)
|
||||
nothing("another's word")
|
||||
if n := len(client.asked()); n != before {
|
||||
t.Errorf("asked again for another's word: %v", client.asked()[before:])
|
||||
}
|
||||
// No longer answered: handed the refusal, once.
|
||||
client.set("claude-code.config.managed_settings", "")
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
if c := next("the refusal"); c.Refused == "" || len(c.Value) != 0 {
|
||||
t.Fatalf("handed %+v, want the refusal", c)
|
||||
}
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
nothing("the same refusal again")
|
||||
// Answered again, and then stopped: nothing after.
|
||||
client.set("claude-code.config.managed_settings", `{"a":3}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
if c := next("answered again"); string(c.Value) != `{"a":3}` {
|
||||
t.Fatalf("handed %+v", c)
|
||||
}
|
||||
stop()
|
||||
client.set("claude-code.config.managed_settings", `{"a":4}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
nothing("a stopped watch")
|
||||
for _, a := range client.asked() {
|
||||
if !strings.Contains(a, `"module":"claude-code"`) {
|
||||
t.Errorf("asked as another module: %s", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A runtime serves a module's bundle as soon as it is in place, again when it is replaced, and no
|
||||
// more once it is gone, with no restart (issue 1945, note 23: mesh-files' bundle arrived on a machine
|
||||
// a quarter of an hour after its runtime started, and its tools were missing until a hand restart).
|
||||
|
||||
// syncLogs is a logger that keeps the lines, safe to read while the runtime's watcher writes.
|
||||
type syncLogs struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (l *syncLogs) Logf(format string, args ...any) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.lines = append(l.lines, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *syncLogs) Has(fragments ...string) bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
for _, line := range l.lines {
|
||||
all := true
|
||||
for _, f := range fragments {
|
||||
all = all && strings.Contains(line, f)
|
||||
}
|
||||
if all {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (l *syncLogs) All() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return strings.Join(l.lines, "\n")
|
||||
}
|
||||
|
||||
// bundleServing is a tools bundle, MCP over stdio with no SDK, whose one tool is named tool.
|
||||
func bundleServing(tool string) string { return bundleScript(tool, false, false) }
|
||||
|
||||
// bundleScript is a tools bundle whose one tool is named tool. With subscribe it asks the runtime for its
|
||||
// module's events and writes "<tool> subscribed" and "<tool> handled <n>" to $EVENT_LOG; with writeBeside it
|
||||
// writes a file beside itself as it starts, as a bundle keeping a cache or a socket there does.
|
||||
func bundleScript(tool string, subscribe, writeBeside bool) string {
|
||||
py := func(b bool) string {
|
||||
if b {
|
||||
return "True"
|
||||
}
|
||||
return "False"
|
||||
}
|
||||
return `#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
TOOL = "` + tool + `"
|
||||
SUBSCRIBE = ` + py(subscribe) + `
|
||||
if ` + py(writeBeside) + `:
|
||||
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "cache"), "w") as f:
|
||||
f.write(TOOL + " " + str(os.getpid()))
|
||||
def say(m):
|
||||
m["jsonrpc"] = "2.0"; sys.stdout.write(json.dumps(m) + "\n"); sys.stdout.flush()
|
||||
def log(line):
|
||||
with open(os.environ["EVENT_LOG"], "a") as f:
|
||||
f.write(line + "\n")
|
||||
TOOLS = [{"name": TOOL, "description": "answers its name", "inputSchema": {"type": "object", "properties": {}}}]
|
||||
for line in sys.stdin:
|
||||
req = json.loads(line); rid = req.get("id"); m = req.get("method"); p = req.get("params") or {}
|
||||
if m is None:
|
||||
if rid == "subscribe":
|
||||
log(TOOL + " subscribed")
|
||||
continue
|
||||
if m == "initialize":
|
||||
say({"id": rid, "result": {"protocolVersion": "2025-03-26", "capabilities": {"tools": {}}, "serverInfo": {"name": "ledger", "version": "1"}}})
|
||||
elif m == "tools/list":
|
||||
say({"id": rid, "result": {"tools": TOOLS}})
|
||||
if SUBSCRIBE:
|
||||
say({"id": "subscribe", "method": "mesh/subscribe", "params": {}})
|
||||
elif m == "tools/call":
|
||||
say({"id": rid, "result": {"content": [{"type": "text", "text": json.dumps({"tool": TOOL})}]}})
|
||||
elif m == "mesh/event":
|
||||
body = (p.get("envelope") or {}).get("body") or {}
|
||||
log(TOOL + " handled " + str(body.get("n")))
|
||||
say({"id": rid, "result": {}})
|
||||
elif rid is not None:
|
||||
say({"id": rid, "result": {}})
|
||||
`
|
||||
}
|
||||
|
||||
// place puts a bundle serving tool at path the way a delivery does: written beside it, made
|
||||
// executable, and renamed into place, its directories made as needed.
|
||||
func place(t *testing.T, path, tool string) {
|
||||
t.Helper()
|
||||
placeScript(t, path, bundleServing(tool))
|
||||
}
|
||||
|
||||
func placeScript(t *testing.T, path, script string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmp := path + ".new"
|
||||
if err := os.WriteFile(tmp, []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// fastSettle makes the watcher act on a burst of changes quickly in a test.
|
||||
func fastSettle(t *testing.T) {
|
||||
was := Settle
|
||||
Settle = 50 * time.Millisecond
|
||||
t.Cleanup(func() { Settle = was })
|
||||
}
|
||||
|
||||
type reloadBed struct {
|
||||
mesh *mt.Mesh
|
||||
nodeTools *bus.Conn
|
||||
asker *bus.Conn
|
||||
logs *syncLogs
|
||||
entry string
|
||||
}
|
||||
|
||||
func newReloadBed(t *testing.T) *reloadBed {
|
||||
t.Helper()
|
||||
fastSettle(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
return &reloadBed{
|
||||
mesh: mesh,
|
||||
nodeTools: connect(t, "node-tools", "anchor"),
|
||||
asker: connect(t, "console", "workstation"),
|
||||
logs: &syncLogs{},
|
||||
// Below directories that do not exist yet, as on a machine before the module's first delivery.
|
||||
entry: filepath.Join(t.TempDir(), "bundles", "ledger", "tools", "serve.py"),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *reloadBed) run(t *testing.T) {
|
||||
t.Helper()
|
||||
stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
|
||||
func (b *reloadBed) answers(t *testing.T, tool string) {
|
||||
t.Helper()
|
||||
mt.Until(t, func() error {
|
||||
got, err := call(t, b.asker, "ledger."+tool+"@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ledger.%s is not served: %v\n%s", tool, err, b.logs.All())
|
||||
}
|
||||
if !strings.Contains(got, `"tool":"`+tool+`"`) {
|
||||
return fmt.Errorf("ledger.%s answered %s", tool, got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (b *reloadBed) silent(t *testing.T, tool string) {
|
||||
t.Helper()
|
||||
mt.Until(t, func() error {
|
||||
if _, err := call(t, b.asker, "ledger."+tool+"@anchor", map[string]any{}); err == nil {
|
||||
return fmt.Errorf("ledger.%s is still served\n%s", tool, b.logs.All())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func announced(t *testing.T) string {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
msg, err := nc.Request("$SRV.INFO.node-tools.anchor", nil, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var i micro.Info
|
||||
if err := json.Unmarshal(msg.Data, &i); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, e := range i.Endpoints {
|
||||
out = append(out, e.Subject)
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
// 1945/AC19: a bundle placed after the runtime started is served, and announced, with no restart.
|
||||
func TestABundlePlacedAfterStartIsServed(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
b.run(t)
|
||||
if !b.logs.Has("ledger's bundle", "does not exist") {
|
||||
t.Errorf("the missing bundle was not said as missing:\n%s", b.logs.All())
|
||||
}
|
||||
place(t, b.entry, "first")
|
||||
b.answers(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
if a := announced(t); !strings.Contains(a, "mesh.mod.ledger.tool.first.anchor") {
|
||||
return fmt.Errorf("discovery does not announce the placed bundle's tool:\n%s", a)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// 1945/AC20: a bundle replaced while served is loaded again: its new tool is served, its old one no more.
|
||||
func TestAReplacedBundleIsReloaded(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
place(t, b.entry, "second")
|
||||
b.answers(t, "second")
|
||||
b.silent(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
a := announced(t)
|
||||
if !strings.Contains(a, "mesh.mod.ledger.tool.second.anchor") || strings.Contains(a, "mesh.mod.ledger.tool.first.anchor") {
|
||||
return fmt.Errorf("discovery does not follow the replaced bundle:\n%s", a)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// The bundle's whole directory swapped for a new one, as an archive unpacked beside it and renamed in.
|
||||
dir := filepath.Dir(b.entry)
|
||||
fresh := dir + ".unpacked"
|
||||
place(t, filepath.Join(fresh, filepath.Base(b.entry)), "third")
|
||||
if err := os.Rename(dir, dir+".old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(fresh, dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.answers(t, "third")
|
||||
b.silent(t, "second")
|
||||
|
||||
// And changed again inside the directory swapped in: the watch follows the new directory.
|
||||
place(t, b.entry, "fourth")
|
||||
b.answers(t, "fourth")
|
||||
}
|
||||
|
||||
// A replaced bundle's old tool stays gone when the module's membership is issued again: a stopped tool
|
||||
// leaves no membership handler behind to bind it once more, and a call on it starts no unmanaged second
|
||||
// copy of the bundle (issue 1945, review round 2).
|
||||
func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
place(t, b.entry, "second")
|
||||
b.answers(t, "second")
|
||||
b.silent(t, "first")
|
||||
|
||||
b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
b.answers(t, "second")
|
||||
// Long enough for the reissued membership to have been heard, before the old tool is asked.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil {
|
||||
t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All())
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: a bundle removed while served is no longer served, and its module says why.
|
||||
func TestARemovedBundleIsNoLongerServed(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
if err := os.RemoveAll(filepath.Dir(b.entry)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.silent(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
got, err := call(t, b.asker, "ledger.tools@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var answer ToolsAnswer
|
||||
if err := json.Unmarshal([]byte(got), &answer); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(answer.Tools) != 0 || !strings.Contains(answer.Failed, "does not exist") {
|
||||
return fmt.Errorf("ledger's tools after its bundle was removed: %s", got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if a := announced(t); strings.Contains(a, "mesh.mod.ledger.tool.first") {
|
||||
t.Errorf("discovery still announces the removed bundle's tool:\n%s", a)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC22: a bundle that cannot be started is said with its real reason: a missing file does not
|
||||
// exist, and a file without an execute bit is not executable with its mode.
|
||||
func TestABundleThatCannotStartSaysTheRealReason(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "nowhere", "serve.py")
|
||||
if why := notStartable(missing); !strings.Contains(why, missing+" does not exist") || strings.Contains(why, "not executable") {
|
||||
t.Errorf("a missing bundle said: %q", why)
|
||||
}
|
||||
plain := filepath.Join(dir, "serve.py")
|
||||
if err := os.WriteFile(plain, []byte(bundleServing("first")), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := notStartable(plain); !strings.Contains(why, plain+" is not executable (mode -rw-r--r--)") {
|
||||
t.Errorf("a bundle without an execute bit said: %q", why)
|
||||
}
|
||||
if why := notStartable(dir); !strings.Contains(why, "is a directory") {
|
||||
t.Errorf("a directory said: %q", why)
|
||||
}
|
||||
if err := os.Chmod(plain, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := notStartable(plain); why != "" {
|
||||
t.Errorf("a startable bundle said: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC23: an event a module's consumer hears while the module is loaded again is offered again, never
|
||||
// a panic of the runtime, and the bundle loaded in its place is handed the next offer.
|
||||
func TestAnEventHeardDuringAReloadIsOfferedAgainNotAPanic(t *testing.T) {
|
||||
c := &consumers{logf: func(string, ...any) {}, of: map[string]*moduleEvents{}}
|
||||
var got []string
|
||||
before := func(json.RawMessage) error { got = append(got, "before"); return nil }
|
||||
old := &moduleEvents{delivers: []*func(json.RawMessage) error{&before}}
|
||||
c.of["ledger"] = old
|
||||
c.dropModule("ledger")
|
||||
env := bus.Envelope{Key: "alpha.happened", Body: json.RawMessage(`{"n":1}`)}
|
||||
hand := func(m *moduleEvents) (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("an event heard during a reload panicked the runtime: %v", r)
|
||||
}
|
||||
}()
|
||||
return c.handOut("ledger", m, env)
|
||||
}
|
||||
if err := hand(old); err == nil {
|
||||
t.Error("an event heard while the module is loaded again was taken, not offered again")
|
||||
}
|
||||
after := func(json.RawMessage) error { got = append(got, "after"); return nil }
|
||||
fresh := &moduleEvents{delivers: []*func(json.RawMessage) error{&after}}
|
||||
c.of["ledger"] = fresh
|
||||
if err := hand(fresh); err != nil {
|
||||
t.Errorf("the bundle loaded again was not handed the event: %v", err)
|
||||
}
|
||||
if err := hand(old); err == nil {
|
||||
t.Error("the stopped consumer's handler handed an event on after the reload")
|
||||
}
|
||||
if strings.Join(got, ",") != "after" {
|
||||
t.Errorf("handed to %v, want the bundle loaded again alone", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC23, whole: a module's events reach its bundle before a reload, around it, and after it.
|
||||
func TestAModulesEventsReachItsBundleLoadedAgain(t *testing.T) {
|
||||
fastSettle(t)
|
||||
restore := bus.NakDelay
|
||||
bus.NakDelay = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.NakDelay = restore })
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
mesh.Consumer(t, "anchor", "ledger", []string{"mesh.mod.alpha.event.>"}, 2*time.Second)
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
logs := &syncLogs{}
|
||||
dir := t.TempDir()
|
||||
log := filepath.Join(dir, "events.log")
|
||||
entry := filepath.Join(dir, "bundles", "ledger", "tools", "serve.py")
|
||||
has := func(line string) error {
|
||||
b, _ := os.ReadFile(log)
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if l == line {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("no %q yet in:\n%s\nruntime:\n%s", line, b, logs.All())
|
||||
}
|
||||
placeScript(t, entry, bundleScript("first", true, false))
|
||||
stop, err := Run(nodeTools, []Served{{"ledger", []string{entry}}}, map[string]map[string]string{"ledger": {"EVENT_LOG": log}}, logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mt.Until(t, func() error { return has("first subscribed") })
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error { return has("first handled 1") })
|
||||
|
||||
placeScript(t, entry, bundleScript("second", true, false))
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 2})
|
||||
mt.Until(t, func() error {
|
||||
if has("first handled 2") != nil && has("second handled 2") != nil {
|
||||
return has("second handled 2")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
mt.Until(t, func() error { return has("second subscribed") })
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 3})
|
||||
mt.Until(t, func() error { return has("second handled 3") })
|
||||
}
|
||||
|
||||
// 1945/AC24: a bundle that writes a file beside itself as it starts is loaded once at start and once
|
||||
// when it is replaced, never again by its own writing.
|
||||
func TestABundleWritingBesideItselfIsNotLoadedAgainByIt(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
t.Setenv("EVENT_LOG", filepath.Join(t.TempDir(), "unused.log"))
|
||||
placeScript(t, b.entry, bundleScript("first", false, true))
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
reloads := func() int {
|
||||
return strings.Count(b.logs.All(), "loading it again")
|
||||
}
|
||||
time.Sleep(20 * Settle)
|
||||
if n := reloads(); n != 0 {
|
||||
t.Fatalf("the bundle was loaded again %d time(s) by its own writing:\n%s", n, b.logs.All())
|
||||
}
|
||||
placeScript(t, b.entry, bundleScript("second", false, true))
|
||||
b.answers(t, "second")
|
||||
time.Sleep(20 * Settle)
|
||||
if n := reloads(); n != 1 {
|
||||
t.Errorf("replaced once, loaded again %d time(s):\n%s", n, b.logs.All())
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
@@ -134,7 +135,51 @@ type registration struct {
|
||||
tools []launch.Tool
|
||||
}
|
||||
|
||||
// Run launches, binds and serves. It answers a stop function.
|
||||
// loadedModule is one served module as its bundles were last loaded: what they registered, what of it
|
||||
// is served, and each entrypoint's signature at that load, which tells the watch that it changed.
|
||||
type loadedModule struct {
|
||||
served Served
|
||||
seen map[string]string // entrypoint → its signature when it was loaded
|
||||
children []func()
|
||||
regs []registration // every registration its bundles made
|
||||
own []registration // those under the name of a module served here
|
||||
failed string
|
||||
names []string // the tools it serves, as module.tool
|
||||
serving []func() // its tools' subscriptions
|
||||
}
|
||||
|
||||
// unload stops serving the module's tools and stops its bundles.
|
||||
func (l *loadedModule) unload() {
|
||||
for i := len(l.serving) - 1; i >= 0; i-- {
|
||||
l.serving[i]()
|
||||
}
|
||||
l.serving, l.names = nil, nil
|
||||
for i := len(l.children) - 1; i >= 0; i-- {
|
||||
l.children[i]()
|
||||
}
|
||||
l.children = nil
|
||||
}
|
||||
|
||||
// runner is the runtime at work: every served module as last loaded, loaded again when its bundle
|
||||
// appears, changes or goes (issue 1945, note 23).
|
||||
type runner struct {
|
||||
conn *bus.Conn
|
||||
logf func(string, ...any)
|
||||
envFor func(module string) []string
|
||||
events *consumers
|
||||
modules []string
|
||||
isServed map[string]bool
|
||||
|
||||
mu sync.Mutex
|
||||
of map[string]*loadedModule
|
||||
verbs map[string]func() // each module's `tools` verb, where it is answered
|
||||
|
||||
seatsAgain func()
|
||||
stops []func() // the watch, the announcement and the seats, in the order they began
|
||||
}
|
||||
|
||||
// Run launches, binds and serves, and serves each module's bundle again when it appears, changes or
|
||||
// goes, with no restart. It answers a stop function.
|
||||
func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, logf func(string, ...any)) (func(), error) {
|
||||
if account := os.Getenv(OperatorAccount); account != "" {
|
||||
home := ""
|
||||
@@ -143,153 +188,324 @@ func Run(conn *bus.Conn, served []Served, envs map[string]map[string]string, log
|
||||
}
|
||||
logf("[mesh-tools] the operator's account here is %s%s", account, home)
|
||||
}
|
||||
modules := make([]string, 0, len(served))
|
||||
isServed := map[string]bool{}
|
||||
r := &runner{conn: conn, logf: logf, isServed: map[string]bool{}, of: map[string]*loadedModule{},
|
||||
verbs: map[string]func(){}}
|
||||
for _, s := range served {
|
||||
modules = append(modules, s.Module)
|
||||
isServed[s.Module] = true
|
||||
r.modules = append(r.modules, s.Module)
|
||||
r.isServed[s.Module] = true
|
||||
conn.Follow(s.Module)
|
||||
}
|
||||
node := conn.Node()
|
||||
|
||||
base := os.Environ()
|
||||
envFor := func(module string) []string { return BundleEnv(base, envs[module], module, node) }
|
||||
r.envFor = func(module string) []string { return BundleEnv(base, envs[module], module, node) }
|
||||
|
||||
// The module's events, for every child of it that subscribes (ADR 0198): one consumer per module,
|
||||
// bound the first time any of its children subscribes, each event handed to every child that did.
|
||||
events := &consumers{conn: conn, logf: logf, of: map[string]*moduleEvents{}}
|
||||
r.events = &consumers{conn: conn, logf: logf, of: map[string]*moduleEvents{}}
|
||||
|
||||
failed := map[string]string{}
|
||||
var registrations []registration
|
||||
var stops []func()
|
||||
for _, s := range served {
|
||||
for _, entry := range s.Entrypoints {
|
||||
path, _ := filepath.Abs(entry)
|
||||
module := s.Module
|
||||
fail := func(why string) {
|
||||
failed[module] = why
|
||||
logf("[mesh-tools] %s's bundle %s failed to load: %s; its tools are not served here", module, entry, why)
|
||||
}
|
||||
if !launch.Executable(path) {
|
||||
fail(path + " is not executable; a bundle the runtime serves is started, never imported, and its build makes it executable (novox/hq ADR 0193)")
|
||||
continue
|
||||
}
|
||||
child, err := launch.Start(module, path, envFor(module), events.forModule(module), logf)
|
||||
if err != nil {
|
||||
fail(err.Error())
|
||||
continue
|
||||
}
|
||||
stops = append(stops, child.Stop)
|
||||
for _, r := range child.Registrations {
|
||||
registrations = append(registrations, registration{module: r.Module, owner: module, tools: r.Tools})
|
||||
}
|
||||
r.of[s.Module] = r.load(s)
|
||||
}
|
||||
for _, m := range r.modules {
|
||||
if err := refusal(r.of[m].own); err != nil {
|
||||
r.stop()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
claimed := map[string]bool{}
|
||||
for _, m := range modules {
|
||||
if mem := conn.Membership(m); mem != nil {
|
||||
for _, s := range mem.Seats {
|
||||
claimed[s.Seat] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
var own []registration
|
||||
for _, r := range registrations {
|
||||
switch {
|
||||
case isServed[r.module]:
|
||||
own = append(own, r)
|
||||
case claimed[r.module]:
|
||||
default:
|
||||
logf(`[mesh-tools] %s registers tools under "%s", which is neither a module served here nor a seat one of them claims; not served until the mesh issues the claim`, r.owner, r.module)
|
||||
}
|
||||
}
|
||||
stopAll := func() {
|
||||
events.stopAll()
|
||||
for i := len(stops) - 1; i >= 0; i-- {
|
||||
stops[i]()
|
||||
}
|
||||
}
|
||||
for _, r := range own {
|
||||
seen := map[string]bool{}
|
||||
for _, t := range r.tools {
|
||||
if t.Name == ToolsVerb {
|
||||
stopAll()
|
||||
return nil, fmt.Errorf(`%s names a tool "%s", which is the verb the runtime answers for every module with what it serves (novox/hq ADR 0152) — refused, rename it`, r.module, ToolsVerb)
|
||||
}
|
||||
if seen[t.Name] {
|
||||
stopAll()
|
||||
return nil, fmt.Errorf("%s exposes two tools named %s — refused", r.module, t.Name)
|
||||
}
|
||||
seen[t.Name] = true
|
||||
}
|
||||
}
|
||||
|
||||
var names []string
|
||||
byModule := map[string][]launch.Tool{}
|
||||
for _, r := range own {
|
||||
for _, t := range r.tools {
|
||||
t := t
|
||||
// Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365).
|
||||
stop, err := conn.HandleCalled(r.module+"."+t.Name, func(caller string, body json.RawMessage) (any, error) {
|
||||
return t.Run(caller, argsOf(body))
|
||||
})
|
||||
if err != nil {
|
||||
logf("[mesh-tools] cannot serve %s.%s: %v", r.module, t.Name, err)
|
||||
continue
|
||||
}
|
||||
names = append(names, r.module+"."+t.Name)
|
||||
stops = append(stops, stop)
|
||||
}
|
||||
byModule[r.module] = append(byModule[r.module], r.tools...)
|
||||
}
|
||||
for _, module := range modules {
|
||||
module := module
|
||||
tools := byModule[module]
|
||||
why := failed[module]
|
||||
if len(tools) == 0 && why == "" {
|
||||
continue // a pure-events module: silent, as it always was
|
||||
}
|
||||
stop, err := conn.Handle(module+"."+ToolsVerb, func(json.RawMessage) (any, error) {
|
||||
answer := ToolsAnswer{Module: module, Tools: []ListedTool{}, Failed: why}
|
||||
for _, t := range tools {
|
||||
answer.Tools = append(answer.Tools, ListedTool{Name: t.Name, Description: t.Description,
|
||||
Input: t.Input, Subjects: subjectsOf(conn, module, t.Name)})
|
||||
}
|
||||
return answer, nil
|
||||
})
|
||||
if err == nil {
|
||||
stops = append(stops, stop)
|
||||
}
|
||||
}
|
||||
|
||||
failedNames := make([]string, 0, len(failed))
|
||||
for _, m := range modules {
|
||||
if _, f := failed[m]; f {
|
||||
r.mu.Lock()
|
||||
var names, failedNames []string
|
||||
for _, m := range r.modules {
|
||||
r.serve(r.of[m])
|
||||
names = append(names, r.of[m].names...)
|
||||
if r.of[m].failed != "" {
|
||||
failedNames = append(failedNames, m)
|
||||
}
|
||||
}
|
||||
line := fmt.Sprintf("[mesh-tools] serving %d tool(s) for %d module(s): %s", len(names), len(modules), orNone(names))
|
||||
r.bindVerbs()
|
||||
r.mu.Unlock()
|
||||
|
||||
line := fmt.Sprintf("[mesh-tools] serving %d tool(s) for %d module(s): %s", len(names), len(r.modules), orNone(names))
|
||||
if len(failedNames) > 0 {
|
||||
line += fmt.Sprintf("; not serving %s, whose bundle(s) failed to load", strings.Join(failedNames, ", "))
|
||||
}
|
||||
logf("%s", line)
|
||||
stops = append(stops, serveSeats(conn, modules, registrations, logf))
|
||||
stopSeats, again := serveSeats(conn, r.modules, r.registrations, logf)
|
||||
r.seatsAgain = again
|
||||
r.stops = append(r.stops, stopSeats)
|
||||
|
||||
// **What it serves, it announces** (novox/hq ADR 0197): the NATS services protocol's discovery,
|
||||
// answered with what is served at the moment it is asked — re-served memberships included.
|
||||
// answered with what is served at the moment it is asked — re-served memberships and bundles loaded
|
||||
// again included, so a bundle served after start is announced as one served at start.
|
||||
announced, err := announce.Serve(conn, announce.Service{
|
||||
Name: conn.Module(), ID: instanceOf(conn),
|
||||
Description: "the mesh's tool runtime on " + node + ": every assigned module's tools and the seats they hold",
|
||||
Metadata: map[string]string{"node": node},
|
||||
}, func() []announce.Endpoint { return endpointsOf(conn, own, registrations, modules) })
|
||||
}, func() []announce.Endpoint { return endpointsOf(conn, r.own(), r.registrations(), r.modules) })
|
||||
if err != nil {
|
||||
logf("[mesh-tools] cannot announce what it serves: %v", err)
|
||||
} else {
|
||||
stops = append(stops, announced)
|
||||
r.stops = append(r.stops, announced)
|
||||
}
|
||||
|
||||
// **A bundle is served as soon as it is in place** (issue 1945, note 23): the entrypoints'
|
||||
// directories are watched, and a module whose bundle appears, changes or goes is loaded again.
|
||||
var paths []string
|
||||
for _, s := range served {
|
||||
for _, entry := range s.Entrypoints {
|
||||
path, _ := filepath.Abs(entry)
|
||||
paths = append(paths, path)
|
||||
}
|
||||
}
|
||||
if len(paths) > 0 {
|
||||
stopWatch, err := watchBundles(paths, r.reconcile, logf)
|
||||
if err != nil {
|
||||
logf("[mesh-tools] cannot watch the bundles: %v; a bundle placed or changed later is served only after a restart", err)
|
||||
} else {
|
||||
r.stops = append(r.stops, stopWatch)
|
||||
}
|
||||
}
|
||||
conn.Flush()
|
||||
return stopAll, nil
|
||||
return r.stop, nil
|
||||
}
|
||||
|
||||
// stop ends the watch, the announcement and the seats, unbinds the modules' consumers, and stops every
|
||||
// module's tools and bundles.
|
||||
func (r *runner) stop() {
|
||||
for i := len(r.stops) - 1; i >= 0; i-- {
|
||||
r.stops[i]()
|
||||
}
|
||||
r.stops = nil
|
||||
r.events.stopAll()
|
||||
r.mu.Lock()
|
||||
for _, stop := range r.verbs {
|
||||
stop()
|
||||
}
|
||||
r.verbs = map[string]func(){}
|
||||
loaded := make([]*loadedModule, 0, len(r.modules))
|
||||
for _, m := range r.modules {
|
||||
if l := r.of[m]; l != nil {
|
||||
loaded = append(loaded, l)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
for i := len(loaded) - 1; i >= 0; i-- {
|
||||
loaded[i].unload()
|
||||
}
|
||||
}
|
||||
|
||||
// claimed is every seat a served module's membership names.
|
||||
func (r *runner) claimed() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, m := range r.modules {
|
||||
if mem := r.conn.Membership(m); mem != nil {
|
||||
for _, s := range mem.Seats {
|
||||
out[s.Seat] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// load launches one module's bundles and learns what they register; it serves nothing yet.
|
||||
func (r *runner) load(s Served) *loadedModule {
|
||||
l := &loadedModule{served: s, seen: map[string]string{}}
|
||||
claimed := r.claimed()
|
||||
for _, entry := range s.Entrypoints {
|
||||
path, _ := filepath.Abs(entry)
|
||||
module := s.Module
|
||||
l.seen[path] = signatureOf(path)
|
||||
fail := func(why string) {
|
||||
l.failed = why
|
||||
r.logf("[mesh-tools] %s's bundle %s failed to load: %s; its tools are not served here", module, entry, why)
|
||||
}
|
||||
if why := notStartable(path); why != "" {
|
||||
fail(why)
|
||||
continue
|
||||
}
|
||||
child, err := launch.Start(module, path, r.envFor(module), r.events.forModule(module), r.logf)
|
||||
if err != nil {
|
||||
fail(err.Error())
|
||||
continue
|
||||
}
|
||||
l.children = append(l.children, child.Stop)
|
||||
for _, reg := range child.Registrations {
|
||||
got := registration{module: reg.Module, owner: module, tools: reg.Tools}
|
||||
l.regs = append(l.regs, got)
|
||||
switch {
|
||||
case r.isServed[got.module]:
|
||||
l.own = append(l.own, got)
|
||||
case claimed[got.module]:
|
||||
default:
|
||||
r.logf(`[mesh-tools] %s registers tools under "%s", which is neither a module served here nor a seat one of them claims; not served until the mesh issues the claim`, got.owner, got.module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// refusal is why a module's registrations cannot be served: a tool named as the verb the runtime
|
||||
// answers for every module, or two tools of one name.
|
||||
func refusal(own []registration) error {
|
||||
for _, r := range own {
|
||||
seen := map[string]bool{}
|
||||
for _, t := range r.tools {
|
||||
if t.Name == ToolsVerb {
|
||||
return fmt.Errorf(`%s names a tool "%s", which is the verb the runtime answers for every module with what it serves (novox/hq ADR 0152) — refused, rename it`, r.module, ToolsVerb)
|
||||
}
|
||||
if seen[t.Name] {
|
||||
return fmt.Errorf("%s exposes two tools named %s — refused", r.module, t.Name)
|
||||
}
|
||||
seen[t.Name] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// serve binds a loaded module's tools. r.mu is held.
|
||||
func (r *runner) serve(l *loadedModule) {
|
||||
for _, reg := range l.own {
|
||||
for _, t := range reg.tools {
|
||||
t := t
|
||||
// Told who called, from the subject the call arrived on, never from its body (novox/hq issue 365).
|
||||
stop, err := r.conn.HandleCalled(reg.module+"."+t.Name, func(caller string, body json.RawMessage) (any, error) {
|
||||
return t.Run(caller, argsOf(body))
|
||||
})
|
||||
if err != nil {
|
||||
r.logf("[mesh-tools] cannot serve %s.%s: %v", reg.module, t.Name, err)
|
||||
continue
|
||||
}
|
||||
l.names = append(l.names, reg.module+"."+t.Name)
|
||||
l.serving = append(l.serving, stop)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// toolsOf is every tool served under a module's name, and why its bundle failed, if it did. r.mu is held.
|
||||
func (r *runner) toolsOf(module string) ([]launch.Tool, string) {
|
||||
var tools []launch.Tool
|
||||
for _, m := range r.modules {
|
||||
for _, reg := range r.of[m].own {
|
||||
if reg.module == module {
|
||||
tools = append(tools, reg.tools...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return tools, r.of[module].failed
|
||||
}
|
||||
|
||||
// bindVerbs answers each module's `tools` verb where it serves a tool or its bundle failed, and not
|
||||
// where neither is so (a pure-events module is silent, as it always was). r.mu is held.
|
||||
func (r *runner) bindVerbs() {
|
||||
for _, module := range r.modules {
|
||||
module := module
|
||||
tools, why := r.toolsOf(module)
|
||||
want := len(tools) > 0 || why != ""
|
||||
stop, bound := r.verbs[module]
|
||||
switch {
|
||||
case want && !bound:
|
||||
stop, err := r.conn.Handle(module+"."+ToolsVerb, func(json.RawMessage) (any, error) {
|
||||
r.mu.Lock()
|
||||
tools, why := r.toolsOf(module)
|
||||
r.mu.Unlock()
|
||||
answer := ToolsAnswer{Module: module, Tools: []ListedTool{}, Failed: why}
|
||||
for _, t := range tools {
|
||||
answer.Tools = append(answer.Tools, ListedTool{Name: t.Name, Description: t.Description,
|
||||
Input: t.Input, Subjects: subjectsOf(r.conn, module, t.Name)})
|
||||
}
|
||||
return answer, nil
|
||||
})
|
||||
if err == nil {
|
||||
r.verbs[module] = stop
|
||||
}
|
||||
case !want && bound:
|
||||
stop()
|
||||
delete(r.verbs, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// own is every registration served under a served module's name, now.
|
||||
func (r *runner) own() []registration {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var out []registration
|
||||
for _, m := range r.modules {
|
||||
out = append(out, r.of[m].own...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// registrations is every registration the bundles made, now.
|
||||
func (r *runner) registrations() []registration {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var out []registration
|
||||
for _, m := range r.modules {
|
||||
out = append(out, r.of[m].regs...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reconcile loads again every module whose entrypoint appeared, changed or went since it was loaded.
|
||||
// The watch calls it once a burst of changes settles; it runs on the watch's one goroutine.
|
||||
func (r *runner) reconcile() {
|
||||
for _, m := range r.modules {
|
||||
r.mu.Lock()
|
||||
l := r.of[m]
|
||||
r.mu.Unlock()
|
||||
for _, entry := range l.served.Entrypoints {
|
||||
path, _ := filepath.Abs(entry)
|
||||
was, now := l.seen[path], signatureOf(path)
|
||||
if was == now {
|
||||
continue
|
||||
}
|
||||
what := "changed"
|
||||
switch {
|
||||
case was == "":
|
||||
what = "appeared"
|
||||
case now == "":
|
||||
what = "is gone"
|
||||
}
|
||||
r.reload(m, entry+" "+what)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reload stops a module's tools and bundles, loads them again, serves what they register now, and
|
||||
// serves its seats' verbs again; discovery answers with it from then on.
|
||||
func (r *runner) reload(module, what string) {
|
||||
r.mu.Lock()
|
||||
old := r.of[module]
|
||||
r.of[module] = &loadedModule{served: old.served, seen: old.seen}
|
||||
r.mu.Unlock()
|
||||
// Its events are refused first, so one heard while the old bundle stops is refused at once.
|
||||
r.events.dropModule(module)
|
||||
old.unload()
|
||||
|
||||
r.logf("[mesh-tools] %s's bundle %s; loading it again", module, what)
|
||||
fresh := r.load(old.served)
|
||||
if err := refusal(fresh.own); err != nil {
|
||||
fresh.unload()
|
||||
fresh.regs, fresh.own, fresh.failed = nil, nil, err.Error()
|
||||
r.events.dropModule(module)
|
||||
r.logf("[mesh-tools] %s's bundle is refused: %v; its tools are not served here", module, err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.of[module] = fresh
|
||||
r.serve(fresh)
|
||||
r.bindVerbs()
|
||||
r.mu.Unlock()
|
||||
if fresh.failed != "" {
|
||||
r.logf("[mesh-tools] not serving %s, whose bundle failed to load", module)
|
||||
} else {
|
||||
r.logf("[mesh-tools] serving %d tool(s) for %s: %s", len(fresh.names), module, orNone(fresh.names))
|
||||
}
|
||||
if r.seatsAgain != nil {
|
||||
r.seatsAgain()
|
||||
}
|
||||
r.conn.Flush()
|
||||
}
|
||||
|
||||
// instanceOf is this runtime's instance on the bus: its machine, which is what tells two instances of
|
||||
@@ -389,26 +605,31 @@ func subjectsOf(conn *bus.Conn, module, tool string) []string {
|
||||
|
||||
// serveSeats serves every verb of every seat a served module holds, where the mesh issued it, by
|
||||
// the tool of the same name registered under the seat's name (ADR 0159, 0160) — and serves again
|
||||
// whenever a membership changes. Whether this machine holds the seat is the bus's to decide.
|
||||
func serveSeats(conn *bus.Conn, modules []string, registrations []registration, logf func(string, ...any)) func() {
|
||||
impl := map[string]map[string]launch.Tool{}
|
||||
for _, r := range registrations {
|
||||
if impl[r.module] == nil {
|
||||
impl[r.module] = map[string]launch.Tool{}
|
||||
}
|
||||
for _, t := range r.tools {
|
||||
impl[r.module][t.Name] = t
|
||||
}
|
||||
}
|
||||
// whenever a membership changes, and when again is called because a bundle was loaded again. Whether
|
||||
// this machine holds the seat is the bus's to decide. It answers its stop and again.
|
||||
func serveSeats(conn *bus.Conn, modules []string, current func() []registration, logf func(string, ...any)) (func(), func()) {
|
||||
var mu sync.Mutex
|
||||
var stops []func()
|
||||
stopped := false
|
||||
serve := func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if stopped {
|
||||
return
|
||||
}
|
||||
for _, s := range stops {
|
||||
s()
|
||||
}
|
||||
stops = nil
|
||||
impl := map[string]map[string]launch.Tool{}
|
||||
for _, r := range current() {
|
||||
if impl[r.module] == nil {
|
||||
impl[r.module] = map[string]launch.Tool{}
|
||||
}
|
||||
for _, t := range r.tools {
|
||||
impl[r.module][t.Name] = t
|
||||
}
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, module := range modules {
|
||||
m := conn.Membership(module)
|
||||
@@ -446,7 +667,8 @@ func serveSeats(conn *bus.Conn, modules []string, registrations []registration,
|
||||
s()
|
||||
}
|
||||
stops = nil
|
||||
}
|
||||
stopped = true
|
||||
}, serve
|
||||
}
|
||||
|
||||
// consumers holds, per module the runtime serves, the one durable consumer its events arrive on and
|
||||
@@ -458,6 +680,8 @@ type consumers struct {
|
||||
of map[string]*moduleEvents
|
||||
// seatBound is each module's seat traffic bound once: a worker taken, a proof subject answered.
|
||||
seatBound map[string]func()
|
||||
// provisioned is every watch a module's code holds of its values at runtime (provision.go).
|
||||
provisioned provisionWatches
|
||||
}
|
||||
|
||||
type moduleEvents struct {
|
||||
@@ -479,6 +703,25 @@ func (c *consumers) stopAll() {
|
||||
stop()
|
||||
}
|
||||
c.seatBound = map[string]func(){}
|
||||
c.provisioned.stopAll()
|
||||
}
|
||||
|
||||
// dropModule unbinds one module's consumer, its seat traffic and its provision watches, so the bundle
|
||||
// loaded again in its place binds them anew and no event is handed to a child that was stopped.
|
||||
func (c *consumers) dropModule(module string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if m := c.of[module]; m != nil && m.stop != nil {
|
||||
m.stop()
|
||||
}
|
||||
delete(c.of, module)
|
||||
for key, stop := range c.seatBound {
|
||||
if strings.HasPrefix(key, module+" ") {
|
||||
stop()
|
||||
delete(c.seatBound, key)
|
||||
}
|
||||
}
|
||||
c.provisioned.stopModule(module)
|
||||
}
|
||||
|
||||
// forModule is the bus one launched bundle of a module reaches the mesh through.
|
||||
@@ -560,23 +803,7 @@ func (b *moduleBus) Subscribe(deliver func(json.RawMessage) error) error {
|
||||
return nil
|
||||
}
|
||||
module := b.module
|
||||
stop, err := c.conn.ConsumeAs(module, func(env bus.Envelope) error {
|
||||
raw, err := json.Marshal(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.mu.Lock()
|
||||
targets := append([]*func(json.RawMessage) error(nil), c.of[module].delivers...)
|
||||
c.mu.Unlock()
|
||||
for _, d := range targets {
|
||||
if err := (*d)(raw); err != nil {
|
||||
c.logf("[mesh-tools] %s did not take %s%s: %s; offered again in %s", module, env.Key,
|
||||
eventRef(env), whyNotTaken(err), bus.NakDelay)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
stop, err := c.conn.ConsumeAs(module, func(env bus.Envelope) error { return c.handOut(module, m, env) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -585,6 +812,35 @@ func (b *moduleBus) Subscribe(deliver func(json.RawMessage) error) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// errModuleReloading refuses an event heard by a module's consumer that was stopped for a reload.
|
||||
var errModuleReloading = errors.New("the module's bundle is being loaded again; offered again")
|
||||
|
||||
// handOut hands one event of a module's consumer m to every child of the module that subscribed; an
|
||||
// event of a consumer that is no longer the module's is refused.
|
||||
func (c *consumers) handOut(module string, m *moduleEvents, env bus.Envelope) error {
|
||||
raw, err := json.Marshal(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.mu.Lock()
|
||||
if c.of[module] != m {
|
||||
// Heard by a consumer stopped while the module is loaded again (issue 1945): refused, so the bus
|
||||
// offers it again, to the bundle loaded in its place once it subscribes.
|
||||
c.mu.Unlock()
|
||||
return errModuleReloading
|
||||
}
|
||||
targets := append([]*func(json.RawMessage) error(nil), m.delivers...)
|
||||
c.mu.Unlock()
|
||||
for _, d := range targets {
|
||||
if err := (*d)(raw); err != nil {
|
||||
c.logf("[mesh-tools] %s did not take %s%s: %s; offered again in %s", module, env.Key,
|
||||
eventRef(env), whyNotTaken(err), bus.NakDelay)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// whyNotTaken says why a bundle did not take an event, so the line names who failed: the module's
|
||||
// handler, in its own words — an error it answered — or the runtime not reaching it. A bundle's
|
||||
// handler error read bare ("Unexpected end of JSON input") looked like the runtime's (issue 276).
|
||||
@@ -612,10 +868,14 @@ type stateAsked struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Revision uint64 `json:"revision"`
|
||||
// TTLSeconds gives a created or put key its own time to live (issue 1952): create makes it only when it has no
|
||||
// value, put renews it whole. Zero is none.
|
||||
TTLSeconds int `json:"ttl_seconds"`
|
||||
}
|
||||
|
||||
// State answers a bundle's `get`, `put`, `delete` and `keys` on its module's state (ADR 0201). The
|
||||
// runtime refuses, with the reason, a state the module was not issued and a write to one it only reads.
|
||||
// State answers a bundle's `get`, `put`, `delete`, `keys`, `create`, `update`, `history` and `info` on its
|
||||
// module's state (ADR 0201, ADR 0259, ADR 0307); `create` and `put` take `ttl_seconds`, a key's own time to live,
|
||||
// whose end every watch hears as an `expire` (issue 1952). The runtime refuses, with the reason, a state the module was not issued and a write to one it only reads.
|
||||
func (b *moduleBus) State(verb string, params json.RawMessage) (json.RawMessage, error) {
|
||||
var asked stateAsked
|
||||
if err := json.Unmarshal(params, &asked); err != nil || asked.State == "" {
|
||||
@@ -633,8 +893,21 @@ func (b *moduleBus) State(verb string, params json.RawMessage) (json.RawMessage,
|
||||
return json.RawMessage("null"), nil
|
||||
}
|
||||
answer = entry
|
||||
case "history":
|
||||
// Every value the bucket still keeps for the key, oldest first (novox/hq ADR 0307).
|
||||
kept, err := conn.StateHistory(b.module, asked.State, asked.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer = kept
|
||||
case "put":
|
||||
revision, err := conn.StatePut(b.module, asked.State, asked.Key, asked.Value)
|
||||
put := conn.StatePut
|
||||
if asked.TTLSeconds != 0 {
|
||||
put = func(module, name, key string, value json.RawMessage) (uint64, error) {
|
||||
return conn.StateRenewTTL(module, name, key, value, time.Duration(asked.TTLSeconds)*time.Second)
|
||||
}
|
||||
}
|
||||
revision, err := put(b.module, asked.State, asked.Key, asked.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -644,6 +917,13 @@ func (b *moduleBus) State(verb string, params json.RawMessage) (json.RawMessage,
|
||||
return nil, err
|
||||
}
|
||||
answer = map[string]any{}
|
||||
case "info":
|
||||
// The bucket's fill against its caps, so a bundle's tools can say how full it is.
|
||||
info, err := conn.StateInfo(b.module, asked.State)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer = info
|
||||
case "keys":
|
||||
keys, err := conn.StateKeys(b.module, asked.State)
|
||||
if err != nil {
|
||||
@@ -652,13 +932,23 @@ func (b *moduleBus) State(verb string, params json.RawMessage) (json.RawMessage,
|
||||
answer = keys
|
||||
case "create":
|
||||
// Only when the key has no value: of two writers making it, one wins (novox/hq ADR 0259).
|
||||
revision, err := conn.StateCreate(b.module, asked.State, asked.Key, asked.Value)
|
||||
create := conn.StateCreate
|
||||
if asked.TTLSeconds != 0 {
|
||||
create = func(module, name, key string, value json.RawMessage) (uint64, error) {
|
||||
return conn.StateCreateTTL(module, name, key, value, time.Duration(asked.TTLSeconds)*time.Second)
|
||||
}
|
||||
}
|
||||
revision, err := create(b.module, asked.State, asked.Key, asked.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer = map[string]any{"revision": revision}
|
||||
case "update":
|
||||
// Only when the key is still at the revision read: a second answer to one ask loses (ADR 0259).
|
||||
if asked.TTLSeconds != 0 {
|
||||
return nil, fmt.Errorf("mesh/state.update gives no time to live: the bus sets one only when a key is " +
|
||||
"created; put the key with ttl_seconds to renew it whole (issue 1952)")
|
||||
}
|
||||
revision, err := conn.StateUpdate(b.module, asked.State, asked.Key, asked.Value, asked.Revision)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -670,6 +960,107 @@ func (b *moduleBus) State(verb string, params json.RawMessage) (json.RawMessage,
|
||||
return json.Marshal(answer)
|
||||
}
|
||||
|
||||
// logAsked is what a bundle names when it reaches its log (novox/hq ADR 0297).
|
||||
type logAsked struct {
|
||||
Log string `json:"log"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Expect *uint64 `json:"expect"`
|
||||
From uint64 `json:"from"`
|
||||
Keys string `json:"keys"`
|
||||
Limit int `json:"limit"`
|
||||
// Since is a read's or a watch's start as an RFC 3339 time, in place of From (novox/hq ADR 0571).
|
||||
Since string `json:"since"`
|
||||
}
|
||||
|
||||
// since is the asked start time, zero when none.
|
||||
func (a logAsked) since() (time.Time, error) {
|
||||
if a.Since == "" {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339Nano, a.Since)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("since is an RFC 3339 time (2026-10-29T09:30:00Z), not %q", a.Since)
|
||||
}
|
||||
return at, nil
|
||||
}
|
||||
|
||||
// Log answers a bundle's `append`, `read`, `newest`, `last` and `info` on its module's log and on a log its module reads
|
||||
// as `<module>.<log>` (ADR 0297, ADR 0571). The runtime
|
||||
// refuses, with the reason, a log the module was not issued; an append whose `expect` is stale is
|
||||
// refused with the code of a lost state update.
|
||||
func (b *moduleBus) Log(verb string, params json.RawMessage) (json.RawMessage, error) {
|
||||
var asked logAsked
|
||||
if err := json.Unmarshal(params, &asked); err != nil || asked.Log == "" {
|
||||
return nil, fmt.Errorf("mesh/log.%s names no log: {log, ...}", verb)
|
||||
}
|
||||
conn := b.all.conn
|
||||
var answer any
|
||||
switch verb {
|
||||
case "append":
|
||||
seq, err := conn.LogAppend(b.module, asked.Log, asked.Key, asked.Value, asked.Expect)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer = map[string]any{"sequence": seq}
|
||||
case "read":
|
||||
since, err := asked.since()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page, err := conn.LogRead(b.module, asked.Log, asked.From, since, asked.Keys, asked.Limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer = page
|
||||
case "newest":
|
||||
// The newest entry of each key the pattern names (mesh-sdk process.LogReader.Newest).
|
||||
entries, err := conn.LogNewest(b.module, asked.Log, asked.Keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer = entries
|
||||
case "last":
|
||||
entry, err := conn.LogLast(b.module, asked.Log, asked.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if entry == nil {
|
||||
return json.RawMessage("null"), nil
|
||||
}
|
||||
answer = entry
|
||||
case "info":
|
||||
info, err := conn.LogInfo(b.module, asked.Log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer = info
|
||||
default:
|
||||
return nil, fmt.Errorf("the runtime answers no mesh/log.%s", verb)
|
||||
}
|
||||
return json.Marshal(answer)
|
||||
}
|
||||
|
||||
// LogWatch hands a bundle each entry of a log, `{log, keys, from, since}`: from a sequence, from a time, or with neither
|
||||
// each new one, one at a time (novox/hq ADR 0571; mesh-sdk process.LogReader.Watch).
|
||||
func (b *moduleBus) LogWatch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
var asked logAsked
|
||||
if err := json.Unmarshal(params, &asked); err != nil || asked.Log == "" {
|
||||
return nil, fmt.Errorf("mesh/log.watch names no log: {log, keys, from, since}")
|
||||
}
|
||||
since, err := asked.since()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.all.conn.LogWatch(b.module, asked.Log, asked.Keys, asked.From, since, func(e bus.LogEntry) error {
|
||||
raw, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return deliver(raw)
|
||||
})
|
||||
}
|
||||
|
||||
// Watch hands a bundle its module's state as it is and as it changes (ADR 0201): `{state, key}`, the
|
||||
// key a pattern with `*` and `**`, empty for every key.
|
||||
func (b *moduleBus) Watch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
@@ -694,6 +1085,9 @@ type seatAsked struct {
|
||||
Worker string `json:"worker"`
|
||||
Bucket string `json:"bucket"`
|
||||
Key string `json:"key"`
|
||||
// WaitMS is how long a pull waits for a message (none when 0); Fate what a settle gives one (issue 1887 A11).
|
||||
WaitMS int `json:"wait_ms"`
|
||||
Fate string `json:"fate"`
|
||||
}
|
||||
|
||||
// Seat answers a bundle's seat traffic, each as the module and only as far as its membership lists:
|
||||
@@ -727,6 +1121,19 @@ func (b *moduleBus) Seat(verb string, params json.RawMessage, handOn func(string
|
||||
return json.RawMessage("null"), err
|
||||
}
|
||||
return json.Marshal(entry)
|
||||
case "pull":
|
||||
// One message from a worker the membership lists, never a prefetch: a build module pulls an ask only when it
|
||||
// has room, from its workers in their order (novox/hq ADR 0525 §4; issue 1887 A11).
|
||||
p, err := conn.SeatPull(b.module, asked.Worker, time.Duration(asked.WaitMS)*time.Millisecond)
|
||||
if err != nil || p == nil {
|
||||
return json.RawMessage("null"), err
|
||||
}
|
||||
return json.Marshal(p)
|
||||
case "settle":
|
||||
if err := conn.SeatSettle(b.module, asked.ID, asked.Fate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.RawMessage(`{}`), nil
|
||||
case "take":
|
||||
key := "take " + asked.Worker
|
||||
if b.bound(key) {
|
||||
|
||||
@@ -46,6 +46,7 @@ func same(t *testing.T, got, want string) {
|
||||
// The node's runtime serves five modules' bundles on one credential — two TypeScript, one broken,
|
||||
// one Python, one written against the protocol — and follows a re-issued membership (ADR 0175, 0193).
|
||||
func TestTheNodesRuntimeServesFiveModulesAndFollowsAReissuedMembership(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", true, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "anchor", false, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
@@ -152,6 +153,7 @@ func TestTheNodesRuntimeServesFiveModulesAndFollowsAReissuedMembership(t *testin
|
||||
// Each bundle is given its own environment and none of another's (ADR 0192), and the composed
|
||||
// environments are not left in the runtime's.
|
||||
func TestEachBundleIsGivenItsOwnEnvironment(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
for _, m := range []string{"gamma", "delta", "zeta"} {
|
||||
mesh.Issue(t, mt.MembershipOf(m, "anchor", false, nil))
|
||||
@@ -192,6 +194,7 @@ func TestEachBundleIsGivenItsOwnEnvironment(t *testing.T) {
|
||||
|
||||
// An entrypoint that is not executable is refused by name, and the others serve (ADR 0193).
|
||||
func TestAnEntrypointThatIsNotExecutableIsRefused(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", false, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("plain", "anchor", false, nil))
|
||||
@@ -206,7 +209,7 @@ func TestAnEntrypointThatIsNotExecutableIsRefused(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
if !logs.Has("plain's bundle", "many-alpha.mjs failed to load:", "is not executable; a bundle the runtime serves is started, never imported") {
|
||||
if !logs.Has("plain's bundle", "many-alpha.mjs failed to load:", "is not executable (mode -", "); a bundle the runtime serves is started, never imported") {
|
||||
t.Errorf("the non-executable entrypoint was not refused by name:\n%s", logs.All())
|
||||
}
|
||||
got, err := call(t, asker, "alpha.one@anchor", map[string]any{})
|
||||
@@ -218,6 +221,7 @@ func TestAnEntrypointThatIsNotExecutableIsRefused(t *testing.T) {
|
||||
|
||||
// A launched bundle is told the module it serves, so its seat's verbs stay the seat's (ADR 0193).
|
||||
func TestALaunchedBundleRegisteringItsSeatFirstServesTheSeat(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("theta", "anchor", false, map[string][]string{"node-shelf": {"list"}}))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
|
||||
@@ -147,3 +147,65 @@ func TestALostCompareAndSetCarriesItsCode(t *testing.T) {
|
||||
t.Error("an update of revision zero was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A11: a bundle pulls one message from its own worker and settles it through its module's bus; another module may
|
||||
// neither pull that worker nor settle what it did not pull.
|
||||
func TestABundlePullsAndSettlesThroughItsModulesBus(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, _ := nc.JetStream()
|
||||
if _, err := js.AddStream(&nats.StreamConfig{Name: "SEAT_NODE_BUILD_GO", Retention: nats.WorkQueuePolicy,
|
||||
Subjects: []string{"mesh.seat.node-build-go.accept.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.AddConsumer("SEAT_NODE_BUILD_GO", &nats.ConsumerConfig{Durable: "SEAT_NODE_BUILD_GO_DELIVERY_worker",
|
||||
AckPolicy: nats.AckExplicitPolicy, FilterSubject: "mesh.seat.node-build-go.accept.delivery"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Publish("mesh.seat.node-build-go.accept.delivery", []byte(`{"id":"ask1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holder := mt.MembershipOf("mesh-build-go", "anchor", false, nil)
|
||||
holder.SeatTraffic = &bus.SeatTraffic{Workers: []bus.SeatWorker{{Stream: "SEAT_NODE_BUILD_GO",
|
||||
Consumer: "SEAT_NODE_BUILD_GO_DELIVERY_worker", Filter: "mesh.seat.node-build-go.accept.delivery"}}}
|
||||
other := mt.MembershipOf("mesh-build-node", "anchor", false, nil)
|
||||
other.SeatTraffic = &bus.SeatTraffic{}
|
||||
mesh.Issue(t, holder)
|
||||
mesh.Issue(t, other)
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("mesh-build-go")
|
||||
conn.Follow("mesh-build-node")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("mesh-build-go") == nil || conn.Membership("mesh-build-node") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
building := all.forModule("mesh-build-go").(*moduleBus)
|
||||
elsewhere := all.forModule("mesh-build-node").(*moduleBus)
|
||||
if _, err := elsewhere.Seat("pull", json.RawMessage(`{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker"}`), nil); err == nil {
|
||||
t.Fatal("another module pulled the Go build seat's worker")
|
||||
}
|
||||
raw, err := building.Seat("pull", json.RawMessage(`{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker","wait_ms":1000}`), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var p bus.Pulled
|
||||
if err := json.Unmarshal(raw, &p); err != nil || string(p.Body) != `{"id":"ask1"}` || p.ID == "" {
|
||||
t.Fatalf("pulled %s (%v)", raw, err)
|
||||
}
|
||||
if _, err := elsewhere.Seat("settle", json.RawMessage(`{"id":"`+p.ID+`","fate":"ack"}`), nil); err == nil {
|
||||
t.Fatal("another module settled the pulled ask")
|
||||
}
|
||||
if _, err := building.Seat("settle", json.RawMessage(`{"id":"`+p.ID+`","fate":"ack"}`), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if none, err := building.Seat("pull", json.RawMessage(`{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker"}`), nil); err != nil || string(none) != "null" {
|
||||
t.Fatalf("an empty worker answered %s, %v", none, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
@@ -18,8 +20,10 @@ func counterBus(t *testing.T) launch.Bus {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
mesh.Bucket(t, "issues_numbers")
|
||||
mesh.BucketKeyTTL(t, "issues_wakes")
|
||||
m := mt.MembershipOf("issues", "anchor", false, nil)
|
||||
m.State = []bus.StateIssued{{Name: "numbers", Bucket: "issues_numbers", Writes: true}}
|
||||
m.State = []bus.StateIssued{{Name: "numbers", Bucket: "issues_numbers", Writes: true},
|
||||
{Name: "wakes", Bucket: "issues_wakes", Writes: true}}
|
||||
mesh.Issue(t, m)
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("issues")
|
||||
@@ -131,6 +135,9 @@ func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) {
|
||||
// A write at a revision that is no longer the key's is refused with the lost-write code and writes
|
||||
// nothing; create is refused for a key with a value and taken for one deleted; an update names the
|
||||
// revision it read, never zero.
|
||||
// A key's own time to live (issue 1952, mesh-sdk process.State.CreateTTL and RenewTTL): created once, renewed whole by a
|
||||
// put, its end heard by a watch as an expiry; refused, writing nothing, on a bucket that takes none, under a second, and
|
||||
// on an update.
|
||||
func TestAStaleWriteIsRefusedAndWritesNothing(t *testing.T) {
|
||||
b := counterBus(t)
|
||||
state := func(verb string, extra map[string]any) (uint64, error) {
|
||||
@@ -200,4 +207,63 @@ func TestAStaleWriteIsRefusedAndWritesNothing(t *testing.T) {
|
||||
if v := value(); v != "1" {
|
||||
t.Fatalf("after delete and create: %s", v)
|
||||
}
|
||||
|
||||
// A time to live per key: refused where it cannot be kept, writing nothing.
|
||||
for _, c := range []struct {
|
||||
verb, state, why string
|
||||
ttl int
|
||||
}{
|
||||
{"create", "numbers", "does not take keys with their own time to live", 60},
|
||||
{"put", "numbers", "does not take keys with their own time to live", 60},
|
||||
{"create", "wakes", "shorter than the bus keeps", -1},
|
||||
{"update", "wakes", "gives no time to live", 60},
|
||||
} {
|
||||
_, err := ask(b, c.verb, map[string]any{"state": c.state, "key": "late", "value": 1, "revision": 1, "ttl_seconds": c.ttl})
|
||||
if err == nil || !strings.Contains(err.Error(), c.why) {
|
||||
t.Errorf("%s with ttl %d on %s: want refused for %q, got %v", c.verb, c.ttl, c.state, c.why, err)
|
||||
}
|
||||
if raw, _ := ask(b, "get", map[string]any{"state": c.state, "key": "late"}); string(raw) != "null" {
|
||||
t.Errorf("%s with ttl %d on %s wrote %s", c.verb, c.ttl, c.state, raw)
|
||||
}
|
||||
}
|
||||
|
||||
// The bucket's fill against its caps, for a bundle's tools.
|
||||
if raw, err := ask(b, "info", map[string]any{"state": "wakes"}); err != nil || !strings.Contains(string(raw),
|
||||
`"bucket":"issues_wakes","bytes":0,"keys":0,"max_bytes":-1,"max_value_bytes":262144,"history":1,"ttl_seconds":0,"key_ttl":true`) {
|
||||
t.Errorf("info of a key-ttl bucket answered %s, %v", raw, err)
|
||||
}
|
||||
|
||||
// Created once, renewed whole, and its end heard: a wake with no timer.
|
||||
heard := make(chan bus.StateChange, 8)
|
||||
stop, err := b.Watch(json.RawMessage(`{"state":"wakes"}`), func(raw json.RawMessage) error {
|
||||
var c bus.StateChange
|
||||
_ = json.Unmarshal(raw, &c)
|
||||
heard <- c
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
if _, err := ask(b, "create", map[string]any{"state": "wakes", "key": "settle", "value": 1, "ttl_seconds": 60}); err != nil {
|
||||
t.Fatalf("create with a time to live: %v", err)
|
||||
}
|
||||
if _, err := ask(b, "create", map[string]any{"state": "wakes", "key": "settle", "value": 2, "ttl_seconds": 60}); !isLost(err) {
|
||||
t.Fatalf("a second create with a time to live: %v", err)
|
||||
}
|
||||
if _, err := ask(b, "put", map[string]any{"state": "wakes", "key": "settle", "value": 3, "ttl_seconds": 1}); err != nil {
|
||||
t.Fatalf("a put renewing the time to live: %v", err)
|
||||
}
|
||||
var ops []string
|
||||
for want := "put delete put expire"; strings.Join(ops, " ") != want; {
|
||||
select {
|
||||
case c := <-heard:
|
||||
ops = append(ops, c.Op)
|
||||
if c.Op == "put" && len(ops) == 3 && string(c.Value) != "3" {
|
||||
t.Fatalf("the renewed key holds %s", c.Value)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the watch heard %q, want %q", strings.Join(ops, " "), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// historyBuses are three modules on one runtime, each reached as a bundle reaches it (novox/hq ADR
|
||||
// 0307): issues keeps a state `issues` in a bucket keeping 8 values a key, triage reads it as
|
||||
// `issues.issues`, and other was issued no state.
|
||||
func historyBuses(t *testing.T) (owner, reader, other launch.Bus) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
mesh.BucketKeeping(t, "issues_issues", 8)
|
||||
m := mt.MembershipOf("issues", "anchor", false, nil)
|
||||
m.State = []bus.StateIssued{{Name: "issues", Bucket: "issues_issues", Writes: true}}
|
||||
mesh.Issue(t, m)
|
||||
r := mt.MembershipOf("triage", "anchor", false, nil)
|
||||
r.State = []bus.StateIssued{{Name: "issues.issues", Bucket: "issues_issues"}}
|
||||
mesh.Issue(t, r)
|
||||
mesh.Issue(t, mt.MembershipOf("other", "anchor", false, nil))
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
for _, module := range []string{"issues", "triage", "other"} {
|
||||
conn.Follow(module)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
for _, module := range []string{"issues", "triage", "other"} {
|
||||
if conn.Membership(module) == nil {
|
||||
return errors.New(module + " not issued yet")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
return all.forModule("issues"), all.forModule("triage"), all.forModule("other")
|
||||
}
|
||||
|
||||
// kept is one value of a history answer, by the field names on the wire.
|
||||
type kept struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Op string `json:"op"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
func historyOf(t *testing.T, b launch.Bus, state, key string) []kept {
|
||||
t.Helper()
|
||||
raw, err := ask(b, "history", map[string]any{"state": state, "key": key})
|
||||
if err != nil {
|
||||
t.Fatalf("history of %s %s: %v", state, key, err)
|
||||
}
|
||||
var h []kept
|
||||
if err := json.Unmarshal(raw, &h); err != nil || h == nil {
|
||||
t.Fatalf("history answered %s, not an array", raw)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// A key's history is every value the bucket keeps, oldest first, each with its revision, op and time
|
||||
// in RFC 3339 — a delete with no value; an unknown key's is []. The owner and a reader are served it,
|
||||
// and reading it writes nothing.
|
||||
func TestAKeysHistoryIsEveryValueKeptOldestFirst(t *testing.T) {
|
||||
owner, reader, _ := historyBuses(t)
|
||||
before := time.Now().Add(-time.Second)
|
||||
var revisions []uint64
|
||||
for _, v := range []string{`{"state":"open"}`, `{"state":"fixing"}`, `{"state":"resolved"}`} {
|
||||
raw, err := ask(owner, "put", map[string]any{"state": "issues", "key": "502", "value": json.RawMessage(v)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var r struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
revisions = append(revisions, r.Revision)
|
||||
}
|
||||
if _, err := ask(owner, "put", map[string]any{"state": "issues", "key": "503", "value": 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ask(owner, "delete", map[string]any{"state": "issues", "key": "502"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for who, b := range map[string]launch.Bus{"the owner": owner, "a reader": reader} {
|
||||
name := "issues"
|
||||
if b == reader {
|
||||
name = "issues.issues"
|
||||
}
|
||||
h := historyOf(t, b, name, "502")
|
||||
if len(h) != 4 {
|
||||
t.Fatalf("%s read %d values of 502, not 4: %+v", who, len(h), h)
|
||||
}
|
||||
for i, v := range []string{`{"state":"open"}`, `{"state":"fixing"}`, `{"state":"resolved"}`} {
|
||||
if h[i].Op != "put" || string(h[i].Value) != v || h[i].Revision != revisions[i] || h[i].Key != "502" {
|
||||
t.Errorf("%s: value %d is %+v", who, i, h[i])
|
||||
}
|
||||
}
|
||||
if h[3].Op != "delete" || len(h[3].Value) != 0 && string(h[3].Value) != "null" || h[3].Revision <= revisions[2] {
|
||||
t.Errorf("%s: the delete reads %+v", who, h[3])
|
||||
}
|
||||
for i, k := range h {
|
||||
at, err := time.Parse(time.RFC3339, k.At)
|
||||
if err != nil || at.Before(before) || at.After(time.Now().Add(time.Second)) {
|
||||
t.Errorf("%s: value %d's time %q is not RFC 3339 and now: %v", who, i, k.At, err)
|
||||
}
|
||||
if i > 0 && k.Revision <= h[i-1].Revision {
|
||||
t.Errorf("%s: not oldest first: %+v", who, h)
|
||||
}
|
||||
}
|
||||
if h := historyOf(t, b, name, "999"); len(h) != 0 {
|
||||
t.Errorf("%s: an unknown key's history is %+v, not []", who, h)
|
||||
}
|
||||
}
|
||||
// The history is read, never written: the key is still deleted and keeps the same four values.
|
||||
if raw, err := ask(owner, "get", map[string]any{"state": "issues", "key": "502"}); err != nil || string(raw) != "null" {
|
||||
t.Fatalf("after history, 502 reads %s %v", raw, err)
|
||||
}
|
||||
if h := historyOf(t, owner, "issues", "502"); len(h) != 4 {
|
||||
t.Fatalf("reading the history changed it: %+v", h)
|
||||
}
|
||||
// A key the bucket holds nothing for answers [] on the wire, not null.
|
||||
if raw, _ := ask(owner, "history", map[string]any{"state": "issues", "key": "999"}); string(raw) != "[]" {
|
||||
t.Fatalf("an unknown key's history is %s on the wire", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// A module the bucket was not issued to is refused its history with the reason, as it is a get.
|
||||
func TestAKeysHistoryIsRefusedToAModuleNotIssuedTheState(t *testing.T) {
|
||||
owner, _, other := historyBuses(t)
|
||||
if _, err := ask(owner, "put", map[string]any{"state": "issues", "key": "1", "value": 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"issues", "issues.issues"} {
|
||||
_, err := ask(other, "history", map[string]any{"state": name, "key": "1"})
|
||||
if err == nil || !strings.Contains(err.Error(), "keeps and reads no state called") ||
|
||||
!strings.Contains(err.Error(), "issued: none") {
|
||||
t.Errorf("history of %s to a module not issued it: %v", name, err)
|
||||
}
|
||||
}
|
||||
if _, err := ask(owner, "history", map[string]any{"state": "issues", "key": "a b"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "not a key") {
|
||||
t.Errorf("history of a key the bus cannot hold: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -139,6 +139,7 @@ func waitFor(t *testing.T, path, line string) {
|
||||
// The TypeScript SDK against the runtime (ADR 0201): two watches of one state, each handed only its
|
||||
// own changes, the current values handled before the module's import goes on.
|
||||
func TestTheSDKsStateReachesTheRuntime(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
kv := mesh.Bucket(t, "sdkkeeper_servers")
|
||||
if _, err := kv.Put("all.one", []byte(`{"url":"http://one"}`)); err != nil {
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Settle is how long the bundles' directories stay quiet before the runtime acts on what changed there:
|
||||
// a delivery unpacks a bundle in many writes, and the runtime loads it once, when they are done
|
||||
// (issue 1945, note 23).
|
||||
var Settle = 300 * time.Millisecond
|
||||
|
||||
// watchMask is what the runtime hears of a watched directory: a file or directory made, written,
|
||||
// renamed in or out, removed or given another mode, and the directory itself removed or renamed.
|
||||
const watchMask = unix.IN_CREATE | unix.IN_CLOSE_WRITE | unix.IN_MOVED_TO | unix.IN_MOVED_FROM | unix.IN_DELETE |
|
||||
unix.IN_ATTRIB | unix.IN_DELETE_SELF | unix.IN_MOVE_SELF | unix.IN_ONLYDIR
|
||||
|
||||
// bundleWatch hears, through inotify, the directory of every entrypoint the runtime serves, or for one
|
||||
// not there yet the nearest directory above it that is, and calls changed once a burst of changes has
|
||||
// settled. Nothing is looked at on a timer: between bursts the watch sleeps in poll(2).
|
||||
type bundleWatch struct {
|
||||
fd int
|
||||
wake [2]int // a pipe: a byte written to wake[1] ends the loop
|
||||
paths []string
|
||||
byWD map[int]string
|
||||
settle time.Duration
|
||||
changed func()
|
||||
logf func(string, ...any)
|
||||
done chan struct{}
|
||||
// what names what is watched, and lost what follows when the watch fails, in the words a journal line carries.
|
||||
what, lost string
|
||||
}
|
||||
|
||||
// WatchFile hears, as the bundles are heard, the directory of the file at path, or the nearest directory above it that
|
||||
// exists, and calls changed once at its start and once after each burst of changes there has settled: a runtime waiting
|
||||
// for its broker credential reads it again when the node-engine writes it, never on a clock (issue 1496). It answers its
|
||||
// stop.
|
||||
func WatchFile(path string, changed func(), logf func(string, ...any), lost string) (func(), error) {
|
||||
return watch([]string{path}, changed, logf, path, lost)
|
||||
}
|
||||
|
||||
// watchBundles starts the watch of the entrypoints at paths. It answers its stop.
|
||||
func watchBundles(paths []string, changed func(), logf func(string, ...any)) (func(), error) {
|
||||
return watch(paths, changed, logf, "the bundles", "a bundle placed or changed there is served only after a restart")
|
||||
}
|
||||
|
||||
// watch starts the watch of the files at paths, named what in its words; lost says what follows when it fails.
|
||||
func watch(paths []string, changed func(), logf func(string, ...any), what, lost string) (func(), error) {
|
||||
fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inotify: %w", err)
|
||||
}
|
||||
w := &bundleWatch{fd: fd, paths: paths, byWD: map[int]string{}, settle: Settle, changed: changed, logf: logf,
|
||||
done: make(chan struct{}), what: what, lost: lost}
|
||||
if err := unix.Pipe2(w.wake[:], unix.O_CLOEXEC|unix.O_NONBLOCK); err != nil {
|
||||
unix.Close(fd)
|
||||
return nil, fmt.Errorf("pipe: %w", err)
|
||||
}
|
||||
w.arm()
|
||||
go w.loop()
|
||||
return func() {
|
||||
_, _ = unix.Write(w.wake[1], []byte{1})
|
||||
<-w.done
|
||||
unix.Close(w.fd)
|
||||
unix.Close(w.wake[0])
|
||||
unix.Close(w.wake[1])
|
||||
}, nil
|
||||
}
|
||||
|
||||
// arm watches, for every entrypoint, its directory, or the nearest directory above it that exists. A
|
||||
// watch already held on the same directory is kept (inotify answers its descriptor again). A directory
|
||||
// made below the watched one between its stat and its watch would be heard by nobody, so after each watch
|
||||
// the path is looked at again, and a deeper directory that came to be is watched in turn.
|
||||
func (w *bundleWatch) arm() {
|
||||
for _, p := range w.paths {
|
||||
target := filepath.Dir(p)
|
||||
watched := ""
|
||||
for {
|
||||
dir := nearestDir(target)
|
||||
if dir == watched {
|
||||
break
|
||||
}
|
||||
wd, err := unix.InotifyAddWatch(w.fd, dir, watchMask)
|
||||
if err != nil {
|
||||
w.logf("[mesh-tools] cannot watch %s for %s: %v; %s", dir, p, err, w.lost)
|
||||
break
|
||||
}
|
||||
w.byWD[wd] = dir
|
||||
watched = dir
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// nearestDir is dir when it is a directory, else the nearest directory above it.
|
||||
func nearestDir(dir string) string {
|
||||
for {
|
||||
if info, err := os.Stat(dir); err == nil && info.IsDir() {
|
||||
return dir
|
||||
}
|
||||
up := filepath.Dir(dir)
|
||||
if up == dir {
|
||||
return dir
|
||||
}
|
||||
dir = up
|
||||
}
|
||||
}
|
||||
|
||||
func (w *bundleWatch) loop() {
|
||||
defer close(w.done)
|
||||
buf := make([]byte, 64*1024)
|
||||
// Looked at once as the watch begins: a bundle that came between its load and the watch is not missed.
|
||||
pending := true
|
||||
quietAt := time.Now()
|
||||
for {
|
||||
timeout := -1
|
||||
if pending {
|
||||
timeout = int((time.Until(quietAt) + time.Millisecond - 1) / time.Millisecond)
|
||||
if timeout < 0 {
|
||||
timeout = 0
|
||||
}
|
||||
}
|
||||
fds := []unix.PollFd{{Fd: int32(w.fd), Events: unix.POLLIN}, {Fd: int32(w.wake[0]), Events: unix.POLLIN}}
|
||||
n, err := unix.Poll(fds, timeout)
|
||||
if errors.Is(err, unix.EINTR) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
w.logf("[mesh-tools] the watch of %s stopped: %v; %s", w.what, err, w.lost)
|
||||
return
|
||||
}
|
||||
if fds[1].Revents != 0 {
|
||||
return
|
||||
}
|
||||
if n == 0 {
|
||||
// Quiet for the settle: watch the directories that came to be, then act on what changed.
|
||||
pending = false
|
||||
w.arm()
|
||||
w.changed()
|
||||
continue
|
||||
}
|
||||
if fds[0].Revents&unix.POLLIN != 0 {
|
||||
w.drain(buf)
|
||||
pending = true
|
||||
quietAt = time.Now().Add(w.settle)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// drain reads every event queued, dropping the watches of directories that went away or moved: the
|
||||
// next arm watches what is at their path now.
|
||||
func (w *bundleWatch) drain(buf []byte) {
|
||||
for {
|
||||
n, err := unix.Read(w.fd, buf)
|
||||
if err != nil || n <= 0 {
|
||||
return
|
||||
}
|
||||
for off := 0; off+unix.SizeofInotifyEvent <= n; {
|
||||
wd := int(int32(binary.NativeEndian.Uint32(buf[off:])))
|
||||
mask := binary.NativeEndian.Uint32(buf[off+4:])
|
||||
length := int(binary.NativeEndian.Uint32(buf[off+12:]))
|
||||
switch {
|
||||
case mask&unix.IN_IGNORED != 0:
|
||||
delete(w.byWD, wd)
|
||||
case mask&unix.IN_MOVE_SELF != 0:
|
||||
_, _ = unix.InotifyRmWatch(w.fd, uint32(wd))
|
||||
delete(w.byWD, wd)
|
||||
}
|
||||
off += unix.SizeofInotifyEvent + length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// signatureOf is what the runtime compares to tell that an entrypoint changed: the file's identity,
|
||||
// mode, size and times; "" is a file that does not exist. Only the entrypoint: a delivery puts every file
|
||||
// of a bundle in place anew (a directory swapped in, or each file renamed in), so the entrypoint is new
|
||||
// whenever its build is, while a file beside it may be one the bundle itself writes (a cache, a socket),
|
||||
// which would load it again after every quiet spell, and modules sharing a directory each other.
|
||||
func signatureOf(path string) string {
|
||||
info, err := os.Stat(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return ""
|
||||
}
|
||||
if err != nil {
|
||||
return "unreadable: " + err.Error()
|
||||
}
|
||||
return statSignature(info)
|
||||
}
|
||||
|
||||
func statSignature(info fs.FileInfo) string {
|
||||
sig := fmt.Sprintf("%s %d %d", info.Mode(), info.Size(), info.ModTime().UnixNano())
|
||||
if st, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
sig += fmt.Sprintf(" %d:%d %d.%d", st.Dev, st.Ino, st.Ctim.Sec, st.Ctim.Nsec)
|
||||
}
|
||||
return sig
|
||||
}
|
||||
|
||||
// notStartable says why the runtime cannot start an entrypoint, in its real reason, or "" when it can
|
||||
// (issue 1945, note 23: a missing file was said as "not executable").
|
||||
func notStartable(path string) string {
|
||||
info, err := os.Stat(path)
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
return path + " does not exist; it is served as soon as it is in place"
|
||||
case err != nil:
|
||||
return path + " cannot be read: " + err.Error()
|
||||
case info.IsDir():
|
||||
return path + " is a directory, not a bundle's entrypoint"
|
||||
case info.Mode().Perm()&0o111 == 0:
|
||||
return fmt.Sprintf("%s is not executable (mode %s); a bundle the runtime serves is started, never imported, "+
|
||||
"and its build makes it executable (novox/hq ADR 0193)", path, info.Mode().Perm())
|
||||
}
|
||||
return ""
|
||||
}
|
||||
+24
-2
@@ -5,10 +5,21 @@
|
||||
"invokes": [
|
||||
"*"
|
||||
],
|
||||
"waits": [
|
||||
"mesh-knowledge.reviewed",
|
||||
"mesh-delivery-gate.transition",
|
||||
"gitea.forge.status",
|
||||
"mesh-self-check.checked",
|
||||
"operator-channel.decided.claude-code"
|
||||
],
|
||||
"provides": [
|
||||
{
|
||||
"name": "mcp-endpoint",
|
||||
"scope": "node"
|
||||
"scope": "node",
|
||||
"credential": {
|
||||
"form": "none"
|
||||
},
|
||||
"answered-by": "helper"
|
||||
}
|
||||
],
|
||||
"serves": {
|
||||
@@ -52,5 +63,16 @@
|
||||
"binary": "node-tools"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"state": [
|
||||
"bound"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-provisioner.provisions",
|
||||
"mesh-provisioner.plans",
|
||||
"mesh-delivery-assigner.seat-holders",
|
||||
"mesh-delivery-assigner.placement",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-indexer.manifests"
|
||||
]
|
||||
}
|
||||
|
||||
Generated
+1771
-2
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,9 @@
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.20.1",
|
||||
"typescript": "^5.9.3",
|
||||
"esbuild": "^0.25.0"
|
||||
"esbuild": "^0.25.0",
|
||||
"mongodb": "^6.21.0",
|
||||
"mssql": "^11.0.2",
|
||||
"pg": "^8"
|
||||
}
|
||||
}
|
||||
|
||||
+33
-15
@@ -26,9 +26,28 @@ import { TOOLS_VERB, type ToolsAnswer } from "./runtime.js";
|
||||
/** Where the catalogue answers which modules the mesh holds. */
|
||||
const CATALOGUE_MODULES = "mesh-catalog.catalog_modules";
|
||||
|
||||
/** Where the mesh answers every role's tools, from its records: the mesh-controller seat's own
|
||||
* `tools` verb (novox/hq ADR 0154, design 33 §5). */
|
||||
const SEAT_TOOLS = "seat:mesh-controller.tools";
|
||||
/** Where every role's tools are answered: the console on this machine, the Go runtime, which holds mesh-sdk's seat
|
||||
* definitions and reads who holds each seat from placement (novox/hq ADR 0571 clauses 4 and 5). Asked over the loopback
|
||||
* channel `mesh --console` already uses, by the method `mesh/seats` beside MCP's; never the controller. */
|
||||
export const SEATS_METHOD = "mesh/seats";
|
||||
const DEFAULT_CONSOLE = "http://127.0.0.1:4270";
|
||||
|
||||
type SeatsAnswer = { seats?: { seat: string; scope?: string; tools?: ToolsAnswer["tools"] }[]; holders_why?: string };
|
||||
|
||||
/** Every role's tools from the console on this machine. */
|
||||
async function seatsFromConsole(consoleUrl: string): Promise<SeatsAnswer> {
|
||||
const endpoint = consoleUrl.endsWith("/mcp") ? consoleUrl : `${consoleUrl.replace(/\/$/, "")}/mcp`;
|
||||
const res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", accept: "application/json" },
|
||||
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: SEATS_METHOD }),
|
||||
signal: AbortSignal.timeout(5_000),
|
||||
});
|
||||
if (!res.ok) throw new Error(`the console at ${endpoint} answered ${res.status}`);
|
||||
const reply = (await res.json()) as { result?: SeatsAnswer; error?: { message: string } };
|
||||
if (reply.error) throw new Error(reply.error.message);
|
||||
return reply.result ?? {};
|
||||
}
|
||||
|
||||
/** A tool as its module describes it. */
|
||||
export interface Tool {
|
||||
@@ -58,8 +77,8 @@ export interface Tool {
|
||||
export interface Listing {
|
||||
tools: Tool[];
|
||||
/** Modules the catalogue holds whose runtime did not answer `tools`: not assigned, not up, or built
|
||||
* before the runtime answered it. Each may still be called by name. The mesh's own records are
|
||||
* listed here as `mesh-controller (seat)` when the control plane did not answer. */
|
||||
* before the runtime answered it. Each may still be called by name. The roles' tools are listed here, with the
|
||||
* reason, when the console on this machine did not answer them. */
|
||||
notAnswering: string[];
|
||||
}
|
||||
|
||||
@@ -159,17 +178,16 @@ export async function connectAsTheConsole(path: string): Promise<{ bus: Broker;
|
||||
* list that goes stale the first time a module is assigned. Every module is asked at once, and the bus
|
||||
* refuses at once a request nothing serves, so the cost is bounded by the modules that are up.
|
||||
*/
|
||||
export async function toolsOn(bus: Broker): Promise<Listing> {
|
||||
export async function toolsOn(bus: Broker, consoleUrl: string = process.env.MESH_CONSOLE || DEFAULT_CONSOLE): Promise<Listing> {
|
||||
let rolesWhy = "";
|
||||
const [answered, roles] = await Promise.all([
|
||||
bus.request<Record<string, never>, { modules?: { module: string }[] }>(CATALOGUE_MODULES, {}),
|
||||
// The roles' tools, from the mesh's records (design 33 §5). Asked beside the modules rather
|
||||
// than first: a control plane that is restarting must not hide every module's tools with it.
|
||||
bus
|
||||
.request<Record<string, never>, { seats?: { seat: string; scope?: string; tools?: ToolsAnswer["tools"] }[] }>(
|
||||
SEAT_TOOLS,
|
||||
{},
|
||||
)
|
||||
.catch(() => undefined),
|
||||
// The roles' tools, from the console on this machine (ADR 0571). Asked beside the modules rather than first: a
|
||||
// console that is restarting must not hide every module's tools with it.
|
||||
seatsFromConsole(consoleUrl).catch((e: unknown) => {
|
||||
rolesWhy = e instanceof Error ? e.message : String(e);
|
||||
return undefined;
|
||||
}),
|
||||
]);
|
||||
const names = (answered.modules ?? []).map((m) => m.module).filter((m) => typeof m === "string");
|
||||
|
||||
@@ -188,7 +206,7 @@ export async function toolsOn(bus: Broker): Promise<Listing> {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
notAnswering.push("mesh-controller (seat)");
|
||||
notAnswering.push(`the roles' tools (the console at ${consoleUrl} did not answer ${SEATS_METHOD}: ${rolesWhy})`);
|
||||
}
|
||||
asked.forEach((outcome, i) => {
|
||||
const module = names[i]!;
|
||||
|
||||
@@ -16,6 +16,8 @@ import { test } from "node:test";
|
||||
import { connectNats } from "../dist/broker-nats.js";
|
||||
import { callTool, seatsIn, toolKey, toolsOn, whyItFailed } from "../dist/client.js";
|
||||
|
||||
import { aConsoleWithSeats } from "./console-seats.ts";
|
||||
|
||||
const url = process.env.MESH_TEST_NATS;
|
||||
|
||||
/** A mesh as discovery sees it (design 34 §3): the catalogue holding three modules, two of them up
|
||||
@@ -36,21 +38,26 @@ async function aMeshWithTools() {
|
||||
tools: [{ name: "price", description: "what something costs", input: { type: "object" } }],
|
||||
}));
|
||||
await shop.handle("price", async (body: { of?: string }) => ({ of: body.of ?? "nothing", cost: 12 }));
|
||||
// And the mesh's own records, served by the holder of the mesh-controller seat (ADR 0154): one
|
||||
// role's tool, and the verb that lists every role's.
|
||||
// Every role's tools, from the console on this machine (ADR 0571), the method beside MCP's; the controller's
|
||||
// verb answers a call of its seat and is never asked for the roles.
|
||||
const controller = await connectNats({ url: url!, module: "mesh-controller" });
|
||||
await controller.handle("seat:mesh-controller.tools", async () => ({
|
||||
seats: [
|
||||
{ seat: "mesh-controller", scope: "mesh", tools: [{ name: "status", description: "what is wrong", input: {} }] },
|
||||
{ seat: "node-dns-resolver", scope: "node", tools: [{ name: "lookup", description: "one machine's", input: {} }] },
|
||||
],
|
||||
}));
|
||||
let controllerToolsAsked = 0;
|
||||
await controller.handle("seat:mesh-controller.tools", async () => {
|
||||
controllerToolsAsked++;
|
||||
return { seats: [] };
|
||||
});
|
||||
await controller.handle("seat:mesh-controller.status", async () => ({ output: "all quiet", ok: true }));
|
||||
const local = await aConsoleWithSeats([
|
||||
{ seat: "mesh-controller", scope: "mesh", tools: [{ name: "status", description: "what is wrong", input: {} }] },
|
||||
{ seat: "node-dns-resolver", scope: "node", tools: [{ name: "lookup", description: "one machine's", input: {} }] },
|
||||
]);
|
||||
return {
|
||||
async close() {
|
||||
await catalogue.close();
|
||||
await shop.close();
|
||||
await controller.close();
|
||||
await local.close();
|
||||
assert.equal(controllerToolsAsked, 0, "the controller was asked for the roles' tools");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* The console on this machine as a client asks it for every role's tools (novox/hq ADR 0571): the method `mesh/seats`
|
||||
* beside MCP's, answered in the shape the Go console's `seatsAnswer` gives (internal/console/mcp.go). Sets
|
||||
* MESH_CONSOLE to it, so a client, and a `mesh` it spawns, asks it.
|
||||
*/
|
||||
import { createServer } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
import { SEATS_METHOD } from "../dist/client.js";
|
||||
|
||||
export type SeatOfConsole = { seat: string; scope: string; tools: { name: string; description?: string; input?: unknown }[] };
|
||||
|
||||
export async function aConsoleWithSeats(seats: SeatOfConsole[]): Promise<{ url: string; close: () => Promise<void> }> {
|
||||
const local = createServer((req, res) => {
|
||||
let body = "";
|
||||
req.on("data", (c) => (body += c));
|
||||
req.on("end", () => {
|
||||
const asked = JSON.parse(body) as { id: number; method: string };
|
||||
res.setHeader("content-type", "application/json");
|
||||
if (req.url !== "/mcp" || asked.method !== SEATS_METHOD) {
|
||||
res.end(JSON.stringify({ jsonrpc: "2.0", id: asked.id, error: { code: -32601, message: "no such method" } }));
|
||||
return;
|
||||
}
|
||||
res.end(JSON.stringify({ jsonrpc: "2.0", id: asked.id, result: { seats: seats.map((s) => ({ ...s, holders: [] })) } }));
|
||||
});
|
||||
});
|
||||
await new Promise<void>((done) => local.listen(0, "127.0.0.1", done));
|
||||
const url = `http://127.0.0.1:${(local.address() as AddressInfo).port}`;
|
||||
process.env.MESH_CONSOLE = url;
|
||||
return { url, close: () => new Promise((done) => local.close(() => done())) };
|
||||
}
|
||||
+12
-10
@@ -12,6 +12,8 @@ import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
|
||||
import { aConsoleWithSeats } from "./console-seats.ts";
|
||||
import { connectNats } from "../dist/broker-nats.js";
|
||||
|
||||
const url = process.env.MESH_TEST_NATS;
|
||||
@@ -29,16 +31,15 @@ async function aMeshAndACredential(t: { after: (fn: () => Promise<void> | void)
|
||||
}));
|
||||
await shop.handle("price", async (body: { of?: string }) => ({ of: body.of ?? "nothing", cost: 12 }));
|
||||
const controller = await connectNats({ url: url!, module: "mesh-controller" });
|
||||
await controller.handle("seat:mesh-controller.tools", async () => ({
|
||||
seats: [
|
||||
{ seat: "mesh-controller", scope: "mesh", tools: [
|
||||
{ name: "status", description: "what is wrong", input: {} },
|
||||
{ name: "push", description: "tell a machine", input: { node: { type: "string" } } },
|
||||
] },
|
||||
// A seat held once per machine (design 33 §4, ADR 0170): its verb is asked of one.
|
||||
{ seat: "node-dns-resolver", scope: "node", tools: [{ name: "lookup", description: "one machine's", input: {} }] },
|
||||
],
|
||||
}));
|
||||
// Every role's tools, from the console on this machine (ADR 0571), which the spawned `mesh` asks through MESH_CONSOLE.
|
||||
const local = await aConsoleWithSeats([
|
||||
{ seat: "mesh-controller", scope: "mesh", tools: [
|
||||
{ name: "status", description: "what is wrong", input: {} },
|
||||
{ name: "push", description: "tell a machine", input: { node: { type: "string" } } },
|
||||
] },
|
||||
// A seat held once per machine (design 33 §4, ADR 0170): its verb is asked of one.
|
||||
{ seat: "node-dns-resolver", scope: "node", tools: [{ name: "lookup", description: "one machine's", input: {} }] },
|
||||
]);
|
||||
await controller.handle("seat:node-dns-resolver.lookup@anchor", async () => ({ machine: "anchor", answered: true }));
|
||||
await controller.handle("seat:mesh-controller.status", async () => ({ output: "all quiet", ok: true }));
|
||||
await controller.handle("seat:mesh-controller.push", async (body: { node?: string }) => ({ told: body.node ?? "nobody" }));
|
||||
@@ -46,6 +47,7 @@ async function aMeshAndACredential(t: { after: (fn: () => Promise<void> | void)
|
||||
await catalogue.close();
|
||||
await shop.close();
|
||||
await controller.close();
|
||||
await local.close();
|
||||
});
|
||||
|
||||
const { mkdtemp, writeFile } = await import("node:fs/promises");
|
||||
|
||||
@@ -172,9 +172,12 @@ test("the node's runtime serves five modules' bundles on one credential — two
|
||||
const catalogue = await connectNats({ url, module: "mesh-catalog" });
|
||||
await catalogue.handle("catalog_modules", async () => ({ modules: [{ module: "alpha" }, { module: "beta" }, { module: "gamma" }, { module: "delta" }, { module: "epsilon" }] }));
|
||||
try {
|
||||
const have = await toolsOn(asker);
|
||||
// No console on this port: the roles' tools are said missing with the reason, never asked of the controller.
|
||||
const have = await toolsOn(asker, "http://127.0.0.1:9");
|
||||
assert.deepEqual(have.tools.map((x) => `${x.module}.${x.name}`), ["alpha.one", "alpha.two", "beta.five", "beta.four", "beta.three", "delta.die", "delta.greet", "epsilon.seven"]);
|
||||
assert.deepEqual(have.notAnswering, ["gamma (its tools bundle failed to load: gamma's bundle exited (1): Error: gamma's bundle cannot find its client)", "mesh-controller (seat)"]);
|
||||
assert.equal(have.notAnswering[0], "gamma (its tools bundle failed to load: gamma's bundle exited (1): Error: gamma's bundle cannot find its client)");
|
||||
assert.match(have.notAnswering[1] ?? "", /^the roles' tools \(the console at http:\/\/127\.0\.0\.1:9 did not answer mesh\/seats: /);
|
||||
assert.equal(have.notAnswering.length, 2);
|
||||
} finally {
|
||||
await catalogue.close();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user