Compare commits
198
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1337bbbf2 | ||
|
|
ade07bfdba | ||
|
|
07ab8069a4 | ||
|
|
b90376ba59 | ||
|
|
95a12105cf | ||
|
|
2966c1e32c | ||
|
|
1a0424117a | ||
|
|
23f5babe33 | ||
|
|
27266f99a3 | ||
|
|
a9b02bc676 | ||
|
|
f016c67a72 | ||
|
|
6cc3398880 | ||
|
|
8cdb7c3032 | ||
|
|
3baeaece61 | ||
|
|
a5097dc69a | ||
|
|
9f25e5c9f6 | ||
|
|
bb6a0e3192 | ||
|
|
3331719407 | ||
|
|
5d6cee9d84 | ||
|
|
2cd5550dc7 | ||
|
|
d8ac69904d | ||
|
|
b0ded57294 | ||
|
|
77b05089c8 | ||
|
|
0211844e6a | ||
|
|
0e985f9bf1 | ||
|
|
860043fd7f | ||
|
|
589bfc0bc5 | ||
|
|
3b5a38e4a4 | ||
|
|
755f77ca45 | ||
|
|
9153a280f8 | ||
|
|
8ee6db7d9a | ||
|
|
5dffb920ff | ||
|
|
e0fadc43d9 | ||
|
|
d295fd4250 | ||
|
|
18260ce9be | ||
|
|
4165ad59b5 | ||
|
|
f183b86fa0 | ||
|
|
06e6721ed6 | ||
|
|
7412c5b6c9 | ||
|
|
33d136e702 | ||
|
|
6b3e15042f | ||
|
|
21d038d339 | ||
|
|
5d70eb3e35 | ||
|
|
d3b0330ed1 | ||
|
|
9921486fd5 | ||
|
|
d0dcae53a8 | ||
|
|
52e275ad2f | ||
|
|
954e1a53ee | ||
|
|
424a7eec82 | ||
|
|
09c5f809cb | ||
|
|
847055c758 | ||
|
|
aee834aeb6 | ||
|
|
494d0787e2 | ||
|
|
172b5af589 | ||
|
|
3f0c1a6cec | ||
|
|
4348b42844 | ||
|
|
82aefe1988 | ||
|
|
d535dd5322 | ||
|
|
336a2b1328 | ||
|
|
541de48e04 | ||
|
|
ab0f888d67 | ||
|
|
6a6ed725c0 | ||
|
|
560876c074 | ||
|
|
aef908b57b | ||
|
|
c761c22c2a | ||
|
|
d0128adf5b | ||
|
|
1eaa1abbe9 | ||
|
|
83c1994183 | ||
|
|
127047edd6 | ||
|
|
c6087129ce | ||
|
|
7b4428c510 | ||
|
|
6d9906b6c9 | ||
|
|
66de6774e0 | ||
|
|
c401bbff2d | ||
|
|
d4845f7c8d | ||
|
|
90c7871276 | ||
|
|
9a4140b49b | ||
|
|
aa209313c2 | ||
|
|
49cb06f6c5 | ||
|
|
26a58c7a79 | ||
|
|
7e83940fcc | ||
|
|
2f045e042f | ||
|
|
41911f7f15 | ||
|
|
0a5dbc7e9a | ||
|
|
dac4d423a8 | ||
|
|
01c98db0a4 | ||
|
|
b86a6ca228 | ||
|
|
14bff5dfd7 | ||
|
|
c2a0683115 | ||
|
|
1792b0d9ba | ||
|
|
f75780f1d0 | ||
|
|
0307df0850 | ||
|
|
730b4047f0 | ||
|
|
fce3dcb98d | ||
|
|
9730bd89c3 | ||
|
|
3a7c9645e0 | ||
|
|
93c1ad8c4a | ||
|
|
52a0cc4fcb | ||
|
|
379d19d907 | ||
|
|
8b789578c1 | ||
|
|
b3ebdd5edd | ||
|
|
51c79d8461 | ||
|
|
8405e32efc | ||
|
|
670a7884ff | ||
|
|
779ea67ea6 | ||
|
|
b149e9fcd6 | ||
|
|
168cf02829 | ||
|
|
70ff0f84af | ||
|
|
328550f920 | ||
|
|
29c24a2367 | ||
|
|
1adfcad88e | ||
|
|
dac8812968 | ||
|
|
6cba894f01 | ||
|
|
47cdfad754 | ||
|
|
85810ccc8c | ||
|
|
2ba7451229 | ||
|
|
6604d44372 | ||
|
|
7b21440962 | ||
|
|
0cea8d286e | ||
|
|
094a7d0d7c | ||
|
|
b52669577d | ||
|
|
7bd76275f9 | ||
|
|
6ba0f4dc1e | ||
|
|
bc05658772 | ||
|
|
4af59636c3 | ||
|
|
6e425a000f | ||
|
|
14b6588839 | ||
|
|
490aedfd36 | ||
|
|
f11ac6441c | ||
|
|
ffe229308c | ||
|
|
df4f492a72 | ||
|
|
66e8be0e31 | ||
|
|
7722668220 | ||
|
|
e8989f3cf5 | ||
|
|
915f372a85 | ||
|
|
486dad99a5 | ||
|
|
65f3b68076 | ||
|
|
a269a76c87 | ||
|
|
e6a33dd969 | ||
|
|
42e0987c64 | ||
|
|
ae6bdc9b06 | ||
|
|
e45389f75a | ||
|
|
b182943c24 | ||
|
|
26e7fcb4cb | ||
|
|
302645dd01 | ||
|
|
de4b63824e | ||
|
|
8e30ea9ff5 | ||
|
|
193ed0ac63 | ||
|
|
4537494150 | ||
|
|
7152148410 | ||
|
|
5d488a45f7 | ||
|
|
c46f9502ee | ||
|
|
2746fd31f0 | ||
|
|
82d7306ee0 | ||
|
|
2818f99b17 | ||
|
|
1436b02755 | ||
|
|
4e0559c31a | ||
|
|
6390d1d7fb | ||
|
|
020003ea3f | ||
|
|
809f07d085 | ||
|
|
fb0dcd0cc1 | ||
|
|
a3ace58362 | ||
|
|
feb13c1de6 | ||
|
|
e6676068c7 | ||
|
|
a01b5f5b78 | ||
|
|
ab8b13f512 | ||
|
|
ce8c37a7fc | ||
|
|
809e2b11ec | ||
|
|
670b486ac0 | ||
|
|
56f82588af | ||
|
|
e7b98f1fbc | ||
|
|
278a25b3e5 | ||
|
|
c65f1993ba | ||
|
|
6a91d144b3 | ||
|
|
71965ef958 | ||
|
|
dea98e509a | ||
|
|
80b02740ab | ||
|
|
621d033d53 | ||
|
|
38831c5c56 | ||
|
|
fdad2f3268 | ||
|
|
81972a4995 | ||
|
|
10e8191717 | ||
|
|
d703cebff4 | ||
|
|
46b56d53a6 | ||
|
|
d4a2802342 | ||
|
|
8acfa7a07d | ||
|
|
f0104b7846 | ||
|
|
cd26131c61 | ||
|
|
9e3ff6fa45 | ||
|
|
a4447f1251 | ||
|
|
3d55aeb1d8 | ||
|
|
9acc40145a | ||
|
|
fbeb373d1a | ||
|
|
2197c36fef | ||
|
|
19560ca6a7 | ||
|
|
c1517c39a0 | ||
|
|
6b380b67e0 | ||
|
|
2990b2d5a4 |
@@ -1,2 +1,3 @@
|
||||
node_modules/
|
||||
dist/
|
||||
.mesh-build/
|
||||
|
||||
-53
@@ -1,53 +0,0 @@
|
||||
# Three stages, two published images: the one modules are COMPILED in, and the one they RUN in.
|
||||
#
|
||||
# **They were the same image, and that was a mistake.** A module's recipe starts from this and
|
||||
# invokes the compiler out of it, so the compiler had to be here — and because the same image was
|
||||
# also what every module ran in, every running container on every machine carried a TypeScript
|
||||
# compiler it would never invoke. The answer is separate stages rather than one image bad at both
|
||||
# jobs. `build.artifacts` in module.json names the published stage each — `toolchain` and `runtime`.
|
||||
#
|
||||
# The `deps` stage is neither published nor named there: it is where the mesh's package registry is
|
||||
# reached, so it is where — and only where — the credential to reach it exists. The toolchain copies
|
||||
# resolved node_modules out of it, so the credential is in no image any machine ever holds
|
||||
# (novox/hq ADR 0076). This is the buildkit-secret's job done without buildkit, because a machine's
|
||||
# docker may carry no buildx.
|
||||
|
||||
# ---- deps: node_modules resolved from the mesh's registry, credential and all ----------------
|
||||
FROM node:22-bookworm-slim AS deps
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY package.json ./
|
||||
# The builder writes .npmrc into the build context; it authenticates to the mesh's package registry
|
||||
# for the @novox scope, which is where @novox/mesh-sdk resolves. This stage is not published, so the
|
||||
# credential travels no further than here. Development dependencies included: the compiler is one.
|
||||
COPY .npmrc ./.npmrc
|
||||
RUN npm install --no-audit --no-fund
|
||||
|
||||
# ---- toolchain: what a module is compiled in, WITHOUT the credential --------------------------
|
||||
FROM node:22-bookworm-slim AS toolchain
|
||||
WORKDIR /app
|
||||
COPY package.json ./
|
||||
# The resolved libraries, but not the .npmrc that resolved them.
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
# The toolkit arrives compiled. It used to arrive as sources, and this compiled it by hand — the
|
||||
# hook that builds it on install was running all along, and the result was then packed out of the
|
||||
# package, because with no explicit file list npm falls back to .gitignore and that ignores the
|
||||
# build output. Fixed where it belonged, in the toolkit.
|
||||
COPY tsconfig.json ./
|
||||
COPY src ./src
|
||||
RUN npm run build
|
||||
|
||||
# ---- what the running image needs, and nothing else -------------------------------------------
|
||||
# Its own stage so the toolchain image keeps its build tools while the runtime image does not.
|
||||
FROM toolchain AS lean
|
||||
RUN npm prune --omit=dev
|
||||
|
||||
# ---- runtime: what a module runs in -----------------------------------------------------------
|
||||
FROM node:22-bookworm-slim AS runtime
|
||||
WORKDIR /app
|
||||
COPY package.json ./
|
||||
COPY --from=lean /app/node_modules ./node_modules
|
||||
COPY --from=toolchain /app/dist ./dist
|
||||
ENTRYPOINT ["node", "dist/main.js"]
|
||||
@@ -1,32 +1,92 @@
|
||||
# mesh-tools
|
||||
|
||||
The Novox Mesh **tool runtime** — the per-node process that makes a module's tools actually serve.
|
||||
One module, `node-tools`; the toolchain image module that stood beside it is retired (novox/hq ADR 0524 §5):
|
||||
|
||||
A module ships its tools (built on [`@novox/mesh-sdk`](https://git.novox.be/novox/mesh-sdk)); this
|
||||
runtime is what loads them and puts them on the mesh. It:
|
||||
- **`node-tools`** (`node-tools/`) — the node's **tool runtime** as a module (ADR 0175, to-be 38
|
||||
WP3): one process per machine the host runs from this bundle, serving every assigned module's tools
|
||||
and every held seat's verbs on the bus, and answering MCP on the machine's loopback — the console
|
||||
(design 34). The code, its tests and the `mesh` client all live there.
|
||||
|
||||
1. connects the mesh broker (novox/hq ADR 0001) — a concrete AMQP implementation of the sdk's
|
||||
`Broker` contract;
|
||||
2. imports the assigned modules' compiled tool entrypoints, each of which registers its tools as it
|
||||
loads;
|
||||
3. serves them through the sdk's `serveTools` harness, answering `tools.invoke` over the broker.
|
||||
The runtime:
|
||||
|
||||
1. connects the mesh bus on the node's credential — a concrete implementation of the sdk's `Broker`
|
||||
contract;
|
||||
2. reads one membership per module it serves — what the mesh issued that module on this machine
|
||||
(ADR 0160): where its tools are answered, which seats it holds — and follows each live;
|
||||
3. imports each module's compiled tool entrypoints, each of which registers its tools as it loads,
|
||||
guarded: a bundle that throws is named, in the log and in what `tools` answers for its module,
|
||||
and the others serve;
|
||||
4. serves every module's tools on that module's subjects and every held seat's verbs on the seat's.
|
||||
|
||||
A bundle that is not plain JavaScript — a Go or Rust binary, a Python script, or a JavaScript file
|
||||
marked executable — is **launched** rather than imported (novox/hq ADR 0188): the runtime starts it
|
||||
as a child with its own environment and speaks MCP over stdio to it, `tools/list` once and
|
||||
`tools/call` per call. A tool it lists as `<seat>.<verb>` is the seat's implementation. A child that
|
||||
exits is named in the log and started again on its next call. So a tools bundle may be written in
|
||||
any language; the mesh's SDK for each is the stdio loop and nothing more (`node-tools/src/launch.ts`
|
||||
is the runtime's side of it).
|
||||
|
||||
Everything hard — dispatch, collection, duplicate-name safety — is the sdk's. This is the thin
|
||||
wrapper that binds the broker and loads the modules. Keeping the AMQP client here, out of the sdk,
|
||||
is deliberate: a broker-client change never rebuilds a module (ADR 0039).
|
||||
wrapper that binds the bus and loads the modules. Keeping the bus client here, out of the sdk, is
|
||||
deliberate: a bus-client change never rebuilds a module (ADR 0039). The runtime is module-agnostic:
|
||||
it knows bundles and subjects, nothing of what any module does. A tool that needs root escalates
|
||||
itself — root is the module's concern, not the runtime's.
|
||||
|
||||
## Running it
|
||||
|
||||
```
|
||||
MESH_BROKER_URL amqp://… the mesh broker
|
||||
MESH_TOOL_MODULES /a/tools/index.js,… the assigned modules' compiled tool entrypoints
|
||||
MESH_BROKER_FILE the node's sealed credential, as the mesh delivered it
|
||||
MESH_TOOL_MODULES alpha=/…/alpha/tools/index.js,beta=/…/beta/dist/index.js,…
|
||||
the modules to serve and their compiled entrypoints; several entries may
|
||||
name one module. A bare path is an entrypoint of the credential's own module
|
||||
— the one-module form a per-module container still sets.
|
||||
MESH_OPERATOR_ACCOUNT whose machine this is, and MESH_OPERATOR_HOME where their home is; set by
|
||||
the mesh when the node has an account, read by tools from their environment
|
||||
MESH_BROKER_URL a plain URL instead of the credential, for the bootstrap case
|
||||
```
|
||||
|
||||
`node dist/main.js`, or the container (`Dockerfile`). On a node the host resolves both variables
|
||||
and starts it like any other supervised workload.
|
||||
`node dist/main.js`. On a node the controller composes the variables and the host supervises the
|
||||
process like any other host-side workload (novox/hq to-be 38). As `node-tools` the same process is
|
||||
the console: MCP on `127.0.0.1:4270` (or `MESH_CONSOLE_LISTEN`).
|
||||
|
||||
## `mesh` — the tools for whoever is on a machine
|
||||
|
||||
The same package carries the client (novox/hq design 25 §7, design 34): `mesh tools`, `mesh call
|
||||
<module>.<tool> [json]`, `mesh mcp` (an MCP server over stdio for a program a person starts) and
|
||||
`mesh serve` (the **console**: MCP over HTTP on a machine's loopback, started by the mesh as the
|
||||
`mesh-console` module on the credential in `MESH_BROKER_FILE` — novox/hq ADR 0152). `mesh serve`
|
||||
refuses to bind anything but loopback. With `--console <url>`, `tools` and `call` go through a console
|
||||
already on the machine and need no credential.
|
||||
|
||||
The console announces six tools and reaches everything else by address (novox/hq ADR 0195):
|
||||
`mesh_overview` (the mesh's seats and machines), `mesh_machine` (one machine's seats and modules),
|
||||
`mesh_search` (a tool by words), `mesh_describe` (one tool's arguments), `mesh_call` (call one by
|
||||
address) and `mesh_runtimes` (which runtimes answered discovery: per runtime its machine, how long its
|
||||
answer took, its size in bytes, how many modules and tools it announced, whether it was shortened to
|
||||
fit the bus, when it was last heard — and who was expected and not heard).
|
||||
|
||||
`mesh_search` also finds a tool by the shell command it replaces (novox/hq ADR 0245): each seat verb and
|
||||
module tool says what it replaces in the controller's records (`replaces`, read from the controller's
|
||||
`tools` and `modules` answers), so `journalctl -u x`, `systemctl status x` or `docker ps` puts the verb
|
||||
for it first, and says the command it matched as `instead_of`. A word also matches through a short table
|
||||
of the words agents use for the mesh's (`logs` finds the journal). Best first: a replaced command, then
|
||||
words in a name or a replaced command, then words in a description; seats before modules on a tie.
|
||||
|
||||
Discovery asks the bus (ADR 0197): every runtime answers the NATS services protocol's `$SRV.PING` and
|
||||
`$SRV.INFO` with what it serves at that moment, and the console reads where the controller's records
|
||||
place each module. The console waits at least 750 ms, and up to 5 s for every runtime that answered
|
||||
PING to send what it serves — a large runtime's answer crossing the bus to a broker on another machine
|
||||
can arrive well after the rest. A runtime that said it is there and did not say what it serves, or
|
||||
that answered earlier and not now, is named: the console never calls a module missing, or on another
|
||||
machine, while a runtime that might serve it was not heard. A runtime whose answer outgrows the bus
|
||||
announces first-line descriptions, then none, and says so. A module may not name a tool of its own
|
||||
`tools`; the runtime refuses it at load.
|
||||
|
||||
## Verified
|
||||
|
||||
`npm test` stands up LavinMQ (the mesh's broker) and proves the whole path over real AMQP: the
|
||||
runtime serves a registered tool, a separate connection invokes it by name and gets the result, and
|
||||
an unknown tool is refused over the wire.
|
||||
`npm test` runs against a real NATS server with JetStream (`MESH_TEST_NATS`, see any test's header
|
||||
for the one-line `docker run`) and proves the whole path over the wire: the runtime serves a
|
||||
registered tool, a separate connection invokes it by name and gets the result, an unknown tool is
|
||||
refused, a runtime serves exactly the subjects it is issued and re-serves on a new membership, and
|
||||
the node's runtime serves three modules' bundles on one credential — one of them broken, named and
|
||||
not fatal — with every seat verb answering where the membership put it.
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
import amqp from "amqplib";
|
||||
|
||||
const PORT = process.argv[2];
|
||||
const MPORT = process.argv[3];
|
||||
const B = `http://127.0.0.1:${MPORT}`;
|
||||
const AUTH = "Basic " + Buffer.from("guest:guest").toString("base64");
|
||||
|
||||
async function api(method, path, body) {
|
||||
const r = await fetch(B + path, {
|
||||
method,
|
||||
headers: { "content-type": "application/json", authorization: AUTH },
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
if (r.status >= 300 && r.status !== 404) throw new Error(`${method} ${path} -> ${r.status}`);
|
||||
}
|
||||
|
||||
await api("PUT", "/api/exchanges/%2f/mesh.events.dead", { type: "topic", durable: true });
|
||||
await api("PUT", "/api/users/al", { password: "s", tags: "" });
|
||||
|
||||
const Q = "anchor.al.events";
|
||||
const D = "mesh.events.dead";
|
||||
const q = Q.replace(/\./g, "\\.");
|
||||
const d = D.replace(/\./g, "\\.");
|
||||
|
||||
// configure, write, read patterns per grant on the dead exchange
|
||||
const combos = {
|
||||
"none": { configure: `^${q}$`, write: `^${q}$`, read: `^${q}$` },
|
||||
"read-dead": { configure: `^${q}$`, write: `^${q}$`, read: `^(${q}|${d})$` },
|
||||
"write-dead": { configure: `^${q}$`, write: `^(${q}|${d})$`, read: `^${q}$` },
|
||||
"configure-dead": { configure: `^(${q}|${d})$`, write: `^${q}$`, read: `^${q}$` },
|
||||
"read+write-dead": { configure: `^${q}$`, write: `^(${q}|${d})$`, read: `^(${q}|${d})$` },
|
||||
};
|
||||
|
||||
let i = 0;
|
||||
for (const [label, perms] of Object.entries(combos)) {
|
||||
await api("PUT", "/api/permissions/%2f/al", perms);
|
||||
const queue = `${Q}.${i++}`; // fresh each time
|
||||
try {
|
||||
const c = await amqp.connect(`amqp://al:s@127.0.0.1:${PORT}/`);
|
||||
const ch = await c.createChannel();
|
||||
ch.on("error", () => {});
|
||||
await ch.assertQueue(queue, { durable: true, deadLetterExchange: D });
|
||||
console.log(`${label}: declare-with-DLX OK`);
|
||||
await c.close();
|
||||
} catch (e) {
|
||||
console.log(`${label}: FAIL - ${String(e.message).slice(0, 70)}`);
|
||||
}
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"module": "mesh-tools",
|
||||
"version": "1",
|
||||
"slug": "tools",
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "build",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile",
|
||||
"target": "toolchain"
|
||||
},
|
||||
{
|
||||
"name": "runtime",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile",
|
||||
"target": "runtime"
|
||||
}
|
||||
]
|
||||
},
|
||||
"resources": []
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
# node-tools
|
||||
|
||||
The node's tool runtime as a module (novox/hq ADR 0175, to-be 38 WP3). Assigned to a machine, it is
|
||||
one process the host runs from this bundle, as the operator's account: it serves every assigned
|
||||
module's tools and every held seat's verbs on the bus, and answers MCP on the machine's loopback —
|
||||
the console (design 34). The controller composes the process (which bundles to load, where the
|
||||
credential is, whose machine it is); this manifest says only what the machine must have for it: the
|
||||
interpreter, a place for the credential, the loopback port, and leave to call every tool.
|
||||
|
||||
The code is the `mesh-tools` package in this directory; the module at the repository root,
|
||||
`mesh-tools`, builds the images TypeScript bundles are compiled in. See the repository README.
|
||||
@@ -0,0 +1,252 @@
|
||||
// node-tools — the node's tool runtime, in Go (novox/hq ADR 0175, ADR 0193; to-be 38 WP4d).
|
||||
//
|
||||
// One process per machine: it connects to the bus on the node's credential, launches every assigned
|
||||
// module's tools bundle and serves its tools and its seats' verbs; as the node-tools module — or
|
||||
// wherever MESH_CONSOLE_LISTEN says — it is also the console, MCP over HTTP on loopback.
|
||||
//
|
||||
// MESH_BROKER_FILE the credential the mesh sealed to this machine for the runtime
|
||||
// MESH_TOOL_MODULES <module>=<entrypoint>,… the bundles to launch
|
||||
// MESH_TOOL_ENV {"<module>": {"<word>": "<value>"}} what each is given (ADR 0192)
|
||||
// MESH_OPERATOR_ACCOUNT whose machine this is, and MESH_OPERATOR_HOME their home
|
||||
// MESH_CONSOLE_LISTEN where the console listens, overriding 127.0.0.1:4270
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/alive"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/console"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/provide"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
// runtimeModule is the module that is the node's tool runtime; on its credential, serving is also
|
||||
// the console.
|
||||
const runtimeModule = "node-tools"
|
||||
|
||||
// consoleListen is where the console listens when nothing says otherwise.
|
||||
const consoleListen = "127.0.0.1:4270"
|
||||
|
||||
func main() {
|
||||
log.SetFlags(0)
|
||||
if len(os.Args) > 1 && os.Args[1] != "serve" {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %q is not a mode; this runtime serves (novox/hq ADR 0193)\n", os.Args[1])
|
||||
os.Exit(2)
|
||||
}
|
||||
until := time.Now().Add(startWait)
|
||||
cred := credential(until)
|
||||
conn := connectPatiently(cred, until)
|
||||
|
||||
served, err := runtime.ServedModulesFrom(os.Getenv(runtime.ToolModules), cred.Module)
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
}
|
||||
envs, err := runtime.TakeToolEnvs()
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
}
|
||||
stop, err := runtime.Run(conn, served, envs, log.Printf)
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
}
|
||||
// And says it is there (novox/hq to-be 45 S11): the machine's runtime, on its machine's name. A
|
||||
// runtime on another credential — a module's own, in a test — is not the machine's.
|
||||
stopSaying := func() {}
|
||||
if cred.Module == runtimeModule && cred.Node != "" {
|
||||
stopSaying = alive.Keep(conn, cred.Node, alive.Every, log.Printf)
|
||||
}
|
||||
|
||||
// As the machine's runtime it is also the provider of `mcp-endpoint`: it answers its consumers through the
|
||||
// SDK's helper wherever it runs, whether or not the module is handed (novox/hq ADR 0349 clause 12, issue 791).
|
||||
stopProviding := provide.Start(conn, cred.Module, cred.Node, time.Minute, log.Printf)
|
||||
|
||||
listen := os.Getenv("MESH_CONSOLE_LISTEN")
|
||||
if listen == "" && cred.Module == runtimeModule {
|
||||
listen = consoleListen
|
||||
}
|
||||
var up *console.Listening
|
||||
if listen != "" {
|
||||
node := cred.Node
|
||||
if node == "" {
|
||||
node = "?"
|
||||
}
|
||||
who := node + "." + cred.Module
|
||||
up, err = console.Serve(console.NewSurface(conn, who), listen)
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
}
|
||||
log.Printf("mesh console listening on http://%s/mcp as %s", up.Address, who)
|
||||
}
|
||||
|
||||
signals := make(chan os.Signal, 1)
|
||||
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
|
||||
<-signals
|
||||
// First of all: the service manager signalled every bundle with this process, and none is started again
|
||||
// (issue 1126).
|
||||
launch.ShutDown()
|
||||
stopSaying()
|
||||
stopProviding()
|
||||
stop()
|
||||
if up != nil {
|
||||
_ = up.Close()
|
||||
}
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
// startWait is how long a runtime waits, from its start, for its broker credential to be on the machine and for the bus
|
||||
// to accept the account it names (issue 1496): the node-engine starts a module's runtime before the provisioner's client
|
||||
// fills the credential, and the bus carries a new account only after it reloads its users. Past it the runtime gives up,
|
||||
// loudly; inside it the wait is said once.
|
||||
var startWait = 5 * time.Minute
|
||||
|
||||
// say is a line in the runtime's journal.
|
||||
func say(format string, args ...any) { fmt.Fprintf(os.Stderr, "node-tools: "+format+"\n", args...) }
|
||||
|
||||
// credential reads the credential the mesh delivered, waiting for it within the start's bound (issue 1496); one that can
|
||||
// never be a credential (no url) is a fault of configuration, said and final.
|
||||
func credential(until time.Time) bus.Credential {
|
||||
file := os.Getenv("MESH_BROKER_FILE")
|
||||
if file == "" {
|
||||
if url := os.Getenv("MESH_BROKER_URL"); url != "" {
|
||||
return bus.Credential{URL: url, Module: runtimeModule}
|
||||
}
|
||||
say("set MESH_BROKER_FILE (a sealed credential) or MESH_BROKER_URL — there is no broker to reach")
|
||||
os.Exit(1)
|
||||
}
|
||||
cred, err := awaitCredential(file, until, say)
|
||||
if err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
// errNotCredential is a file that is there and readable and still no broker credential: waiting will not fix it.
|
||||
var errNotCredential = errors.New("it carries no url — it is not a broker credential")
|
||||
|
||||
// readCredential reads the credential at file. A file not there yet, empty, or not yet whole JSON is one the node-engine
|
||||
// has still to write: the error says so and waiting may fix it. errNotCredential is final.
|
||||
func readCredential(file string) (bus.Credential, error) {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return bus.Credential{}, err
|
||||
}
|
||||
if len(bytes.TrimSpace(raw)) == 0 {
|
||||
return bus.Credential{}, errors.New("the file is empty")
|
||||
}
|
||||
var cred bus.Credential
|
||||
if err := json.Unmarshal(raw, &cred); err != nil {
|
||||
return bus.Credential{}, err
|
||||
}
|
||||
if cred.URL == "" {
|
||||
return bus.Credential{}, errNotCredential
|
||||
}
|
||||
return cred, nil
|
||||
}
|
||||
|
||||
// awaitCredential is the credential at file, read at once and, while it is not there or not whole, again each time its
|
||||
// directory changes (runtime.WatchFile, inotify), until the time until. The wait is said once; at until it ends with an
|
||||
// error naming the file, how long it waited and what the last read found. A file that is not a credential ends it at once.
|
||||
func awaitCredential(file string, until time.Time, say func(string, ...any)) (bus.Credential, error) {
|
||||
cred, err := readCredential(file)
|
||||
if err == nil {
|
||||
return cred, nil
|
||||
}
|
||||
if errors.Is(err, errNotCredential) {
|
||||
return bus.Credential{}, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
began := time.Now()
|
||||
say("waits for its broker credential at %s (%v): the node-engine writes it once the provisioner's client fills it; "+
|
||||
"it gives up at %s", file, err, until.Format("15:04:05"))
|
||||
changed := make(chan struct{}, 1)
|
||||
stop, werr := runtime.WatchFile(file, func() {
|
||||
select {
|
||||
case changed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}, func(format string, args ...any) { say(format, args...) },
|
||||
fmt.Sprintf("the credential is read again every %s instead", rereadEvery))
|
||||
if werr != nil {
|
||||
say("cannot watch %s (%v): it is read again every %s instead", file, werr, rereadEvery)
|
||||
} else {
|
||||
defer stop()
|
||||
}
|
||||
for {
|
||||
left := time.Until(until)
|
||||
if left <= 0 {
|
||||
return bus.Credential{}, fmt.Errorf("cannot read the broker credential at %s: waited %s for it, and the last read "+
|
||||
"found: %v", file, time.Since(began).Round(time.Second), err)
|
||||
}
|
||||
// The watch wakes the read; the clock is only the backstop for a watch that could not be made or missed a change.
|
||||
select {
|
||||
case <-changed:
|
||||
case <-time.After(min(left, rereadEvery)):
|
||||
}
|
||||
if cred, err = readCredential(file); err == nil {
|
||||
say("read its broker credential at %s after %s", file, time.Since(began).Round(time.Second))
|
||||
return cred, nil
|
||||
}
|
||||
if errors.Is(err, errNotCredential) {
|
||||
return bus.Credential{}, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rereadEvery is the backstop of the wait for the credential, beside the watch of its directory.
|
||||
var rereadEvery = 30 * time.Second
|
||||
|
||||
// connectPatiently retries while the bus is merely not reachable yet — the normal case at startup — and, until the start's
|
||||
// bound, while the bus refuses the account: the bus carries a new account only after it reloads its users, which may come
|
||||
// after the credential is on the machine (issue 1496, issue 1423). What waiting cannot fix (a pin that does not match, an
|
||||
// address that is no URL) gives up at once (issue 058), and so does a refusal past the bound.
|
||||
func connectPatiently(cred bus.Credential, until time.Time) *bus.Conn {
|
||||
conn, err := connectWithin(cred, until, bus.Connect, time.Sleep, say)
|
||||
if err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
// connectWithin is connectPatiently with its dial, its sleep and its words given, as a test gives them.
|
||||
func connectWithin(cred bus.Credential, until time.Time, dial func(bus.Credential) (*bus.Conn, error), sleep func(time.Duration),
|
||||
say func(string, ...any)) (*bus.Conn, error) {
|
||||
refusedSaid := false
|
||||
for delay := 2 * time.Second; ; delay = min(delay*2, 30*time.Second) {
|
||||
conn, err := dial(cred)
|
||||
if err == nil {
|
||||
if refusedSaid {
|
||||
say("the bus accepts the account now")
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
wait := delay + time.Duration(rand.IntN(1000))*time.Millisecond
|
||||
switch fatal := bus.Fatal(err); {
|
||||
case bus.Refused(err) && time.Now().Add(wait).Before(until):
|
||||
if !refusedSaid {
|
||||
say("the bus refuses this account (%v): it may not carry it yet, since it loads a new account by a reload; "+
|
||||
"retrying until %s", err, until.Format("15:04:05"))
|
||||
refusedSaid = true
|
||||
}
|
||||
case bus.Refused(err) && refusedSaid:
|
||||
return nil, fmt.Errorf("the bus still refuses this account (%v) at the end of the start's bound, %s; giving up", err,
|
||||
until.Format("15:04:05"))
|
||||
case fatal != "":
|
||||
return nil, fmt.Errorf("%s — waiting will not fix this; giving up", fatal)
|
||||
default:
|
||||
say("the broker is not reachable yet (%v); retrying in %ds", err, int(wait.Round(time.Second)/time.Second))
|
||||
}
|
||||
sleep(wait)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
// A runtime started before its credential is on the machine waits for it, woken by the watch of its directory, and gives
|
||||
// up loudly at its bound; a file that is no credential ends the wait at once (issue 1496: on ace mesh-build-node's and
|
||||
// mesh-build-image's runtimes exited twice on "cannot read the broker credential ... no such file").
|
||||
func TestAwaitCredential(t *testing.T) {
|
||||
const good = `{"url":"tls://bus.example:4222","module":"m"}`
|
||||
rereadEvery = time.Hour // only the watch can wake the read in time
|
||||
settle := runtime.Settle
|
||||
runtime.Settle = 10 * time.Millisecond // a unit test stays well under a second
|
||||
t.Cleanup(func() { rereadEvery, runtime.Settle = 30*time.Second, settle })
|
||||
place := func(path, body string) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
before string // written before the wait, "" none
|
||||
later string // written 20ms into the wait, "" none
|
||||
deeper bool // the file's directory is not there at the start
|
||||
bound time.Duration
|
||||
want string // "" a credential; else a word of the error
|
||||
said int
|
||||
}{
|
||||
{name: "there at the start", before: good, bound: time.Second, said: 0},
|
||||
{name: "written during the wait", later: good, bound: 5 * time.Second, said: 2},
|
||||
{name: "empty, then filled", before: " ", later: good, bound: 5 * time.Second, said: 2},
|
||||
{name: "its directory made during the wait", later: good, deeper: true, bound: 5 * time.Second, said: 2},
|
||||
{name: "never written", bound: 100 * time.Millisecond, want: "waited", said: 1},
|
||||
{name: "no url", before: `{"module":"m"}`, bound: 5 * time.Second, want: "not a broker credential", said: 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
file := filepath.Join(dir, "broker")
|
||||
if c.deeper {
|
||||
file = filepath.Join(dir, "var", "lib", "m", "broker")
|
||||
}
|
||||
if c.before != "" {
|
||||
place(file, c.before)
|
||||
}
|
||||
if c.later != "" {
|
||||
go func() { time.Sleep(20 * time.Millisecond); place(file, c.later) }()
|
||||
}
|
||||
var said []string
|
||||
began := time.Now()
|
||||
cred, err := awaitCredential(file, time.Now().Add(c.bound), func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
switch {
|
||||
case c.want == "" && (err != nil || cred.URL == ""):
|
||||
t.Fatalf("no credential: %v", err)
|
||||
case c.want != "" && (err == nil || !strings.Contains(err.Error(), c.want)):
|
||||
t.Fatalf("error %v, want one saying %q", err, c.want)
|
||||
case c.want == "" && c.later != "" && time.Since(began) > 3*time.Second:
|
||||
t.Fatalf("read after %s: the watch did not wake it", time.Since(began))
|
||||
}
|
||||
if len(said) != c.said {
|
||||
t.Fatalf("said %d lines, want %d: %q", len(said), c.said, said)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A bus that refuses the account at a start is asked again until the start's bound, said once; a refusal past it, a pin
|
||||
// that does not match and an address that is no URL give up (issue 1496, issue 058).
|
||||
func TestConnectWithin(t *testing.T) {
|
||||
refused := errors.New("nats: Authorization Violation")
|
||||
unreachable := errors.New("nats: no servers available for connection")
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
errs []error // what each dial answers, then a connection
|
||||
bound time.Duration
|
||||
fails string
|
||||
saidAt int // lines said
|
||||
}{
|
||||
{name: "refused twice, then accepted", errs: []error{refused, refused}, bound: time.Hour, saidAt: 2},
|
||||
{name: "unreachable, then accepted", errs: []error{unreachable}, bound: time.Hour, saidAt: 1},
|
||||
{name: "refused past the bound", errs: []error{refused, refused}, bound: 0, fails: "refused this account"},
|
||||
{name: "refused, then the bound ends", errs: []error{refused, refused, refused, refused, refused, refused}, bound: 5 * time.Second,
|
||||
fails: "still refuses"},
|
||||
{name: "pin", errs: []error{fmt.Errorf("tls: %w", bus.ErrPin)}, bound: time.Hour, fails: "pin"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
dials := 0
|
||||
dial := func(bus.Credential) (*bus.Conn, error) {
|
||||
dials++
|
||||
if dials <= len(c.errs) {
|
||||
return nil, c.errs[dials-1]
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
until := time.Now().Add(c.bound)
|
||||
var said []string
|
||||
_, err := connectWithin(bus.Credential{URL: "tls://bus.example:4222"}, until, dial, func(time.Duration) {},
|
||||
func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
switch {
|
||||
case c.fails == "" && err != nil:
|
||||
t.Fatalf("gave up: %v", err)
|
||||
case c.fails != "" && (err == nil || !strings.Contains(err.Error(), c.fails)):
|
||||
t.Fatalf("error %v, want one saying %q", err, c.fails)
|
||||
case c.fails == "" && len(said) != c.saidAt:
|
||||
t.Fatalf("said %d lines, want %d: %q", len(said), c.saidAt, said)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
module github.com/novox/mesh-tools/node-tools
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/sys v0.48.0
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030100935-c2ec8b6b2fde
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030095131-16bc0dcf3e9d h1:jFOz6wUfZvIayd4DctwlaFKKZHQNME6NLs2OS1rNv30=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030095131-16bc0dcf3e9d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030095816-530d8d3007dc h1:Wdjl2gomooHswJ5qpkwZPq/ureZgXVw6FMCsgzPJhPo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030095816-530d8d3007dc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030100935-c2ec8b6b2fde h1:eKXSllHCS31GgMazfvZ7BeiLoXIUibnUmzxWVkj+Vcw=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030100935-c2ec8b6b2fde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,80 @@
|
||||
// Package alive is the node tools saying they are there (novox/hq to-be 45 §3, S11).
|
||||
//
|
||||
// **A machine heard and its runtime gone is a machine nobody can ask anything.** The node-engine says
|
||||
// every minute that the machine is there; nothing said whether the runtime serving every module's
|
||||
// tools and every held seat's verbs was. Now it says so too, on its own subject, every interval, with
|
||||
// the interval — the controller's watchdog raises `tools-silent` after three missed — and its build.
|
||||
// Core NATS, kept by nobody, as the host's heartbeat: a lost one is the next one.
|
||||
package alive
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Every is how often the runtime says it is there.
|
||||
const Every = 60 * time.Second
|
||||
|
||||
// Subject is where one machine's node tools say it.
|
||||
func Subject(node string) string { return "mesh.control." + node + ".tools-alive" }
|
||||
|
||||
// Beat is what is said: the machine, the interval, and the build.
|
||||
type Beat struct {
|
||||
Node string `json:"node"`
|
||||
IntervalSeconds int `json:"interval_seconds"`
|
||||
Version string `json:"version,omitempty"`
|
||||
}
|
||||
|
||||
// Sayer publishes one message, kept by nobody.
|
||||
type Sayer interface {
|
||||
Say(subject string, body []byte) error
|
||||
}
|
||||
|
||||
// Version is this build's commit, as the toolchain stamped it; empty when it did not.
|
||||
func Version() string {
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
for _, s := range info.Settings {
|
||||
if s.Key == "vcs.revision" {
|
||||
return s.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Keep says the runtime is there now and every interval until the returned function is called. A
|
||||
// heartbeat that cannot be said is logged when that starts and when it stops — not every minute.
|
||||
func Keep(bus Sayer, node string, every time.Duration, logf func(string, ...any)) func() {
|
||||
body, _ := json.Marshal(Beat{Node: node, IntervalSeconds: int(every / time.Second), Version: Version()})
|
||||
done := make(chan struct{})
|
||||
var once sync.Once
|
||||
go func() {
|
||||
failing := ""
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
why := ""
|
||||
if err := bus.Say(Subject(node), body); err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
logf("node-tools: could not say this runtime is there — the mesh will call it silent: %s", why)
|
||||
} else {
|
||||
logf("node-tools: says again that this runtime is there")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { once.Do(func() { close(done) }) }
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package alive
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type said struct {
|
||||
mu sync.Mutex
|
||||
subs []string
|
||||
body [][]byte
|
||||
fail error
|
||||
}
|
||||
|
||||
func (s *said) Say(subject string, body []byte) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.fail != nil {
|
||||
return s.fail
|
||||
}
|
||||
s.subs, s.body = append(s.subs, subject), append(s.body, body)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *said) count() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.subs)
|
||||
}
|
||||
|
||||
// **The runtime says it is there at once and every interval, as its machine, with the interval**: the
|
||||
// controller's bound is three of them (to-be 45 S11).
|
||||
func TestTheRuntimeSaysItIsThereEveryInterval(t *testing.T) {
|
||||
s := &said{}
|
||||
stop := Keep(s, "anchor", 20*time.Millisecond, t.Logf)
|
||||
time.Sleep(70 * time.Millisecond)
|
||||
stop()
|
||||
stop()
|
||||
if n := s.count(); n < 3 {
|
||||
t.Fatalf("said %d times in three intervals", n)
|
||||
}
|
||||
var beat Beat
|
||||
if err := json.Unmarshal(s.body[0], &beat); err != nil || beat.Node != "anchor" || beat.IntervalSeconds != 0 ||
|
||||
s.subs[0] != "mesh.control.anchor.tools-alive" {
|
||||
t.Fatalf("%s on %s: %v", s.body[0], s.subs[0], err)
|
||||
}
|
||||
n := s.count()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if s.count() != n {
|
||||
t.Fatal("said after it was stopped")
|
||||
}
|
||||
}
|
||||
|
||||
// **A heartbeat the bus will not take is said once, and again when it is taken**: not a line a minute.
|
||||
func TestAHeartbeatThatCannotBeSaidIsLoggedOnce(t *testing.T) {
|
||||
s := &said{fail: errors.New("permissions violation")}
|
||||
var mu sync.Mutex
|
||||
var lines []string
|
||||
stop := Keep(s, "anchor", 10*time.Millisecond, func(f string, a ...any) {
|
||||
mu.Lock()
|
||||
lines = append(lines, f)
|
||||
mu.Unlock()
|
||||
})
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
stop()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(lines) != 1 {
|
||||
t.Fatalf("logged %d lines for one failure", len(lines))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,472 @@
|
||||
// Package announce answers the NATS services protocol's discovery for what a runtime serves, and
|
||||
// gathers the answers (novox/hq ADR 0197).
|
||||
//
|
||||
// A runtime does not re-serve its tools through a services library: serving is unchanged. It answers
|
||||
// `$SRV.PING`, `$SRV.INFO` and `$SRV.STATS` — and the same followed by its service name, and by its
|
||||
// name and id — in the format NATS's own tools read, with what it is serving at the moment it is
|
||||
// asked. One service per runtime process: the bus admits one reply per request from each responder,
|
||||
// so a runtime serving many modules and seats answers once, one endpoint per tool per subject, and
|
||||
// says in each endpoint's metadata which module, seat, scope and machine it is.
|
||||
package announce
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// Version is the announced service version (semver, as the protocol requires).
|
||||
const Version = "0.1.0"
|
||||
|
||||
// Window is how long the console gathers discovery answers at the least: every instance answers one
|
||||
// request, and how many will is what is being found out.
|
||||
var Window = 750 * time.Millisecond
|
||||
|
||||
// Patience is how long the console waits, at the most, for the full answer of a service that has
|
||||
// said it is there. **A fixed window lost the largest runtime** (2026-10-05): the laptop's answer —
|
||||
// 48 modules, 341 endpoints, 164 kB — went up to the broker on another machine and back, arrived
|
||||
// last every time (a median of 365 ms, one in 25 after 813 ms on a quiet link, later still while the
|
||||
// runtime re-served), and when it missed the 750 ms window the console said its modules ran nowhere.
|
||||
// A PING answer is a hundred bytes and arrives at once, so who is there is known early; what each
|
||||
// serves is waited for until it arrives or this passes, and a service that never sends it is named.
|
||||
var Patience = 5 * time.Second
|
||||
|
||||
// Digest is the PING answer's metadata key for a digest of what the instance serves now: a console
|
||||
// that holds the answer of that digest does not ask for it again (novox/hq issue 554). Every round
|
||||
// pulled every runtime's whole announcement — some 880 kB, every five seconds while a session works —
|
||||
// to the laptop, and the hundred-byte PING answers queued behind it.
|
||||
const Digest = "digest"
|
||||
|
||||
// Shortened is the service metadata key a runtime sets when its announcement was cut to fit the bus.
|
||||
const Shortened = "shortened"
|
||||
|
||||
// Kinds of endpoint.
|
||||
const (
|
||||
KindTool = "tool" // a module's own tool
|
||||
KindSeat = "seat" // a seat's verb, served by the module holding it
|
||||
)
|
||||
|
||||
// Endpoint is one tool served on one subject, as it is announced.
|
||||
type Endpoint struct {
|
||||
Kind string
|
||||
Module string // the module whose code answers
|
||||
Tool string // the tool's or the verb's name
|
||||
Seat string // for a seat's verb
|
||||
Scope string // "mesh" or "node", for a seat's verb
|
||||
Node string
|
||||
Description string
|
||||
Schema json.RawMessage
|
||||
Interchangeable bool
|
||||
Subject string
|
||||
Queue string
|
||||
}
|
||||
|
||||
// Name is the endpoint's name as the protocol allows it — letters, digits, `-` and `_` — the
|
||||
// prefix and the tool joined by `__`; the metadata, not the name, is what identifies it.
|
||||
func (e Endpoint) Name() string {
|
||||
prefix := e.Module
|
||||
if e.Kind == KindSeat {
|
||||
prefix = e.Seat
|
||||
}
|
||||
return clean(prefix) + "__" + clean(e.Tool)
|
||||
}
|
||||
|
||||
func clean(s string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
if r == '-' || r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
|
||||
b.WriteRune(r)
|
||||
} else {
|
||||
b.WriteRune('_')
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (e Endpoint) info() micro.EndpointInfo {
|
||||
md := map[string]string{
|
||||
"kind": e.Kind, "module": e.Module, "tool": e.Tool, "node": e.Node,
|
||||
"description": e.Description, "interchangeable": boolWord(e.Interchangeable),
|
||||
}
|
||||
// **A seat's verb carries its schema; a module's tool does not.** Every tool's schema, once per
|
||||
// subject it answers on, made a runtime's one answer outgrow the bus's largest message (1 MiB) once
|
||||
// machines served a couple of hundred tools — and the answer that cannot be sent is silence: the
|
||||
// machine vanished from discovery (2026-10-04). A module's tool's schema is asked of the runtime that
|
||||
// serves it when a person describes it (its `tools` verb); a seat's verbs are few, and a seat has no
|
||||
// `tools` verb of its own to ask.
|
||||
if e.Kind == KindSeat {
|
||||
schema := strings.TrimSpace(string(e.Schema))
|
||||
if schema == "" || schema == "null" {
|
||||
schema = "{}"
|
||||
}
|
||||
md["schema"] = schema
|
||||
}
|
||||
if e.Kind == KindSeat {
|
||||
md["seat"] = e.Seat
|
||||
md["scope"] = e.Scope
|
||||
}
|
||||
return micro.EndpointInfo{Name: e.Name(), Subject: e.Subject, QueueGroup: e.Queue, Metadata: md}
|
||||
}
|
||||
|
||||
func boolWord(b bool) string {
|
||||
if b {
|
||||
return "true"
|
||||
}
|
||||
return "false"
|
||||
}
|
||||
|
||||
// Service is who answers: the runtime's name, its instance, and a word about it.
|
||||
type Service struct {
|
||||
Name string
|
||||
ID string
|
||||
Description string
|
||||
Metadata map[string]string
|
||||
}
|
||||
|
||||
func (s Service) identity() micro.ServiceIdentity {
|
||||
md := s.Metadata
|
||||
if md == nil {
|
||||
md = map[string]string{}
|
||||
}
|
||||
return micro.ServiceIdentity{Name: s.Name, ID: s.ID, Version: Version, Metadata: md}
|
||||
}
|
||||
|
||||
// Info is the info_response for these endpoints.
|
||||
func Info(s Service, endpoints []Endpoint) micro.Info {
|
||||
out := micro.Info{ServiceIdentity: s.identity(), Type: micro.InfoResponseType,
|
||||
Description: s.Description, Endpoints: []micro.EndpointInfo{}}
|
||||
for _, e := range endpoints {
|
||||
out.Endpoints = append(out.Endpoints, e.info())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Serve answers discovery for one service until stopped, asking `current` for its endpoints each
|
||||
// time — so what is announced is what is served now, re-served memberships included.
|
||||
func Serve(conn *bus.Conn, s Service, current func() []Endpoint) (func(), error) {
|
||||
started := time.Now().UTC()
|
||||
answer := func(subject string, _ []byte) []byte {
|
||||
parts := strings.Split(subject, ".")
|
||||
if len(parts) < 2 || parts[0] != "$SRV" {
|
||||
return nil
|
||||
}
|
||||
if len(parts) >= 3 && parts[2] != s.Name {
|
||||
return nil // another service's
|
||||
}
|
||||
if len(parts) >= 4 && parts[3] != s.ID {
|
||||
return nil // another instance's
|
||||
}
|
||||
var v any
|
||||
switch parts[1] {
|
||||
case "PING":
|
||||
id := s.identity()
|
||||
md := map[string]string{}
|
||||
for k, v := range id.Metadata {
|
||||
md[k] = v
|
||||
}
|
||||
md[Digest] = DigestOf(Info(s, current()))
|
||||
id.Metadata = md
|
||||
v = micro.Ping{ServiceIdentity: id, Type: micro.PingResponseType}
|
||||
case "INFO":
|
||||
v = Info(s, current())
|
||||
case "STATS":
|
||||
st := micro.Stats{ServiceIdentity: s.identity(), Type: micro.StatsResponseType, Started: started,
|
||||
Endpoints: []*micro.EndpointStats{}}
|
||||
for _, e := range current() {
|
||||
st.Endpoints = append(st.Endpoints, µ.EndpointStats{Name: e.Name(), Subject: e.Subject, QueueGroup: e.Queue})
|
||||
}
|
||||
v = st
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
body, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
// Too large for the bus is no answer at all, and was silent: shorten every description to its
|
||||
// first line and say so, rather than vanish; still too large, leave descriptions out.
|
||||
if limit := conn.MaxPayload(); limit > 0 && int64(len(body)) > limit {
|
||||
if info, ok := v.(micro.Info); ok {
|
||||
body = shorten(conn.Logf, info, body, limit)
|
||||
}
|
||||
}
|
||||
return body
|
||||
}
|
||||
var stops []func()
|
||||
for _, verb := range []string{"PING", "INFO", "STATS"} {
|
||||
// Exactly the questions asked of every service and of this one by name and instance — what the
|
||||
// grants allow (novox/hq ADR 0197). A wildcard is refused by the bus.
|
||||
for _, subject := range []string{"$SRV." + verb, "$SRV." + verb + "." + s.Name, "$SRV." + verb + "." + s.Name + "." + s.ID} {
|
||||
stop, err := conn.Raw(subject, answer)
|
||||
if err != nil {
|
||||
for _, st := range stops {
|
||||
st()
|
||||
}
|
||||
return func() {}, err
|
||||
}
|
||||
stops = append(stops, stop)
|
||||
}
|
||||
}
|
||||
return func() {
|
||||
for _, st := range stops {
|
||||
st()
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DigestOf is a digest of an info_response: equal for equal announcements, whatever the moment.
|
||||
func DigestOf(i micro.Info) string {
|
||||
body, err := json.Marshal(i)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return hex.EncodeToString(sum[:12])
|
||||
}
|
||||
|
||||
// shorten is an info_response cut to fit the bus: descriptions to their first line, then none, the
|
||||
// service's metadata saying which (so discovery can tell a person what it was given).
|
||||
func shorten(logf func(string, ...any), info micro.Info, body []byte, limit int64) []byte {
|
||||
md := map[string]string{}
|
||||
for k, v := range info.Metadata {
|
||||
md[k] = v
|
||||
}
|
||||
info.Metadata = md
|
||||
for _, step := range []string{"descriptions cut to their first line", "descriptions left out"} {
|
||||
for i := range info.Endpoints {
|
||||
e := info.Endpoints[i].Metadata
|
||||
if step == "descriptions left out" {
|
||||
delete(e, "description")
|
||||
} else {
|
||||
e["description"] = firstLine(e["description"])
|
||||
}
|
||||
}
|
||||
md[Shortened] = step
|
||||
shorter, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
logf("[mesh-tools] what this runtime serves is %d bytes, beyond the bus's %d; announced with %s (%d bytes)",
|
||||
len(body), limit, step, len(shorter))
|
||||
if int64(len(shorter)) <= limit {
|
||||
return shorter
|
||||
}
|
||||
body = shorter
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// Heard is one service instance's answer to discovery: what it serves, how long it took to arrive,
|
||||
// and how large it was.
|
||||
type Heard struct {
|
||||
Info micro.Info
|
||||
Took time.Duration
|
||||
Bytes int
|
||||
Shortened string // why its descriptions are not whole, or ""
|
||||
// Digest is the digest its PING gave for this answer, or "" when it gave none.
|
||||
Digest string
|
||||
// Reused is true when the instance's PING said it serves what the console already held, so its
|
||||
// announcement was not asked for again; Took is then how long its PING took.
|
||||
Reused bool
|
||||
}
|
||||
|
||||
// Machine is the machine an instance runs on: its metadata's, or the one its endpoints name.
|
||||
func (h Heard) Machine() string {
|
||||
if n := h.Info.Metadata["node"]; n != "" {
|
||||
return n
|
||||
}
|
||||
for _, e := range h.Info.Endpoints {
|
||||
if n := e.Metadata["node"]; n != "" {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Instance names one service instance.
|
||||
type Instance struct {
|
||||
Name string
|
||||
ID string
|
||||
Machine string
|
||||
}
|
||||
|
||||
// Discovery is what one round of discovery heard, and who said it was there but did not say what it
|
||||
// serves in time.
|
||||
type Discovery struct {
|
||||
At time.Time
|
||||
Waited time.Duration
|
||||
Heard []Heard
|
||||
Silent []Instance
|
||||
// Pinged is how many PING answers arrived.
|
||||
Pinged int
|
||||
}
|
||||
|
||||
// Empty is a round in which nothing arrived at all, not even a PING: no answer about the mesh, but a
|
||||
// round that failed (novox/hq issue 554).
|
||||
func (d Discovery) Empty() bool { return d.Pinged == 0 && len(d.Heard) == 0 }
|
||||
|
||||
// Known is the announcement a console already holds for an instance at this digest, if any.
|
||||
type Known func(name, id, digest string) (Heard, bool)
|
||||
|
||||
// Gather asks every service on the bus who it is (`$SRV.PING`), and each instance that answers what it
|
||||
// serves (`$SRV.INFO.<name>.<id>`) unless its PING's digest is one `known` holds; it waits at least the
|
||||
// window and at most the patience: until at least one instance answered PING and every instance that
|
||||
// did has said what it serves. A round in which no PING arrives within the window is not taken as the
|
||||
// mesh's answer that nothing is there (novox/hq issue 554): it waits on, up to the patience. An answer
|
||||
// that is not the protocol's is skipped.
|
||||
func Gather(conn *bus.Conn, known Known) (Discovery, error) {
|
||||
start := time.Now()
|
||||
d := Discovery{At: start.UTC()}
|
||||
pings, stopPings, err := conn.Collect("$SRV.PING", nil)
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
defer stopPings()
|
||||
infos, askInfo, stopInfos, err := conn.Collector("$SRV.INFO")
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
defer stopInfos()
|
||||
|
||||
type key struct{ name, id string }
|
||||
pinged := map[key]int{}
|
||||
answered := map[key]int{}
|
||||
where := map[key]string{}
|
||||
digest := map[key]string{}
|
||||
asked := map[key]bool{}
|
||||
reused := map[key]int{} // index in d.Heard of an answer held already
|
||||
var order []key
|
||||
complete := func() bool {
|
||||
if len(pinged) == 0 {
|
||||
return false // nobody said they are there: not an answer, a round that has not arrived
|
||||
}
|
||||
for k, n := range pinged {
|
||||
if answered[k] < n {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
ask := func(k key) {
|
||||
if asked[k] {
|
||||
return
|
||||
}
|
||||
asked[k] = true
|
||||
if err := askInfo("$SRV.INFO."+k.name+"."+k.id, nil); err != nil {
|
||||
conn.Logf("[mesh-tools] discovery could not ask %s/%s what it serves: %v", k.name, k.id, err)
|
||||
}
|
||||
}
|
||||
least := time.NewTimer(Window)
|
||||
defer least.Stop()
|
||||
most := time.NewTimer(Patience)
|
||||
defer most.Stop()
|
||||
windowOver := false
|
||||
for {
|
||||
select {
|
||||
case a := <-pings:
|
||||
var p micro.Ping
|
||||
if json.Unmarshal(a.Data, &p) != nil || p.Type != micro.PingResponseType {
|
||||
continue
|
||||
}
|
||||
d.Pinged++
|
||||
k := key{p.Name, p.ID}
|
||||
_, seen := pinged[k]
|
||||
if !seen {
|
||||
order = append(order, k)
|
||||
}
|
||||
pinged[k]++
|
||||
if where[k] == "" {
|
||||
where[k] = p.Metadata["node"]
|
||||
}
|
||||
if seen {
|
||||
// Two instances under one name and id: what each serves is asked of both, never assumed.
|
||||
if i, ok := reused[k]; ok {
|
||||
d.Heard = append(d.Heard[:i], d.Heard[i+1:]...)
|
||||
for kk, j := range reused {
|
||||
if j > i {
|
||||
reused[kk] = j - 1
|
||||
}
|
||||
}
|
||||
delete(reused, k)
|
||||
answered[k]--
|
||||
}
|
||||
ask(k)
|
||||
continue
|
||||
}
|
||||
digest[k] = p.Metadata[Digest]
|
||||
if h, ok := heldAt(known, k.name, k.id, digest[k]); ok {
|
||||
h.Took, h.Reused, h.Digest = a.At.Sub(start), true, digest[k]
|
||||
reused[k] = len(d.Heard)
|
||||
d.Heard = append(d.Heard, h)
|
||||
answered[k]++
|
||||
break
|
||||
}
|
||||
ask(k)
|
||||
case a := <-infos:
|
||||
var i micro.Info
|
||||
if json.Unmarshal(a.Data, &i) != nil || i.Type != micro.InfoResponseType {
|
||||
continue
|
||||
}
|
||||
k := key{i.Name, i.ID}
|
||||
answered[k]++
|
||||
d.Heard = append(d.Heard, Heard{Info: i, Took: a.At.Sub(start), Bytes: len(a.Data), Shortened: i.Metadata[Shortened],
|
||||
Digest: digest[k]})
|
||||
case <-least.C:
|
||||
windowOver = true
|
||||
case <-most.C:
|
||||
d.Waited = time.Since(start)
|
||||
for _, k := range order {
|
||||
if answered[k] < pinged[k] {
|
||||
d.Silent = append(d.Silent, Instance{Name: k.name, ID: k.id, Machine: where[k]})
|
||||
}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
if windowOver && complete() {
|
||||
d.Waited = time.Since(start)
|
||||
return d, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// heldAt is the answer `known` holds for an instance at this digest; never for a PING without one.
|
||||
func heldAt(known Known, name, id, digest string) (Heard, bool) {
|
||||
if known == nil || digest == "" {
|
||||
return Heard{}, false
|
||||
}
|
||||
return known(name, id, digest)
|
||||
}
|
||||
|
||||
// Endpoints reads an info_response's endpoints back into what they announce.
|
||||
func Endpoints(i micro.Info) []Endpoint {
|
||||
var out []Endpoint
|
||||
for _, e := range i.Endpoints {
|
||||
md := e.Metadata
|
||||
out = append(out, Endpoint{
|
||||
Kind: md["kind"], Module: md["module"], Tool: md["tool"], Seat: md["seat"], Scope: md["scope"],
|
||||
Node: md["node"], Description: md["description"], Schema: json.RawMessage(md["schema"]),
|
||||
Interchangeable: md["interchangeable"] == "true", Subject: e.Subject, Queue: e.QueueGroup,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// firstLine is a description's first sentence or line, at most 160 characters.
|
||||
func firstLine(s string) string {
|
||||
if i := strings.IndexAny(s, "\n"); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
if i := strings.Index(s, ". "); i >= 0 {
|
||||
s = s[:i+1]
|
||||
}
|
||||
if len(s) > 160 {
|
||||
s = s[:157] + "..."
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// What a machine serves fits the bus's largest message (1 MiB) with room to spare at several hundred
|
||||
// tools: a module's tool is announced without its schema, which made a machine with ~200 tools outgrow it
|
||||
// and vanish from discovery (2026-10-04). A seat's verb keeps its schema.
|
||||
func TestAMachinesAnnouncementFitsTheBusAtSeveralHundredTools(t *testing.T) {
|
||||
schema := json.RawMessage(`{"type":"object","properties":{` + strings.Repeat(`"field_with_a_long_name":{"type":"string","description":"a description of what this argument means, long enough to matter"},`, 12) + `"last":{"type":"string"}}}`)
|
||||
description := strings.Repeat("A tool that does something useful, described at the length the catalogue's tools are. ", 3)
|
||||
var endpoints []Endpoint
|
||||
for i := 0; i < 400; i++ {
|
||||
for _, subject := range []string{fmt.Sprintf("mesh.mod.m%d.tool.t", i), fmt.Sprintf("mesh.mod.m%d.tool.t.laptop", i)} {
|
||||
endpoints = append(endpoints, Endpoint{Kind: KindTool, Module: fmt.Sprintf("m%d", i), Tool: "t", Node: "laptop",
|
||||
Description: description, Schema: schema, Subject: subject})
|
||||
}
|
||||
}
|
||||
endpoints = append(endpoints, Endpoint{Kind: KindSeat, Module: "m0", Tool: "verb", Seat: "a-seat", Scope: "mesh",
|
||||
Description: "a verb", Schema: schema, Subject: "mesh.seat.a-seat.tool.verb"})
|
||||
body, err := json.Marshal(Info(Service{Name: "node-tools", ID: "laptop"}, endpoints))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(body) > 1<<20 {
|
||||
t.Fatalf("400 tools announce %d bytes, beyond the bus's 1 MiB", len(body))
|
||||
}
|
||||
t.Logf("400 tools on two subjects each announce %d bytes", len(body))
|
||||
back := Endpoints(Info(Service{Name: "node-tools", ID: "laptop"}, endpoints))
|
||||
if len(back[0].Schema) != 0 {
|
||||
t.Fatalf("a module's tool still carries its schema: %s", back[0].Schema)
|
||||
}
|
||||
if seat := back[len(back)-1]; !strings.Contains(string(seat.Schema), "field_with_a_long_name") {
|
||||
t.Fatalf("a seat's verb lost its schema: %s", seat.Schema)
|
||||
}
|
||||
withSchemas := 0
|
||||
for range endpoints {
|
||||
withSchemas += len(schema) * 2 // escaped inside a string
|
||||
}
|
||||
t.Logf("with every schema it would have been over %d bytes", len(body)+withSchemas)
|
||||
}
|
||||
|
||||
func TestFirstLineShortensADescription(t *testing.T) {
|
||||
if got := firstLine("One thing. And more after it."); got != "One thing." {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
if got := firstLine(strings.Repeat("x", 300)); len(got) != 160 {
|
||||
t.Fatalf("%d", len(got))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
)
|
||||
|
||||
// novox/hq issue 554: the laptop's console took a round in which not even a PING arrived within the
|
||||
// 750 ms window as the mesh's answer that nothing is there, and refused every seat and module call. A
|
||||
// PING that arrives after the window is waited for, up to the patience.
|
||||
func TestARoundWaitsForAPingThatArrivesAfterTheWindow(t *testing.T) {
|
||||
was := Patience
|
||||
Patience = 2500 * time.Millisecond
|
||||
t.Cleanup(func() { Patience = was })
|
||||
|
||||
late := connect(t, "node-tools", "laptop")
|
||||
s := Service{Name: "node-tools", ID: "laptop", Metadata: map[string]string{"node": "laptop"}}
|
||||
ping, _ := json.Marshal(micro.Ping{ServiceIdentity: s.identity(), Type: micro.PingResponseType})
|
||||
stop, err := late.Raw("$SRV.PING", func(string, []byte) []byte { time.Sleep(Window + 300*time.Millisecond); return ping })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
body, _ := json.Marshal(Info(s, []Endpoint{{Kind: KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Subject: "mesh.mod.slack.tool.slack_check.laptop"}}))
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO.node-tools.laptop"} {
|
||||
stop, err := late.Raw(subject, func(string, []byte) []byte { return body })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
late.Flush()
|
||||
|
||||
d, err := Gather(connect(t, "console", "laptop"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.Empty() || len(d.Heard) != 1 || d.Heard[0].Machine() != "laptop" {
|
||||
t.Fatalf("a PING after the window was not waited for: waited %s, pinged %d, heard %+v", d.Waited, d.Pinged, d.Heard)
|
||||
}
|
||||
if d.Waited < Window || d.Waited >= Patience {
|
||||
t.Errorf("waited %s; the PING came after %s and patience is %s", d.Waited, Window+300*time.Millisecond, Patience)
|
||||
}
|
||||
}
|
||||
|
||||
// A round in which nothing at all arrives is said to be empty, after the patience — not answered as an
|
||||
// empty mesh at the window.
|
||||
func TestARoundThatHearsNothingIsEmptyAfterThePatience(t *testing.T) {
|
||||
was := Patience
|
||||
Patience = 1500 * time.Millisecond
|
||||
t.Cleanup(func() { Patience = was })
|
||||
d, err := Gather(connect(t, "console", "laptop"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !d.Empty() || d.Waited < Patience {
|
||||
t.Errorf("empty %v after %s (patience %s)", d.Empty(), d.Waited, Patience)
|
||||
}
|
||||
}
|
||||
|
||||
// What a runtime announces is fetched again only when its PING's digest changed: every round pulled
|
||||
// some 880 kB of announcements to the laptop, and the PING answers queued behind them (issue 554).
|
||||
func TestAnUnchangedAnnouncementIsNotFetchedAgain(t *testing.T) {
|
||||
runtime := connect(t, "node-tools", "desk")
|
||||
var endpoints atomic.Value
|
||||
endpoints.Store([]Endpoint{{Kind: KindTool, Module: "alpha", Tool: "one", Node: "desk", Subject: "mesh.mod.alpha.tool.one.desk"}})
|
||||
stop, err := Serve(runtime, Service{Name: "node-tools", ID: "desk", Metadata: map[string]string{"node": "desk"}},
|
||||
func() []Endpoint { return endpoints.Load().([]Endpoint) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
runtime.Flush()
|
||||
// Every request for what desk serves, counted where it is answered.
|
||||
var asked atomic.Int32
|
||||
spy := connect(t, "node-tools", "spy")
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO.node-tools", "$SRV.INFO.node-tools.desk"} {
|
||||
stop, err := spy.Raw(subject, func(string, []byte) []byte { asked.Add(1); return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
spy.Flush()
|
||||
|
||||
console := connect(t, "console", "laptop")
|
||||
first, err := Gather(console, nil)
|
||||
if err != nil || len(first.Heard) != 1 || first.Heard[0].Digest == "" || first.Heard[0].Reused {
|
||||
t.Fatalf("first round: %v %+v", err, first.Heard)
|
||||
}
|
||||
if asked.Load() != 1 {
|
||||
t.Errorf("the first round asked what desk serves %d time(s)", asked.Load())
|
||||
}
|
||||
held := first.Heard[0]
|
||||
known := func(name, id, digest string) (Heard, bool) {
|
||||
if name == held.Info.Name && id == held.Info.ID && digest == held.Digest {
|
||||
return held, true
|
||||
}
|
||||
return Heard{}, false
|
||||
}
|
||||
asked.Store(0)
|
||||
second, err := Gather(console, known)
|
||||
if err != nil || len(second.Heard) != 1 || !second.Heard[0].Reused || len(Endpoints(second.Heard[0].Info)) != 1 {
|
||||
t.Fatalf("an unchanged announcement: %v %+v", err, second.Heard)
|
||||
}
|
||||
if asked.Load() != 0 {
|
||||
t.Errorf("an unchanged announcement was asked for %d time(s)", asked.Load())
|
||||
}
|
||||
|
||||
endpoints.Store([]Endpoint{
|
||||
{Kind: KindTool, Module: "alpha", Tool: "one", Node: "desk", Subject: "mesh.mod.alpha.tool.one.desk"},
|
||||
{Kind: KindTool, Module: "alpha", Tool: "two", Node: "desk", Subject: "mesh.mod.alpha.tool.two.desk"},
|
||||
})
|
||||
third, err := Gather(console, known)
|
||||
if err != nil || len(third.Heard) != 1 || third.Heard[0].Reused || len(Endpoints(third.Heard[0].Info)) != 2 ||
|
||||
third.Heard[0].Digest == held.Digest {
|
||||
t.Fatalf("a changed announcement was not fetched: %v %+v", err, third.Heard)
|
||||
}
|
||||
if asked.Load() != 1 {
|
||||
t.Errorf("a changed announcement was asked for %d time(s)", asked.Load())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
func connect(t *testing.T, module, node string) *bus.Conn {
|
||||
t.Helper()
|
||||
c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: module, Node: node})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Logf = func(string, ...any) {}
|
||||
t.Cleanup(c.Close)
|
||||
return c
|
||||
}
|
||||
|
||||
// lagging answers PING at once and INFO after `lag` — or never, with a negative lag: a runtime whose
|
||||
// one large answer is slow to cross the bus, as the laptop's was (2026-10-05).
|
||||
func lagging(t *testing.T, conn *bus.Conn, s Service, lag time.Duration, endpoints []Endpoint) {
|
||||
t.Helper()
|
||||
ping, _ := json.Marshal(micro.Ping{ServiceIdentity: s.identity(), Type: micro.PingResponseType})
|
||||
stop, err := conn.Raw("$SRV.PING", func(string, []byte) []byte { return ping })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
// Asked of every service, and of this instance by name and id — as discovery asks it (issue 554).
|
||||
info := func(string, []byte) []byte {
|
||||
if lag < 0 {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(lag)
|
||||
body, _ := json.Marshal(Info(s, endpoints))
|
||||
return body
|
||||
}
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO." + s.Name + "." + s.ID} {
|
||||
stop, err := conn.Raw(subject, info)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
conn.Flush()
|
||||
}
|
||||
|
||||
// A runtime whose answer arrives after the window is waited for, because it said it was there; one
|
||||
// that never says what it serves is named, not dropped. With the fixed 750 ms window the laptop's
|
||||
// modules were said to run nowhere whenever its answer was late (2026-10-05).
|
||||
func TestDiscoveryWaitsForEveryRuntimeThatSaidItIsThere(t *testing.T) {
|
||||
was := Patience
|
||||
Patience = 2500 * time.Millisecond
|
||||
t.Cleanup(func() { Patience = was })
|
||||
|
||||
fast := connect(t, "node-tools", "desk")
|
||||
stopFast, err := Serve(fast, Service{Name: "node-tools", ID: "desk", Metadata: map[string]string{"node": "desk"}}, func() []Endpoint {
|
||||
return []Endpoint{{Kind: KindTool, Module: "alpha", Tool: "one", Node: "desk", Subject: "mesh.mod.alpha.tool.one.desk"}}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopFast)
|
||||
fast.Flush()
|
||||
slow := connect(t, "node-tools", "laptop")
|
||||
lagging(t, slow, Service{Name: "node-tools", ID: "laptop", Metadata: map[string]string{"node": "laptop"}}, Window+400*time.Millisecond,
|
||||
[]Endpoint{{Kind: KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Subject: "mesh.mod.slack.tool.slack_check.laptop"}})
|
||||
mute := connect(t, "node-tools", "sleeper")
|
||||
lagging(t, mute, Service{Name: "node-tools", ID: "sleeper", Metadata: map[string]string{"node": "sleeper"}}, -1, nil)
|
||||
|
||||
asker := connect(t, "console", "desk")
|
||||
start := time.Now()
|
||||
d, err := Gather(asker, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
took := time.Since(start)
|
||||
|
||||
heard := map[string]Heard{}
|
||||
for _, h := range d.Heard {
|
||||
heard[h.Info.ID] = h
|
||||
}
|
||||
if h, ok := heard["laptop"]; !ok {
|
||||
t.Fatalf("the slow runtime was not heard: %+v", d.Heard)
|
||||
} else if h.Took <= Window || h.Machine() != "laptop" || h.Bytes == 0 {
|
||||
t.Errorf("the slow runtime's answer: took %s (window %s), machine %q, %d bytes", h.Took, Window, h.Machine(), h.Bytes)
|
||||
}
|
||||
if _, ok := heard["desk"]; !ok {
|
||||
t.Errorf("the fast runtime was not heard: %+v", d.Heard)
|
||||
}
|
||||
if len(d.Silent) != 1 || d.Silent[0].ID != "sleeper" || d.Silent[0].Machine != "sleeper" {
|
||||
t.Errorf("the runtime that never said what it serves is not named: %+v", d.Silent)
|
||||
}
|
||||
if took < Patience || took > Patience+time.Second {
|
||||
t.Errorf("waited %s with a silent runtime; patience is %s", took, Patience)
|
||||
}
|
||||
}
|
||||
|
||||
// With nobody slow, discovery takes the window and no longer.
|
||||
func TestDiscoveryDoesNotWaitWhenEveryoneAnswered(t *testing.T) {
|
||||
fast := connect(t, "node-tools", "desk")
|
||||
stop, err := Serve(fast, Service{Name: "node-tools", ID: "desk"}, func() []Endpoint { return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
fast.Flush()
|
||||
start := time.Now()
|
||||
d, err := Gather(connect(t, "console", "desk"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if took := time.Since(start); took > Window+500*time.Millisecond || len(d.Silent) != 0 || len(d.Heard) == 0 {
|
||||
t.Errorf("took %s, heard %d, silent %+v", took, len(d.Heard), d.Silent)
|
||||
}
|
||||
}
|
||||
|
||||
// Too large even with first lines, descriptions are left out — and the answer says which, without
|
||||
// touching the service's own metadata.
|
||||
func TestAnAnswerTooLargeIsShortenedUntilItFits(t *testing.T) {
|
||||
long := strings.Repeat("A sentence that is long. ", 20)
|
||||
var endpoints []Endpoint
|
||||
for i := 0; i < 50; i++ {
|
||||
endpoints = append(endpoints, Endpoint{Kind: KindTool, Module: "m", Tool: "t" + strings.Repeat("x", i), Node: "laptop",
|
||||
Description: long, Subject: "mesh.mod.m.tool.t"})
|
||||
}
|
||||
own := map[string]string{"node": "laptop"}
|
||||
info := Info(Service{Name: "node-tools", ID: "laptop", Metadata: own}, endpoints)
|
||||
full, _ := json.Marshal(info)
|
||||
firstLines := 0
|
||||
for range endpoints {
|
||||
firstLines += len(firstLine(long))
|
||||
}
|
||||
limit := int64(len(full) - len(long)*len(endpoints) + firstLines/2) // first lines alone do not fit
|
||||
var said []string
|
||||
got := shorten(func(f string, a ...any) { said = append(said, f) }, info, full, limit)
|
||||
if int64(len(got)) > limit {
|
||||
t.Fatalf("still %d bytes, beyond %d", len(got), limit)
|
||||
}
|
||||
var back micro.Info
|
||||
if err := json.Unmarshal(got, &back); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if back.Metadata[Shortened] != "descriptions left out" || back.Metadata["node"] != "laptop" {
|
||||
t.Errorf("metadata: %v", back.Metadata)
|
||||
}
|
||||
if _, has := own[Shortened]; has {
|
||||
t.Error("the service's own metadata was changed")
|
||||
}
|
||||
if len(said) != 2 {
|
||||
t.Errorf("each step is said: %v", said)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,62 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSubjectsAreTheOnesTheTypeScriptRuntimeUses(t *testing.T) {
|
||||
for key, want := range map[string]string{
|
||||
"status": "mesh.mod.self.tool.status",
|
||||
"alpha.one": "mesh.mod.alpha.tool.one",
|
||||
"alpha.one@anchor": "mesh.mod.alpha.tool.one.anchor",
|
||||
"seat:node-shelf.list": "mesh.seat.node-shelf.tool.list",
|
||||
"seat:node-shelf.list@anchor": "mesh.seat.node-shelf.tool.list.anchor",
|
||||
} {
|
||||
if got, err := ToolSubject(key, "self"); err != nil || got != want {
|
||||
t.Errorf("%s: %s %v, want %s", key, got, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := ToolSubject("seat:nothing", "self"); err == nil {
|
||||
t.Error("a seat with no verb was accepted")
|
||||
}
|
||||
if SeatToolSubject("s", "v", "node", "n") != "mesh.seat.s.tool.v.n" || SeatToolSubject("s", "v", "mesh", "n") != "mesh.seat.s.tool.v" {
|
||||
t.Error("seat subjects")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFingerprintIsReadHoweverItIsWritten(t *testing.T) {
|
||||
for _, f := range []string{"sha256:AB:CD:ef", "abcdef", "ABCDEF", "SHA256:abcdef"} {
|
||||
if got := normalizeFingerprint(f); got != "abcdef" {
|
||||
t.Errorf("%s → %s", f, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatWaitingCannotFixIsFinal(t *testing.T) {
|
||||
for err, want := range map[error]string{
|
||||
fmt.Errorf("x509: %w: it presented aa", ErrPin): "the bus's certificate does not match the pin",
|
||||
errors.New("nats: Authorization Violation"): "the bus refused this account",
|
||||
errors.New("nats: invalid url"): "the bus address is not a usable URL",
|
||||
errors.New("dial tcp 10.0.0.1:4222: connect: connection refused"): "",
|
||||
} {
|
||||
if got := Fatal(err); got != want {
|
||||
t.Errorf("%v → %q, want %q", err, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An event given up on and delivered again by the controller arrives under `mesh.again.<consumer>.` and
|
||||
// the event's own subject without its `mesh.` (novox/hq issue 330): the handler sees the key it saw the
|
||||
// first time, for a module's event and a seat's.
|
||||
func TestAnEventDeliveredAgainHasItsOwnKey(t *testing.T) {
|
||||
for original, again := range map[string]string{
|
||||
"mesh.mod.gitea.event.pull.merged": "mesh.again.ace_sonarr.mod.gitea.event.pull.merged",
|
||||
"mesh.seat.node-build-agent.event.built": "mesh.again.ace_sonarr.seat.node-build-agent.event.built",
|
||||
} {
|
||||
if keyOf(again) != keyOf(original) {
|
||||
t.Errorf("%s reads as %q, the event as %q", again, keyOf(again), keyOf(original))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A tool call names its caller (novox/hq issue 365, by ADR 0259 §3's precedent for asks).
|
||||
//
|
||||
// mesh.mod.<module>.call.<tool>.<caller> to whichever instance answers
|
||||
// mesh.mod.<module>.call.<tool>.<node>.<caller> to the instance on one machine
|
||||
// mesh.seat.<seat>.call.<verb>.<caller> a mesh seat's verb
|
||||
// mesh.seat.<seat>.call.<verb>.<node>.<caller> a node seat's verb on one machine
|
||||
//
|
||||
// The last token is the bus user that published the call, and the controller grants each user the publish of
|
||||
// these subjects with its own name there and no other — so the caller is a fact the bus server enforces, and
|
||||
// the runtime hands it to the module from the subject a call arrived on, never from the payload.
|
||||
//
|
||||
// **A kind of its own, `call`, not the `tool` subject with a token appended.** Every grant to call a tool is a
|
||||
// wildcard over the `tool` kind — `mesh.mod.<m>.tool.<t>.*` for the instance on any machine, `mesh.mod.*.tool.>`
|
||||
// for every tool — and either matches a `tool` subject with any caller appended, so a caller could name
|
||||
// another. Under `call` no grant but the caller-named ones reaches the caller token.
|
||||
//
|
||||
// The `tool` subjects are served beside these for one release, and say Unproven, so callers move without a
|
||||
// gap; their retirement is hq issue 464.
|
||||
const CallKind = "call"
|
||||
|
||||
// A bus user's name carries dots (`<node>.<module>`, `person.<name>`); a subject token cannot. In the token
|
||||
// each dot is written `~`, which no part of a bus user's name may hold (the controller refuses any part that
|
||||
// is not letters, digits, `_` or `-`), so the token is the user's name and nothing else.
|
||||
const userDot = "~"
|
||||
|
||||
var userName = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`)
|
||||
|
||||
// CallerToken is a bus user's name as the last token of a call it publishes, or "" for a name that is not a
|
||||
// bus user's.
|
||||
func CallerToken(user string) string {
|
||||
if !userName.MatchString(user) {
|
||||
return ""
|
||||
}
|
||||
return strings.ReplaceAll(user, ".", userDot)
|
||||
}
|
||||
|
||||
// UserOfToken is the bus user a caller token names, or "" when it names none.
|
||||
func UserOfToken(token string) string {
|
||||
user := strings.ReplaceAll(token, userDot, ".")
|
||||
if !userName.MatchString(user) {
|
||||
return ""
|
||||
}
|
||||
return user
|
||||
}
|
||||
|
||||
// toolShape splits a tool subject — `mesh.mod.<m>.tool.<rest>` or `mesh.seat.<s>.tool.<rest>` — into what
|
||||
// comes before the kind and what after; ok is false for any other subject.
|
||||
func toolShape(subject string) (head, rest string, ok bool) {
|
||||
parts := strings.SplitN(subject, ".", 5)
|
||||
if len(parts) != 5 || parts[0] != "mesh" || (parts[1] != "mod" && parts[1] != "seat") || parts[3] != "tool" ||
|
||||
parts[2] == "" || parts[4] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0] + "." + parts[1] + "." + parts[2], parts[4], true
|
||||
}
|
||||
|
||||
// CalledSubject is where a call to a tool subject goes naming its caller: the same address under the `call`
|
||||
// kind, the caller's token last. "" for a subject that is not a tool's, or a caller that is not a bus user.
|
||||
func CalledSubject(subject, user string) string {
|
||||
head, rest, ok := toolShape(subject)
|
||||
token := CallerToken(user)
|
||||
if !ok || token == "" || strings.ContainsAny(rest, "*>") {
|
||||
return ""
|
||||
}
|
||||
return head + "." + CallKind + "." + rest + "." + token
|
||||
}
|
||||
|
||||
// calledPattern is what a runtime answering a tool subject also subscribes, to hear the calls that name their
|
||||
// caller: the same address under the `call` kind, with any caller. "" for a subject that is not a tool's.
|
||||
func calledPattern(subject string) string {
|
||||
head, rest, ok := toolShape(subject)
|
||||
if !ok || strings.ContainsAny(rest, "*>") {
|
||||
return ""
|
||||
}
|
||||
return head + "." + CallKind + "." + rest + ".*"
|
||||
}
|
||||
|
||||
// callerOn is the bus user a call names, from the subject it arrived on: its last token, under a pattern
|
||||
// calledPattern made. "" — said by the module as Unproven — for a call on the subject that names no caller.
|
||||
func callerOn(pattern, subject string) string {
|
||||
// Only the `call` kind names a caller: a `tool` pattern ending in a wildcard (a node seat's verb on any
|
||||
// machine) is never read as one.
|
||||
parts := strings.SplitN(pattern, ".", 5)
|
||||
if len(parts) != 5 || parts[0] != "mesh" || (parts[1] != "mod" && parts[1] != "seat") || parts[3] != CallKind ||
|
||||
!strings.HasSuffix(pattern, ".*") {
|
||||
return ""
|
||||
}
|
||||
dot := strings.LastIndex(subject, ".")
|
||||
if dot < 0 || subject[:dot] != strings.TrimSuffix(pattern, ".*") {
|
||||
return ""
|
||||
}
|
||||
return UserOfToken(subject[dot+1:])
|
||||
}
|
||||
|
||||
// CalledHandler answers one call, told who called: the bus user the subject named, or "" when it named none.
|
||||
type CalledHandler func(caller string, body json.RawMessage) (any, error)
|
||||
|
||||
// callers is what a connection keeps of its caller-named calls: those in flight, by subject, so a refused
|
||||
// publish ends its call at once instead of after RequestTimeout.
|
||||
type callers struct {
|
||||
mu sync.Mutex
|
||||
// fellBack are the caller-named subjects already said to have reached nobody, so each is said once.
|
||||
fellBack map[string]bool
|
||||
next uint64
|
||||
pending map[string]map[uint64]context.CancelCauseFunc
|
||||
}
|
||||
|
||||
// errCallRefused is the bus refusing a caller-named publish: this user is not granted it yet (the controller's
|
||||
// grant not delivered to the bus), so the call is asked again on the subject that names no caller.
|
||||
var errCallRefused = errors.New("the bus refused the caller-named subject")
|
||||
|
||||
var refusedPublish = regexp.MustCompile(`(?i)Publish to "(\S+)"`)
|
||||
|
||||
// publishRefused ends every call in flight on a subject the bus refused this connection the publish of.
|
||||
func (c *Conn) publishRefused(err error) bool {
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
return false
|
||||
}
|
||||
m := refusedPublish.FindStringSubmatch(err.Error())
|
||||
if len(m) < 2 {
|
||||
return false
|
||||
}
|
||||
c.calls.mu.Lock()
|
||||
cancels := c.calls.pending[m[1]]
|
||||
delete(c.calls.pending, m[1])
|
||||
c.calls.mu.Unlock()
|
||||
if len(cancels) > 0 {
|
||||
c.Logf("[mesh-tools] the bus refused this account the caller-named %s; asked again on the subject that "+
|
||||
"names no caller, until the mesh grants it (novox/hq issue 365)", m[1])
|
||||
}
|
||||
for _, cancel := range cancels {
|
||||
cancel(errCallRefused)
|
||||
}
|
||||
return len(cancels) > 0
|
||||
}
|
||||
|
||||
// askCalled asks once on a caller-named subject. fallBack says the call reached nobody — nothing serves that
|
||||
// subject yet, or the bus refused this user its publish — so it may be asked on the subject that names no
|
||||
// caller without being done twice.
|
||||
func (c *Conn) askCalled(subject string, data []byte) (answered Answered, handingOver, fallBack bool, err error) {
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
ctx, stop := context.WithTimeout(ctx, RequestTimeout)
|
||||
defer stop()
|
||||
c.calls.mu.Lock()
|
||||
if c.calls.pending == nil {
|
||||
c.calls.pending = map[string]map[uint64]context.CancelCauseFunc{}
|
||||
}
|
||||
if c.calls.pending[subject] == nil {
|
||||
c.calls.pending[subject] = map[uint64]context.CancelCauseFunc{}
|
||||
}
|
||||
c.calls.next++
|
||||
id := c.calls.next
|
||||
c.calls.pending[subject][id] = cancel
|
||||
c.calls.mu.Unlock()
|
||||
defer func() {
|
||||
c.calls.mu.Lock()
|
||||
if list := c.calls.pending[subject]; list != nil {
|
||||
delete(list, id)
|
||||
if len(list) == 0 {
|
||||
delete(c.calls.pending, subject)
|
||||
}
|
||||
}
|
||||
c.calls.mu.Unlock()
|
||||
cancel(nil)
|
||||
}()
|
||||
msg, err := c.nc.RequestWithContext(ctx, subject, data)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(context.Cause(ctx), errCallRefused):
|
||||
return Answered{}, false, true, nil
|
||||
case errors.Is(err, nats.ErrNoResponders):
|
||||
c.saidFallBack(subject)
|
||||
return Answered{}, false, true, nil
|
||||
case errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout):
|
||||
return Answered{}, false, false, errors.New("timeout")
|
||||
}
|
||||
return Answered{}, false, false, err
|
||||
}
|
||||
answered, handingOver, err = c.answerOf(subject, msg.Data)
|
||||
return answered, handingOver, false, err
|
||||
}
|
||||
|
||||
// saidFallBack says, once per subject, that a caller-named call reached nobody and is asked on the subject that
|
||||
// names no caller: a runtime or holder not yet serving it, or a grant that drifted, is seen in the log rather
|
||||
// than read later as a quiet "unproven" (novox/hq issue 365; expected for the controller's seat until hq issue
|
||||
// 464).
|
||||
func (c *Conn) saidFallBack(subject string) {
|
||||
c.calls.mu.Lock()
|
||||
if c.calls.fellBack == nil {
|
||||
c.calls.fellBack = map[string]bool{}
|
||||
}
|
||||
first := !c.calls.fellBack[subject]
|
||||
c.calls.fellBack[subject] = true
|
||||
c.calls.mu.Unlock()
|
||||
if first {
|
||||
c.Logf("[mesh-tools] nothing answers the caller-named %s; asked on the subject that names no caller, where "+
|
||||
"the module is told no caller (said once per subject; novox/hq issue 365)", subject)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A bus whose users are granted as the controller grants them (mesh-controller internal/broker): each named
|
||||
// user publishes the subjects listed for it.
|
||||
func grantedBus(t *testing.T, publish map[string][]string) string {
|
||||
t.Helper()
|
||||
var users []*server.User
|
||||
for user, allow := range publish {
|
||||
// And the membership's direct get every connection asks once, answered "no responders" here.
|
||||
allow = append(allow, "$JS.API.DIRECT.GET.>")
|
||||
users = append(users, &server.User{Username: user, Password: "pw", Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: allow},
|
||||
Subscribe: &server.SubjectPermission{Allow: []string{">"}},
|
||||
}})
|
||||
}
|
||||
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, NoLog: true, NoSigs: true,
|
||||
Users: users})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
return s.ClientURL()
|
||||
}
|
||||
|
||||
func dial(t *testing.T, url, module, node, user string) *bus.Conn {
|
||||
t.Helper()
|
||||
c, err := bus.Connect(bus.Credential{URL: url, Module: module, Node: node, User: user, Password: "pw"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Logf = func(string, ...any) {}
|
||||
t.Cleanup(c.Close)
|
||||
return c
|
||||
}
|
||||
|
||||
type heard struct {
|
||||
mu sync.Mutex
|
||||
callers []string
|
||||
}
|
||||
|
||||
func (h *heard) handler(caller string, _ json.RawMessage) (any, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.callers = append(h.callers, caller)
|
||||
return map[string]any{"caller": caller}, nil
|
||||
}
|
||||
|
||||
func (h *heard) all() []string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]string(nil), h.callers...)
|
||||
}
|
||||
|
||||
// A caller asks on the caller-named subject first; where nothing serves it yet — a runtime from before issue
|
||||
// 365 — the same call is asked once on the subject that names no caller, at once, and done once.
|
||||
func TestACallerFallsBackAtOnceWhereNothingServesTheCallerNamedSubject(t *testing.T) {
|
||||
url := mt.URL(t)
|
||||
server := dial(t, url, "ledger", "anchor", "")
|
||||
h := &heard{}
|
||||
// An older runtime: the subject that names no caller, alone.
|
||||
stop, err := server.HandleSubject("mesh.mod.ledger.tool.who", func(body json.RawMessage) (any, error) {
|
||||
return h.handler("", body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
server.Flush()
|
||||
asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools")
|
||||
began := time.Now()
|
||||
answered, err := asker.Ask("ledger.who", map[string]any{}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if took := time.Since(began); took > 2*time.Second {
|
||||
t.Errorf("the fall-back took %s", took)
|
||||
}
|
||||
if got := h.all(); len(got) != 1 || got[0] != "" || !strings.Contains(string(answered.Result), `"caller":""`) {
|
||||
t.Errorf("the call was answered %v, heard %v; want once, on the old subject", string(answered.Result), got)
|
||||
}
|
||||
}
|
||||
|
||||
// Where the bus does not grant the caller-named subject yet — the controller's grants not yet delivered — the
|
||||
// refusal ends the call at once and it is asked on the subject that names no caller: no caller waits out
|
||||
// RequestTimeout for a grant that is on its way.
|
||||
func TestACallerTheBusRefusesTheCallerNamedSubjectFallsBackAtOnce(t *testing.T) {
|
||||
url := grantedBus(t, map[string][]string{
|
||||
"anchor.node-tools": {"_INBOX.>"},
|
||||
"shanks.node-tools": {"mesh.mod.*.tool.>"},
|
||||
})
|
||||
server := dial(t, url, "node-tools", "anchor", "anchor.node-tools")
|
||||
h := &heard{}
|
||||
stop, err := server.HandleSubjectCalled("mesh.mod.ledger.tool.who", h.handler)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
server.Flush()
|
||||
asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools")
|
||||
var said []string
|
||||
var mu sync.Mutex
|
||||
asker.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
began := time.Now()
|
||||
if _, err := asker.Ask("ledger.who", map[string]any{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
if !strings.Contains(strings.Join(said, "\n"), "the bus refused this account the caller-named mesh.mod.ledger.call.who.shanks~node-tools") {
|
||||
t.Errorf("the refusal was not said: %v", said)
|
||||
}
|
||||
mu.Unlock()
|
||||
if took := time.Since(began); took > 3*time.Second {
|
||||
t.Errorf("the refused caller-named call held the caller %s", took)
|
||||
}
|
||||
if got := h.all(); len(got) != 1 || got[0] != "" {
|
||||
t.Errorf("heard %v; want one call, on the subject that names no caller", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With the controller's grant — the caller-named subjects with its own name last — a caller is heard as itself;
|
||||
// and a caller cannot publish on a subject naming another, however it asks.
|
||||
func TestAGrantedCallerIsHeardAsItselfAndCannotNameAnother(t *testing.T) {
|
||||
url := grantedBus(t, map[string][]string{
|
||||
"anchor.node-tools": {"_INBOX.>"},
|
||||
"shanks.node-tools": {"mesh.mod.*.call.*.shanks~node-tools", "mesh.mod.*.call.*.*.shanks~node-tools"},
|
||||
})
|
||||
server := dial(t, url, "node-tools", "anchor", "anchor.node-tools")
|
||||
h := &heard{}
|
||||
stop, err := server.HandleSubjectCalled("mesh.mod.ledger.tool.who", h.handler)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
server.Flush()
|
||||
asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools")
|
||||
if _, err := asker.Ask("ledger.who", map[string]any{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Naming another on the subject itself: refused by the bus, so nothing is heard.
|
||||
refusals := make(chan error, 4)
|
||||
forger, err := nats.Connect(url, nats.UserInfo("shanks.node-tools", "pw"),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer forger.Close()
|
||||
if err := forger.Publish("mesh.mod.ledger.call.who.novox~node-tools", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = forger.Flush()
|
||||
select {
|
||||
case err := <-refusals:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("the bus said %v, want a permissions violation", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("the bus did not refuse a call naming another caller")
|
||||
}
|
||||
if got := h.all(); len(got) != 1 || got[0] != "shanks.node-tools" {
|
||||
t.Errorf("heard %v; want the asker alone, as itself", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A fall-back for want of a responder on the caller-named subject is said, once per subject, so a runtime or
|
||||
// grant that drifted behind is seen rather than read as a quiet "unproven".
|
||||
func TestAFallBackForWantOfAResponderIsSaidOncePerSubject(t *testing.T) {
|
||||
url := mt.URL(t)
|
||||
// A runtime from before issue 365: the subject that names no caller, alone.
|
||||
older, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer older.Close()
|
||||
if _, err := older.Subscribe("mesh.mod.ledger.tool.who", func(m *nats.Msg) { _ = m.Respond([]byte(`{"result":{}}`)) }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = older.Flush()
|
||||
asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools")
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
asker.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := asker.Ask("ledger.who", map[string]any{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, "nothing answers the caller-named mesh.mod.ledger.call.who.shanks~node-tools") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("the fall-back was said %d times, want once: %v", n, said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package bus
|
||||
|
||||
import "testing"
|
||||
|
||||
// A call names its caller under the `call` kind, the caller's bus user its last token (novox/hq issue 365).
|
||||
func TestACallerNamedSubjectIsTheToolsAddressUnderCallWithTheCallerLast(t *testing.T) {
|
||||
for subject, want := range map[string]string{
|
||||
"mesh.mod.ledger.tool.who": "mesh.mod.ledger.call.who.shanks~node-tools",
|
||||
"mesh.mod.ledger.tool.who.anchor": "mesh.mod.ledger.call.who.anchor.shanks~node-tools",
|
||||
"mesh.seat.issue-tracker.tool.open": "mesh.seat.issue-tracker.call.open.shanks~node-tools",
|
||||
"mesh.seat.node-desk.tool.who.anchor": "mesh.seat.node-desk.call.who.anchor.shanks~node-tools",
|
||||
"mesh.mod.ledger.event.opened": "",
|
||||
"mesh.seat.intake.accept.ask.x": "",
|
||||
"mesh.mod.ledger.tool.>": "",
|
||||
"mesh.mod.*.tool.who": "mesh.mod.*.call.who.shanks~node-tools",
|
||||
"mesh.node.anchor.ask.report": "",
|
||||
"mesh.seat.operator-channel.proof.x.y": "",
|
||||
} {
|
||||
if subject == "mesh.mod.*.tool.who" {
|
||||
continue // a wildcard is never asked on
|
||||
}
|
||||
if got := CalledSubject(subject, "shanks.node-tools"); got != want {
|
||||
t.Errorf("%s → %q, want %q", subject, got, want)
|
||||
}
|
||||
}
|
||||
for _, user := range []string{"", "a b", "x.*", "x.>", "a~b", "a..b"} {
|
||||
if got := CalledSubject("mesh.mod.ledger.tool.who", user); got != "" {
|
||||
t.Errorf("a caller %q that is not a bus user was named: %s", user, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The caller is read back from the subject a call arrived on, and only under the pattern that names one.
|
||||
func TestTheCallerIsTheLastTokenOfTheSubjectTheCallArrivedOn(t *testing.T) {
|
||||
pattern := calledPattern("mesh.mod.ledger.tool.who.anchor")
|
||||
if pattern != "mesh.mod.ledger.call.who.anchor.*" {
|
||||
t.Fatalf("pattern %q", pattern)
|
||||
}
|
||||
for subject, want := range map[string]string{
|
||||
"mesh.mod.ledger.call.who.anchor.shanks~node-tools": "shanks.node-tools",
|
||||
"mesh.mod.ledger.call.who.anchor.controller": "controller",
|
||||
"mesh.mod.ledger.call.who.anchor.person~jochen": "person.jochen",
|
||||
"mesh.mod.ledger.call.who.anchor.a~~b": "",
|
||||
"mesh.mod.ledger.call.what.anchor.controller": "",
|
||||
} {
|
||||
if got := callerOn(pattern, subject); got != want {
|
||||
t.Errorf("%s → %q, want %q", subject, got, want)
|
||||
}
|
||||
}
|
||||
if got := callerOn("mesh.mod.ledger.tool.who.anchor", "mesh.mod.ledger.tool.who.anchor"); got != "" {
|
||||
t.Errorf("a call on the subject that names no caller was handed %q", got)
|
||||
}
|
||||
for _, user := range []string{"controller", "shanks.node-tools", "person.jochen", "node.ace"} {
|
||||
if back := UserOfToken(CallerToken(user)); back != user {
|
||||
t.Errorf("%s → %s → %s", user, CallerToken(user), back)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only a pattern of the `call` kind names a caller: a `tool` subscription with a wildcard last — a node seat's
|
||||
// verb on any machine, say — never yields one, whatever token a call arrived with.
|
||||
func TestAToolPatternNeverYieldsACaller(t *testing.T) {
|
||||
for pattern, subject := range map[string]string{
|
||||
"mesh.mod.ledger.tool.who.*": "mesh.mod.ledger.tool.who.controller",
|
||||
"mesh.seat.node-desk.tool.who.*": "mesh.seat.node-desk.tool.who.shanks~node-tools",
|
||||
"mesh.mod.ledger.event.*": "mesh.mod.ledger.event.controller",
|
||||
} {
|
||||
if got := callerOn(pattern, subject); got != "" {
|
||||
t.Errorf("%s on %s yielded caller %q", pattern, subject, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq issue 387: an event a bundle handles for longer than the consumer's ack wait is kept alive while
|
||||
// it is handled, and the events behind it wait on the bus rather than in the client, so a slow handler and a
|
||||
// burst are each handed over once — never offered again with nothing failed, never toward max-deliver.
|
||||
func TestASlowHandlerAndABurstAreEachHandedOverOnce(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
restore := bus.EventHeartbeat
|
||||
bus.EventHeartbeat = 300 * time.Millisecond
|
||||
t.Cleanup(func() { bus.EventHeartbeat = restore })
|
||||
mesh.Consumer(t, "anchor", "slow", []string{"mesh.mod.alpha.event.>"}, time.Second)
|
||||
c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Logf = func(string, ...any) {}
|
||||
t.Cleanup(c.Close)
|
||||
|
||||
var mu sync.Mutex
|
||||
handled := map[int]int{}
|
||||
stop, err := c.ConsumeAs("slow", func(env bus.Envelope) error {
|
||||
var body struct{ N int }
|
||||
_ = json.Unmarshal(env.Body, &body)
|
||||
time.Sleep(1500 * time.Millisecond) // longer than the ack wait
|
||||
mu.Lock()
|
||||
handled[body.N]++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
for n := 1; n <= 3; n++ {
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": n})
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
ack, notYet := mesh.Pending(t, "anchor", "slow")
|
||||
if ack != 0 || notYet != 0 {
|
||||
return fmt.Errorf("still pending: %d unacknowledged, %d undelivered", ack, notYet)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for n := 1; n <= 3; n++ {
|
||||
if handled[n] != 1 {
|
||||
t.Errorf("event %d was handed over %d times: %v", n, handled[n], handled)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// **A call the controller refused because it is handing over is asked once more** (novox/hq issue 289):
|
||||
// the controller being replaced did nothing and marked its refusal, and the one after it answers. A
|
||||
// refusal not so marked is final, and a second handover refusal is said, not asked a third time.
|
||||
func TestAHandoverRefusalIsAskedOnceMore(t *testing.T) {
|
||||
was := bus.HandoverPause
|
||||
bus.HandoverPause = 10 * time.Millisecond
|
||||
t.Cleanup(func() { bus.HandoverPause = was })
|
||||
|
||||
url := meshtest.URL(t)
|
||||
seat, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer seat.Close()
|
||||
c, err := bus.Connect(bus.Credential{URL: url, Module: "console", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
serve := func(t *testing.T, verb string, answers ...map[string]any) *atomic.Int32 {
|
||||
t.Helper()
|
||||
var asked atomic.Int32
|
||||
sub, err := seat.Subscribe(bus.SeatToolSubject("mesh-controller", verb, "", ""), func(m *nats.Msg) {
|
||||
n := int(asked.Add(1)) - 1
|
||||
if n >= len(answers) {
|
||||
n = len(answers) - 1
|
||||
}
|
||||
body, _ := json.Marshal(answers[n])
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
_ = seat.Flush()
|
||||
return &asked
|
||||
}
|
||||
handingOver := map[string]any{"error": "the controller is handing over to the next one; nothing was done, ask again",
|
||||
"retry": bus.RetryHandingOver}
|
||||
|
||||
asked := serve(t, "rotate", handingOver, map[string]any{"result": "rotated"})
|
||||
got, err := c.Ask("seat:mesh-controller.rotate", map[string]any{}, "")
|
||||
if err != nil || string(got.Result) != `"rotated"` || asked.Load() != 2 {
|
||||
t.Fatalf("after a handover refusal: %s %v, asked %d times", got.Result, err, asked.Load())
|
||||
}
|
||||
|
||||
asked = serve(t, "push", map[string]any{"error": "refused for its own reason"})
|
||||
if _, err := c.Ask("seat:mesh-controller.push", map[string]any{}, ""); err == nil || asked.Load() != 1 {
|
||||
t.Fatalf("an unmarked refusal was asked again: %v, asked %d times", err, asked.Load())
|
||||
}
|
||||
|
||||
asked = serve(t, "status", handingOver)
|
||||
_, err = c.Ask("seat:mesh-controller.status", map[string]any{}, "")
|
||||
if err == nil || asked.Load() != 2 || !strings.Contains(err.Error(), "both times refused as a handover") {
|
||||
t.Fatalf("two handover refusals: %v, asked %d times", err, asked.Load())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// heardMsg is a delivered event as a consumer's loop hands it, saying how it was settled.
|
||||
type heardMsg struct {
|
||||
jetstream.Msg
|
||||
mu sync.Mutex
|
||||
settled []string
|
||||
}
|
||||
|
||||
func (m *heardMsg) Subject() string { return "mesh.mod.alpha.event.happened" }
|
||||
func (m *heardMsg) Headers() nats.Header { return nats.Header{} }
|
||||
func (m *heardMsg) Data() []byte { return []byte(`{"n":1}`) }
|
||||
func (m *heardMsg) InProgress() error { return nil }
|
||||
func (m *heardMsg) Ack() error { return m.say("ack") }
|
||||
func (m *heardMsg) Nak() error { return m.say("nak") }
|
||||
func (m *heardMsg) NakWithDelay(time.Duration) error { return m.say("nak-delayed") }
|
||||
func (m *heardMsg) Term() error { return m.say("term") }
|
||||
func (m *heardMsg) say(how string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.settled = append(m.settled, how)
|
||||
return nil
|
||||
}
|
||||
|
||||
// An event the bus hands a hearing after it was stopped (a module's bundle loaded again, issue 1945) is
|
||||
// offered again at once and never handed to the module: the module's handler that heard it may be gone.
|
||||
func TestAHearingStoppedHandsNoEventOnAndOffersItAgain(t *testing.T) {
|
||||
delivered := 0
|
||||
h := newHearing(nil, nil, "alpha", func(Envelope) error { delivered++; return nil })
|
||||
h.stop()
|
||||
msg := &heardMsg{}
|
||||
h.handle(msg)
|
||||
if delivered != 0 {
|
||||
t.Errorf("a stopped hearing handed the event to the module")
|
||||
}
|
||||
if len(msg.settled) != 1 || msg.settled[0] != "nak" {
|
||||
t.Errorf("a stopped hearing settled the event as %v, want it offered again at once (nak)", msg.settled)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's log on the bus (novox/hq ADR 0297): a stream of operations, one subject per key, that the
|
||||
// controller creates from what the module declared under `logs` and issues to the module's own instances
|
||||
// in its membership. It mirrors a bucket (ADR 0201) in how it is reached: what a module may reach is
|
||||
// decided here, from its membership, and refused with the reason before anything is sent.
|
||||
//
|
||||
// **The runtime uses no consumer of a log.** The bus grants it, per issued log, only the publish of an
|
||||
// entry, the stream's info and the direct get: an append is a publish, a read is a batch direct get
|
||||
// (NATS 2.11), the last entry of a key is a direct get by its subject.
|
||||
|
||||
// LogIssued is one log an assignment may reach (ADR 0297): its owner's alone, so there are no reads.
|
||||
type LogIssued struct {
|
||||
Name string `json:"name"`
|
||||
Stream string `json:"stream"`
|
||||
Subject string `json:"subject"`
|
||||
Writes bool `json:"writes,omitempty"`
|
||||
}
|
||||
|
||||
// LogEntry is one operation in a log: its sequence is its identity.
|
||||
type LogEntry struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// LogPage is one page of a read: Next is the sequence to continue from, 0 when nothing follows; Last
|
||||
// is the stream's last sequence.
|
||||
type LogPage struct {
|
||||
Entries []LogEntry `json:"entries"`
|
||||
Next uint64 `json:"next"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
// LogInfo is how full a log is.
|
||||
type LogInfo struct {
|
||||
Entries uint64 `json:"entries"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
MaxBytes uint64 `json:"max_bytes"`
|
||||
First uint64 `json:"first"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
const (
|
||||
// LogValueMax is the most one entry's value may hold (ADR 0297 §3); the stream's MaxMsgSize is 4 KiB
|
||||
// more, for the headers an append carries.
|
||||
LogValueMax = 256 * 1024
|
||||
// LogKeyMax is the longest key.
|
||||
LogKeyMax = 255
|
||||
// LogReadDefault and LogReadMax are a page's entries when none is asked, and at most.
|
||||
LogReadDefault = 100
|
||||
LogReadMax = 1000
|
||||
// LogPageBytes bounds the bytes of one page.
|
||||
LogPageBytes = 4 * 1024 * 1024
|
||||
)
|
||||
|
||||
// logToken is one name of a key: letters, digits, `_` and `-`.
|
||||
var logToken = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
|
||||
func checkLogKey(key string) error {
|
||||
if key == "" || len(key) > LogKeyMax {
|
||||
return fmt.Errorf("%q is not a log key: one or more names of letters, digits, _ and -, joined by "+
|
||||
"dots, at most %d bytes (novox/hq ADR 0297)", key, LogKeyMax)
|
||||
}
|
||||
for _, t := range strings.Split(key, ".") {
|
||||
if !logToken.MatchString(t) {
|
||||
return fmt.Errorf("%q is not a log key: one or more names of letters, digits, _ and -, joined by "+
|
||||
"dots, at most %d bytes (novox/hq ADR 0297)", key, LogKeyMax)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// logFilter is a read's key pattern as a subject filter under the log's subject: `*` for one name, `>`
|
||||
// for the rest; empty is every key.
|
||||
func logFilter(l LogIssued, pattern string) (string, error) {
|
||||
if pattern == "" {
|
||||
return l.Subject + ".>", nil
|
||||
}
|
||||
parts := strings.Split(pattern, ".")
|
||||
for i, p := range parts {
|
||||
switch {
|
||||
case p == ">" && i == len(parts)-1, p == "*", logToken.MatchString(p):
|
||||
default:
|
||||
return "", fmt.Errorf("%q is not a key pattern: names, `*` for one and `>` for the rest, last "+
|
||||
"(novox/hq ADR 0297)", pattern)
|
||||
}
|
||||
}
|
||||
if len(pattern) > LogKeyMax {
|
||||
return "", fmt.Errorf("%q is longer than a key can be (%d bytes)", pattern, LogKeyMax)
|
||||
}
|
||||
return l.Subject + "." + pattern, nil
|
||||
}
|
||||
|
||||
// issuedLog is the log a module may reach by a name, or the reason it may not.
|
||||
func (c *Conn) issuedLog(module, name string) (LogIssued, error) {
|
||||
m := c.Membership(module)
|
||||
if m == nil {
|
||||
return LogIssued{}, fmt.Errorf("%s has no membership issued on %s yet, so no log of it is reachable "+
|
||||
"until the mesh issues one (novox/hq ADR 0297)", module, c.node)
|
||||
}
|
||||
var names []string
|
||||
for _, l := range m.Logs {
|
||||
if l.Name == name {
|
||||
return l, nil
|
||||
}
|
||||
names = append(names, l.Name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
issued := "none"
|
||||
if len(names) > 0 {
|
||||
issued = strings.Join(names, ", ")
|
||||
}
|
||||
return LogIssued{}, fmt.Errorf("%s keeps no log called %q: a module declares the logs it keeps under `logs`, "+
|
||||
"and a log is its owner's alone; it was issued: %s (novox/hq ADR 0297)", module, name, issued)
|
||||
}
|
||||
|
||||
// notOnTheBus says a log is issued and its stream is not there yet.
|
||||
func notOnTheBus(l LogIssued) error {
|
||||
return fmt.Errorf("the log %q is issued and its stream %s is not on the bus yet: the controller creates "+
|
||||
"it from the catalogue on its next raise (novox/hq ADR 0297)", l.Name, l.Stream)
|
||||
}
|
||||
|
||||
// LogAppend writes one entry for a key, as the module, and answers its sequence. With expect, it is
|
||||
// written only when the key's last entry is still at that sequence (0: the key has none); otherwise it
|
||||
// is refused as a lost compare-and-set, the same error a lost state update answers (ADR 0259).
|
||||
func (c *Conn) LogAppend(module, name, key string, value json.RawMessage, expect *uint64) (uint64, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !l.Writes {
|
||||
return 0, fmt.Errorf("%s was issued the log %s without writing it (novox/hq ADR 0297)", module, name)
|
||||
}
|
||||
if err := checkLogKey(key); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(value) == 0 || !json.Valid(value) {
|
||||
return 0, fmt.Errorf("a log entry's value is JSON (novox/hq ADR 0297)")
|
||||
}
|
||||
if len(value) > LogValueMax {
|
||||
return 0, fmt.Errorf("a log entry's value is at most %d bytes (256 KiB), and this one is %d (novox/hq ADR 0297)",
|
||||
LogValueMax, len(value))
|
||||
}
|
||||
if field := credentialField(value); field != "" {
|
||||
return 0, fmt.Errorf("%s's %s.%s carries a field %q, which names a credential: no secret is kept in a log, "+
|
||||
"which keeps it for good. Name the secret and fetch it on request/reply (novox/hq ADR 0201 §13, ADR 0297)",
|
||||
module, name, key, field)
|
||||
}
|
||||
msg := nats.NewMsg(l.Subject + "." + key)
|
||||
msg.Data = value
|
||||
if expect != nil {
|
||||
msg.Header.Set(jetstream.ExpectedLastSubjSeqHeader, strconv.FormatUint(*expect, 10))
|
||||
}
|
||||
reply, err := c.nc.RequestMsg(msg, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return 0, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var ack struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
ErrCode uint16 `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &ack); err != nil {
|
||||
return 0, fmt.Errorf("the bus answered the append to %s with what is not an acknowledgement: %q", name, reply.Data)
|
||||
}
|
||||
if ack.Error != nil {
|
||||
// The key's last entry is no longer the one expected: a lost compare-and-set, answered as a lost
|
||||
// state update is (10164 is the same refusal from a replicated stream).
|
||||
if code := jetstream.ErrorCode(ack.Error.ErrCode); code == jetstream.JSErrCodeStreamWrongLastSequence ||
|
||||
code == jetstream.JSErrCodeStreamWrongLastSequenceConstant {
|
||||
return 0, changed{name + "." + key}
|
||||
}
|
||||
return 0, fmt.Errorf("the bus refused the append to %s.%s: %s (%d)", name, key, ack.Error.Description, ack.Error.ErrCode)
|
||||
}
|
||||
if ack.Seq == 0 {
|
||||
return 0, fmt.Errorf("the bus answered the append to %s.%s with no sequence", name, key)
|
||||
}
|
||||
return ack.Seq, nil
|
||||
}
|
||||
|
||||
// LogRead is the entries from a sequence on, of the keys matching a pattern, at most limit of them and
|
||||
// about 4 MiB, by one batch direct get.
|
||||
func (c *Conn) LogRead(module, name string, from uint64, keys string, limit int) (*LogPage, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case limit == 0:
|
||||
limit = LogReadDefault
|
||||
case limit < 0 || limit > LogReadMax:
|
||||
return nil, fmt.Errorf("a read's limit is 1 to %d entries, %d when none is given, not %d (novox/hq ADR 0297)",
|
||||
LogReadMax, LogReadDefault, limit)
|
||||
}
|
||||
filter, err := logFilter(l, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if from == 0 {
|
||||
from = 1
|
||||
}
|
||||
msgs, next, err := c.directBatch(l.Stream, filter, from, limit, name, func() error { return notOnTheBus(l) })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page := &LogPage{Entries: []LogEntry{}, Next: next}
|
||||
for _, msg := range msgs {
|
||||
e, err := logEntryOf(l, msg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page.Entries = append(page.Entries, *e)
|
||||
}
|
||||
info, err := c.logInfo(l)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page.Last = info.Last
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// directBatch is the messages of a stream from a sequence on, on the subjects matching filter, at most limit and about
|
||||
// LogPageBytes, by one batch direct get (NATS 2.11), and the sequence to read on from, 0 when nothing follows. name is
|
||||
// what is read, for its errors; missing is the error of a stream that is not on the bus.
|
||||
func (c *Conn) directBatch(stream, filter string, from uint64, limit int, name string, missing func() error) ([]*nats.Msg, uint64, error) {
|
||||
if from == 0 {
|
||||
from = 1
|
||||
}
|
||||
req, _ := json.Marshal(map[string]any{"seq": from, "next_by_subj": filter, "batch": limit, "max_bytes": LogPageBytes})
|
||||
inbox := c.nc.NewInbox()
|
||||
sub, err := c.nc.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := c.nc.PublishRequest("$JS.API.DIRECT.GET."+stream, inbox, req); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var msgs []*nats.Msg
|
||||
var next uint64
|
||||
deadline := time.Now().Add(StateTimeout)
|
||||
for done := false; !done; {
|
||||
msg, err := sub.NextMsg(time.Until(deadline))
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, 0, missing()
|
||||
}
|
||||
if errors.Is(err, nats.ErrTimeout) {
|
||||
return nil, 0, fmt.Errorf("the read of %s did not end in %s", name, StateTimeout)
|
||||
}
|
||||
return nil, 0, err
|
||||
}
|
||||
switch status := msg.Header.Get("Status"); status {
|
||||
case "":
|
||||
msgs = append(msgs, msg)
|
||||
case "204": // the end of the batch: what is still pending after the last message sent
|
||||
pending, _ := strconv.ParseUint(msg.Header.Get("Nats-Num-Pending"), 10, 64)
|
||||
if pending > 0 && len(msgs) > 0 {
|
||||
last, _ := strconv.ParseUint(msgs[len(msgs)-1].Header.Get("Nats-Sequence"), 10, 64)
|
||||
next = last + 1
|
||||
}
|
||||
done = true
|
||||
case "404": // nothing from that sequence on
|
||||
done = true
|
||||
case "503":
|
||||
return nil, 0, missing()
|
||||
default:
|
||||
return nil, 0, fmt.Errorf("the read of %s was refused by the bus: %s %s", name, status, msg.Header.Get("Description"))
|
||||
}
|
||||
}
|
||||
return msgs, next, nil
|
||||
}
|
||||
|
||||
// LogLast is a key's last entry, or nil when it has none.
|
||||
func (c *Conn) LogLast(module, name, key string) (*LogEntry, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkLogKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := c.nc.Request("$JS.API.DIRECT.GET."+l.Stream+"."+l.Subject+"."+key, nil, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch status := reply.Header.Get("Status"); status {
|
||||
case "":
|
||||
return logEntryOf(l, reply)
|
||||
case "404":
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("the last entry of %s.%s could not be read: %s %s", name, key, status, reply.Header.Get("Description"))
|
||||
}
|
||||
}
|
||||
|
||||
// LogInfo is a log's entries, bytes, cap and first and last sequences.
|
||||
func (c *Conn) LogInfo(module, name string) (*LogInfo, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.logInfo(l)
|
||||
}
|
||||
|
||||
func (c *Conn) logInfo(l LogIssued) (*LogInfo, error) {
|
||||
reply, err := c.nc.Request("$JS.API.STREAM.INFO."+l.Stream, nil, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var info struct {
|
||||
Config struct {
|
||||
MaxBytes int64 `json:"max_bytes"`
|
||||
} `json:"config"`
|
||||
State struct {
|
||||
Msgs uint64 `json:"messages"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
} `json:"state"`
|
||||
Error *struct {
|
||||
ErrCode uint16 `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &info); err != nil {
|
||||
return nil, fmt.Errorf("the bus answered the info of %s with what is not one: %q", l.Name, reply.Data)
|
||||
}
|
||||
if info.Error != nil {
|
||||
if info.Error.ErrCode == uint16(jetstream.JSErrCodeStreamNotFound) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
return nil, fmt.Errorf("the info of the log %s was refused: %s (%d)", l.Name, info.Error.Description, info.Error.ErrCode)
|
||||
}
|
||||
var maxBytes uint64
|
||||
if info.Config.MaxBytes > 0 {
|
||||
maxBytes = uint64(info.Config.MaxBytes)
|
||||
}
|
||||
return &LogInfo{Entries: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: maxBytes,
|
||||
First: info.State.FirstSeq, Last: info.State.LastSeq}, nil
|
||||
}
|
||||
|
||||
// logEntryOf is an entry a direct get answered: its key is its subject under the log's.
|
||||
func logEntryOf(l LogIssued, msg *nats.Msg) (*LogEntry, error) {
|
||||
subject := msg.Header.Get("Nats-Subject")
|
||||
key, ok := strings.CutPrefix(subject, l.Subject+".")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the bus answered an entry of %s on %q, which is not under %s", l.Name, subject, l.Subject)
|
||||
}
|
||||
seq, err := strconv.ParseUint(msg.Header.Get("Nats-Sequence"), 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus answered an entry of %s with no sequence", l.Name)
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339Nano, msg.Header.Get("Nats-Time-Stamp"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus answered entry %d of %s with no time: %v", seq, l.Name, err)
|
||||
}
|
||||
return &LogEntry{Sequence: seq, Key: key, Value: valueOf(msg.Data), At: at.UTC()}, nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/wire"
|
||||
)
|
||||
|
||||
// An answer larger than the bus carries in one message (novox/hq issue 314).
|
||||
//
|
||||
// The bus carries one message of at most its max_payload, and the client library refuses a larger one
|
||||
// before it leaves the process that sent it. Until this, a holder whose answer was too large wrote one
|
||||
// line to its own log and its caller waited out RequestTimeout and read "did not answer" — the
|
||||
// controller's `conditions` and `status` for hours on 2026-10-08, and the operator's channel with them.
|
||||
//
|
||||
// The controller pages such an answer (mesh-controller internal/link, pages.go): it answers an error
|
||||
// carrying `paged` — the call it holds the answer under, its size and its parts — and a caller asks the
|
||||
// same subject again once per part with PageHeader, each part answered raw with PartHeader. Asking the
|
||||
// same subject means no grant beyond the one the caller already had. A module's answer too large is
|
||||
// refused here, loudly, in a message that fits: its instances may be several, and a part asked again
|
||||
// could reach another.
|
||||
|
||||
const (
|
||||
// PageHeader asks for one part of a paged answer: `<call> <part>`, from zero.
|
||||
PageHeader = "Mesh-Page"
|
||||
// PartHeader is on a part: `<part>/<parts>`.
|
||||
PartHeader = "Mesh-Part"
|
||||
// partTries is how often one part is asked: during a handover a part can reach the controller that
|
||||
// did not hold it.
|
||||
partTries = 3
|
||||
)
|
||||
|
||||
// PartTimeout bounds the asking of one part.
|
||||
var PartTimeout = 10 * time.Second
|
||||
|
||||
// paged is what a too-large answer says instead of itself.
|
||||
type paged struct {
|
||||
Call string `json:"call"`
|
||||
Bytes int `json:"bytes"`
|
||||
Parts int `json:"parts"`
|
||||
}
|
||||
|
||||
func pagedIn(data []byte) (paged, bool) {
|
||||
if !bytes.Contains(data, []byte(`"paged"`)) {
|
||||
return paged{}, false
|
||||
}
|
||||
var r struct {
|
||||
Paged *paged `json:"paged"`
|
||||
}
|
||||
if json.Unmarshal(data, &r) != nil || r.Paged == nil || r.Paged.Call == "" || r.Paged.Parts < 1 {
|
||||
return paged{}, false
|
||||
}
|
||||
return *r.Paged, true
|
||||
}
|
||||
|
||||
// whole is a reply's whole answer: the reply itself, or every part of a paged one, joined. A part that
|
||||
// cannot be had is an error naming it, never a shorter answer.
|
||||
func (c *Conn) whole(subject string, data []byte) ([]byte, error) {
|
||||
p, ok := pagedIn(data)
|
||||
if !ok {
|
||||
return data, nil
|
||||
}
|
||||
out := make([]byte, 0, p.Bytes)
|
||||
for n := 0; n < p.Parts; n++ {
|
||||
var part []byte
|
||||
var err error
|
||||
for try := 0; try < partTries; try++ {
|
||||
if part, err = c.askPart(subject, p, n); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the answer was %d bytes, more than the bus carries in one message, paged as %s "+
|
||||
"in %d parts, and part %d could not be had: %w", p.Bytes, p.Call, p.Parts, n, err)
|
||||
}
|
||||
out = append(out, part...)
|
||||
}
|
||||
if len(out) != p.Bytes {
|
||||
return nil, fmt.Errorf("the answer paged as %s was %d bytes, and its parts joined are %d", p.Call, p.Bytes, len(out))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Conn) askPart(subject string, p paged, n int) ([]byte, error) {
|
||||
ask := nats.NewMsg(subject)
|
||||
ask.Header.Set(PageHeader, fmt.Sprintf("%s %d", p.Call, n))
|
||||
ask.Data = []byte(`{}`)
|
||||
reply, err := c.nc.RequestMsg(ask, PartTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if got := reply.Header.Get(PartHeader); got != fmt.Sprintf("%d/%d", n, p.Parts) {
|
||||
var r struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(reply.Data, &r) == nil && r.Error != "" {
|
||||
return nil, errors.New(r.Error)
|
||||
}
|
||||
return nil, fmt.Errorf("asked part %d/%d, answered %q", n, p.Parts, got)
|
||||
}
|
||||
return reply.Data, nil
|
||||
}
|
||||
|
||||
// fits is an answer the bus can carry: the answer itself, or — when it is larger than one message —
|
||||
// an error that says so, which is.
|
||||
func (c *Conn) fits(subject string, body []byte) []byte {
|
||||
limit := c.nc.MaxPayload()
|
||||
if limit <= 0 || int64(len(body)) <= limit {
|
||||
return body
|
||||
}
|
||||
c.Logf("[mesh-tools] the answer on %s is %d bytes, more than the bus carries in one message (%d): its caller "+
|
||||
"is told so instead", subject, len(body), limit)
|
||||
said, _ := wire.Marshal(reply{Node: c.node, Error: fmt.Sprintf("the answer is %d bytes, more than the bus "+
|
||||
"carries in one message (%d), so it was not sent: ask for less — a narrower question, a shorter window, "+
|
||||
"fewer lines", len(body), limit)})
|
||||
return said
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq issue 314: an answer larger than the bus carries in one message arrives whole when the
|
||||
// controller pages it, and a module's is said to be too large — neither is a 30-second silence.
|
||||
func TestAnAnswerLargerThanOneMessage(t *testing.T) {
|
||||
url := meshtest.URL(t)
|
||||
seat, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer seat.Close()
|
||||
c, err := bus.Connect(bus.Credential{URL: url, Module: "console", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
limit := int(seat.MaxPayload())
|
||||
|
||||
// The controller as it pages (mesh-controller internal/link, pages.go): the first answer an error
|
||||
// with `paged`, each part asked again on the same subject with the page header.
|
||||
large := strings.Repeat("a condition held past its bound; ", 3*limit/32)
|
||||
whole, _ := json.Marshal(map[string]any{"result": large})
|
||||
partSize := limit - 1024
|
||||
parts := (len(whole) + partSize - 1) / partSize
|
||||
sub, err := seat.Subscribe(bus.SeatToolSubject("mesh-controller", "conditions", "", ""), func(m *nats.Msg) {
|
||||
if page := m.Header.Get(bus.PageHeader); page != "" {
|
||||
var id string
|
||||
var n int
|
||||
fmt.Sscanf(page, "%s %d", &id, &n)
|
||||
end := min((n+1)*partSize, len(whole))
|
||||
part := nats.NewMsg(m.Reply)
|
||||
part.Header.Set(bus.PartHeader, fmt.Sprintf("%d/%d", n, parts))
|
||||
part.Data = whole[n*partSize : end]
|
||||
_ = m.RespondMsg(part)
|
||||
return
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"error": "too large for one message",
|
||||
"paged": map[string]any{"call": "call-1-1", "bytes": len(whole), "parts": parts}})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
_ = seat.Flush()
|
||||
|
||||
got, err := c.Ask("seat:mesh-controller.conditions", map[string]any{}, "")
|
||||
var said string
|
||||
if err != nil || json.Unmarshal(got.Result, &said) != nil || said != large {
|
||||
t.Fatalf("an answer of %d bytes in %d parts arrived as %d bytes: %v", len(whole), parts, len(got.Result), err)
|
||||
}
|
||||
|
||||
// A module's answer too large is refused here in words, at once.
|
||||
stop, err := c.HandleSubject("mesh.mod.journal.tool.lines", func(json.RawMessage) (any, error) { return large, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
asked := time.Now()
|
||||
_, err = c.Ask("journal.lines", map[string]any{}, "mesh.mod.journal.tool.lines")
|
||||
if err == nil || !strings.Contains(err.Error(), "more than the bus carries") {
|
||||
t.Fatalf("a module's answer too large for one message read as %v", err)
|
||||
}
|
||||
if took := time.Since(asked); took > 5*time.Second {
|
||||
t.Fatalf("it took %s to say", took)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A module's code asks its machine's provisioner client for its values at runtime (novox/hq ADR 0349 clause 14 as
|
||||
// ADR 0370 changes it, issue 685).
|
||||
//
|
||||
// While a module's code runs here, its asks reach the bus under this runtime's account, and the bus proves only
|
||||
// that account. So the runtime asks its own machine's client on ProvisionAskSubject — which the controller grants
|
||||
// this machine's tool runner alone to publish (ADR 0370 §3) — naming the module itself, from the bundle that asks
|
||||
// and never from the module's code or arguments (§4). The client answers that module's config values, for what its
|
||||
// manifest declares, and no secret: the vault hands a secret only to a caller the bus proves (§2).
|
||||
|
||||
// ProvisionAskSubject is where this machine's tool runner asks its provisioner client (mesh-controller
|
||||
// internal/broker ProvisionerAskSubject): `prov.ask.<node>`.
|
||||
func ProvisionAskSubject(node string) string { return "prov.ask." + node }
|
||||
|
||||
// ProvisionChangedSubjects is every module's word of what moved on a machine (mesh-controller internal/broker
|
||||
// ProvisionerChangedSubject): `prov.changed.<node>.<module>`, a module's name dots and all.
|
||||
func ProvisionChangedSubjects(node string) string { return "prov.changed." + node + ".>" }
|
||||
|
||||
// ProvisionTimeout bounds one ask of the client; a variable so a test need not wait.
|
||||
var ProvisionTimeout = 10 * time.Second
|
||||
|
||||
// ProvisionChanged is what the client says moved of one module: the kind, and the key, or every key of the kind when
|
||||
// empty. Never the value.
|
||||
type ProvisionChanged struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// ProvisionRefused is the client's refusal, in its own words: a key the module does not declare, a module not placed
|
||||
// on the machine, a secret, a kind it does not answer yet.
|
||||
type ProvisionRefused struct{ Reason string }
|
||||
|
||||
func (r *ProvisionRefused) Error() string { return r.Reason }
|
||||
|
||||
// AskProvisioner asks this machine's provisioner client for one value of module: kind `config` or `bound`, and the
|
||||
// key. The module is the caller's to name from the bundle it runs; this never takes it from what a bundle sent.
|
||||
func (c *Conn) AskProvisioner(module, kind, key string) (json.RawMessage, error) {
|
||||
if c.node == "" {
|
||||
return nil, errors.New("this runtime names no machine, so it has no provisioner client to ask")
|
||||
}
|
||||
body, err := json.Marshal(map[string]string{"module": module, "kind": kind, "key": key})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
msg, err := c.nc.Request(ProvisionAskSubject(c.node), body, ProvisionTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, fmt.Errorf("%s's provisioner client does not answer: it is not running there, or the bus does "+
|
||||
"not let %s ask it", c.node, c.cred.User)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("asking %s's provisioner client for %s's %s %s: %w", c.node, module, kind, key, err)
|
||||
}
|
||||
var answered struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &answered); err != nil {
|
||||
return nil, fmt.Errorf("%s's provisioner client answered something that is not its answer: %w", c.node, err)
|
||||
}
|
||||
if answered.Error != "" {
|
||||
return nil, &ProvisionRefused{Reason: answered.Error}
|
||||
}
|
||||
if len(answered.Value) == 0 {
|
||||
return nil, fmt.Errorf("%s's provisioner client answered no value and no reason", c.node)
|
||||
}
|
||||
return answered.Value, nil
|
||||
}
|
||||
|
||||
// OnProvisionChanged hands told every word of this machine's provisioner client of what moved, by module, until
|
||||
// stopped.
|
||||
func (c *Conn) OnProvisionChanged(told func(module string, changed ProvisionChanged)) (func(), error) {
|
||||
if c.node == "" {
|
||||
return nil, errors.New("this runtime names no machine, so it hears no provisioner client")
|
||||
}
|
||||
prefix := "prov.changed." + c.node + "."
|
||||
sub, err := c.nc.Subscribe(ProvisionChangedSubjects(c.node), func(m *nats.Msg) {
|
||||
var changed ProvisionChanged
|
||||
if json.Unmarshal(m.Data, &changed) != nil || changed.Kind == "" {
|
||||
return
|
||||
}
|
||||
told(strings.TrimPrefix(m.Subject, prefix), changed)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() { _ = sub.Unsubscribe() }, c.nc.Flush()
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// novox/hq issue 222: a subscription the bus refused is asked for again. A push sends the machine
|
||||
// that needs a grant and the bus's machine together; the runtime may subscribe the moment before the
|
||||
// bus reloads, and a refusal must not leave the module unreachable until some later membership.
|
||||
func TestARefusedSubscriptionIsAskedForAgain(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
c, err := Connect(Credential{URL: url, Module: "alpha", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
c.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
c.RetryAfter = []time.Duration{50 * time.Millisecond, 50 * time.Millisecond}
|
||||
|
||||
subject := "mesh.mod.alpha.tool.ping.anchor"
|
||||
stop, err := c.answerOn(subject, "", func(json.RawMessage) (any, error) { return "pong", nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := c.answering[answeringKey(subject, "")].sub
|
||||
|
||||
// What the bus says when the grant is not there yet.
|
||||
c.refused(fmt.Errorf("%w: Permissions Violation for Subscription to %q", nats.ErrPermissionViolation, subject))
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
c.mu.Lock()
|
||||
again := c.answering[answeringKey(subject, "")].sub
|
||||
c.mu.Unlock()
|
||||
if again != first {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the refused subscription was not asked for again")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
asker, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer asker.Close()
|
||||
reply, err := asker.Request(subject, []byte("{}"), 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("the subject asked for again does not answer: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(reply.Data), "pong") {
|
||||
t.Fatalf("the subject asked for again answered %s", reply.Data)
|
||||
}
|
||||
|
||||
// Past its attempts it is given up, and said.
|
||||
for i := 0; i < 3; i++ {
|
||||
c.refused(fmt.Errorf("%w: Permissions Violation for Subscription to %q", nats.ErrPermissionViolation, subject))
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
}
|
||||
mu.Lock()
|
||||
gaveUp := strings.Contains(strings.Join(said, "\n"), "still refuses "+subject)
|
||||
mu.Unlock()
|
||||
if !gaveUp {
|
||||
t.Errorf("never gave up on a subject that stays refused:\n%s", strings.Join(said, "\n"))
|
||||
}
|
||||
|
||||
// A subject the runtime stopped answering is not asked for again.
|
||||
stop()
|
||||
c.refused(fmt.Errorf("%w: Permissions Violation for Subscription to %q", nats.ErrPermissionViolation, subject))
|
||||
if _, still := c.answering[answeringKey(subject, "")]; still {
|
||||
t.Error("a stopped subject is still tracked")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A bundle's traffic on seats that name their caller or their kind (novox/hq ADR 0259 §3): submitting an
|
||||
// accept and saying an event under its own name or kind, taking a held seat's work from its worker, asking a
|
||||
// proof and answering one, and reading a record kept for it.
|
||||
//
|
||||
// **The runtime keeps each module to its own name and kind.** One account per machine carries every module
|
||||
// on it, so the bus enforces only the union; what one module may reach is its membership's `seat-traffic`,
|
||||
// and anything else is refused here, with the reason, before it is sent. A bundle reaches this only through
|
||||
// its own standard input and output: no tool call does.
|
||||
|
||||
// SeatTraffic is what a membership lists of a module's seat traffic (the controller's broker.SeatTraffic).
|
||||
type SeatTraffic struct {
|
||||
Publish []string `json:"publish,omitempty"`
|
||||
Subscribe []string `json:"subscribe,omitempty"`
|
||||
Answers []string `json:"answers,omitempty"`
|
||||
Workers []SeatWorker `json:"workers,omitempty"`
|
||||
Records []string `json:"records,omitempty"`
|
||||
// Kinds are the holders of every kinded bench the module uses or watches, with what each promises: the
|
||||
// controller's record of their claims (novox/hq ADR 0259 §5).
|
||||
Kinds []KindHeld `json:"kinds,omitempty"`
|
||||
}
|
||||
|
||||
// KindHeld is one kind of a kinded bench and who holds it.
|
||||
type KindHeld struct {
|
||||
Seat string `json:"seat"`
|
||||
Kind string `json:"kind"`
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
|
||||
// SeatKinds is the kinds the module's membership names, as the controller issued them.
|
||||
func (c *Conn) SeatKinds(module string) ([]KindHeld, error) {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]KindHeld{}, t.Kinds...), nil
|
||||
}
|
||||
|
||||
// SeatWorker is one work queue a holder takes from.
|
||||
type SeatWorker struct {
|
||||
Stream string `json:"Stream"`
|
||||
Consumer string `json:"Consumer"`
|
||||
Filter string `json:"Filter"`
|
||||
}
|
||||
|
||||
// ProofTimeout bounds how long a proof waits for its answer.
|
||||
var ProofTimeout = 15 * time.Second
|
||||
|
||||
// WorkHeartbeat is how often work a bundle is still doing is said to be in progress, well inside the
|
||||
// worker's ack wait.
|
||||
var WorkHeartbeat = 15 * time.Second
|
||||
|
||||
// WorkBackoff is how long a piece of work the bundle did not take waits before it is offered again: from
|
||||
// five seconds, doubling, to ten minutes.
|
||||
func WorkBackoff(delivered uint64) time.Duration {
|
||||
wait := 5 * time.Second
|
||||
for i := uint64(1); i < delivered && wait < 10*time.Minute; i++ {
|
||||
wait *= 2
|
||||
}
|
||||
return min(wait, 10*time.Minute)
|
||||
}
|
||||
|
||||
// SubjectMatches is a subject against a permission pattern, in the server's wildcards.
|
||||
func SubjectMatches(pattern, subject string) bool {
|
||||
p, s := strings.Split(pattern, "."), strings.Split(subject, ".")
|
||||
for i, part := range p {
|
||||
if part == ">" {
|
||||
return len(s) > i
|
||||
}
|
||||
if i >= len(s) || (part != "*" && part != s[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(s)
|
||||
}
|
||||
|
||||
func (c *Conn) trafficOf(module string) (*SeatTraffic, error) {
|
||||
m := c.Membership(module)
|
||||
if m == nil {
|
||||
return nil, fmt.Errorf("%s has no membership issued on %s yet, so it reaches no seat (novox/hq ADR 0259)",
|
||||
module, c.node)
|
||||
}
|
||||
if m.SeatTraffic == nil {
|
||||
return nil, fmt.Errorf("%s holds, uses and watches no seat that names its caller or its kind (novox/hq "+
|
||||
"ADR 0259): its manifest says so under `uses`, `claims` and `consumes`", module)
|
||||
}
|
||||
return m.SeatTraffic, nil
|
||||
}
|
||||
|
||||
func listed(patterns []string, subject string) bool {
|
||||
for _, p := range patterns {
|
||||
if SubjectMatches(p, subject) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// mayPublish refuses a subject the module's seat traffic does not list. A wildcard is never published.
|
||||
func (c *Conn) mayPublish(module, subject string) error {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.ContainsAny(subject, "*>") || !listed(t.Publish, subject) {
|
||||
return fmt.Errorf("%s may not publish %s: a module submits and says only under its own name or kind, "+
|
||||
"and its membership lists %s (novox/hq ADR 0259)", module, subject, strings.Join(t.Publish, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newID() string {
|
||||
var b [16]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// SeatPublish submits an accept or says an event on a seat, as the module, into the stream that keeps it,
|
||||
// awaited and de-duplicated by its id: the same id twice is one message. A proof is never published here.
|
||||
func (c *Conn) SeatPublish(module, subject string, body json.RawMessage, id string) (uint64, error) {
|
||||
seq, _, err := c.SeatPublishSaid(module, subject, body, id)
|
||||
return seq, err
|
||||
}
|
||||
|
||||
// SeatPublishSaid is SeatPublish, also saying whether the bus took it as a duplicate of one published before
|
||||
// under the same id: the same message, kept once.
|
||||
func (c *Conn) SeatPublishSaid(module, subject string, body json.RawMessage, id string) (uint64, bool, error) {
|
||||
if strings.Contains(subject, ".proof.") {
|
||||
return 0, false, fmt.Errorf("%s is a proof, asked and answered, never kept: ask it as one", subject)
|
||||
}
|
||||
if err := c.mayPublish(module, subject); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if len(body) == 0 || !json.Valid(body) {
|
||||
return 0, false, fmt.Errorf("what is said on %s is JSON", subject)
|
||||
}
|
||||
if id == "" {
|
||||
id = newID()
|
||||
}
|
||||
// **The publisher's name is part of the id the bus de-duplicates by** (security review 2026-10-08): ids
|
||||
// are the publisher's own, and one module could otherwise suppress another's warrant by publishing first
|
||||
// under the same id.
|
||||
id = module + "." + id
|
||||
msg := nats.NewMsg(subject)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", module)
|
||||
msg.Header.Set("x-node", c.node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = body
|
||||
ack, err := c.js.PublishMsg(msg, nats.MsgId(id))
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
return ack.Sequence, ack.Duplicate, nil
|
||||
}
|
||||
|
||||
// SeatProve asks a proof and answers its reply: core request and reply, on no stream.
|
||||
func (c *Conn) SeatProve(module, subject string, body json.RawMessage) (json.RawMessage, error) {
|
||||
if !strings.Contains(subject, ".proof.") {
|
||||
return nil, fmt.Errorf("%s is not a proof", subject)
|
||||
}
|
||||
if err := c.mayPublish(module, subject); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := c.nc.Request(subject, body, ProofTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, fmt.Errorf("nothing answers %s: the module watching the seat is not running", subject)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !json.Valid(reply.Data) {
|
||||
return nil, fmt.Errorf("the answer to %s is not JSON", subject)
|
||||
}
|
||||
return json.RawMessage(reply.Data), nil
|
||||
}
|
||||
|
||||
// SeatAnswer answers every proof on a subject the module's seat traffic lists among its answers, with what
|
||||
// answer returns. A returned error is answered as `{"error": "<words>"}`, never as silence.
|
||||
func (c *Conn) SeatAnswer(module, subject string, answer func(subject string, body json.RawMessage) (json.RawMessage, error)) (func(), error) {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allowed := false
|
||||
for _, a := range t.Answers {
|
||||
allowed = allowed || a == subject
|
||||
}
|
||||
if !allowed {
|
||||
return nil, fmt.Errorf("%s does not answer %s: its membership lists %s (novox/hq ADR 0259)", module,
|
||||
subject, strings.Join(t.Answers, ", "))
|
||||
}
|
||||
sub, err := c.nc.Subscribe(subject, func(msg *nats.Msg) {
|
||||
if msg.Reply == "" {
|
||||
return
|
||||
}
|
||||
body := json.RawMessage(msg.Data)
|
||||
if !json.Valid(body) {
|
||||
_ = msg.Respond([]byte(`{"error":"a proof is JSON"}`))
|
||||
return
|
||||
}
|
||||
out, err := answer(msg.Subject, body)
|
||||
if err != nil {
|
||||
b, _ := json.Marshal(map[string]string{"error": err.Error()})
|
||||
_ = msg.Respond(b)
|
||||
return
|
||||
}
|
||||
if len(out) == 0 {
|
||||
out = json.RawMessage(`{}`)
|
||||
}
|
||||
if err := msg.Respond(out); err != nil {
|
||||
c.Logf("[mesh-tools] %s could not answer the proof on %s: %v", module, msg.Subject, err)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.track(sub)
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { _ = sub.Unsubscribe() }) }, nil
|
||||
}
|
||||
|
||||
// Work is one message a holder took from a seat's work queue.
|
||||
type Work struct {
|
||||
Worker string `json:"worker"`
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
}
|
||||
|
||||
// SeatTake reads a worker the module's seat traffic lists and hands each message to deliver: acknowledged
|
||||
// when deliver returns nil, offered again after NakDelay when it returns an error, ended when it is not
|
||||
// JSON. The worker is the controller's to create; this binds to it by name and never makes one.
|
||||
func (c *Conn) SeatTake(module, consumer string, deliver func(Work) error) (func(), error) {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var w *SeatWorker
|
||||
for i := range t.Workers {
|
||||
if t.Workers[i].Consumer == consumer {
|
||||
w = &t.Workers[i]
|
||||
}
|
||||
}
|
||||
if w == nil {
|
||||
var names []string
|
||||
for _, x := range t.Workers {
|
||||
names = append(names, x.Consumer)
|
||||
}
|
||||
return nil, fmt.Errorf("%s takes no work from %s: its membership lists %s (novox/hq ADR 0259)", module,
|
||||
consumer, strings.Join(names, ", "))
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
bound, err := js.Consumer(ctx, w.Stream, w.Consumer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("binding %s's worker %s on %s: %w", module, w.Consumer, w.Stream, err)
|
||||
}
|
||||
reading, err := bound.Consume(func(msg jetstream.Msg) {
|
||||
if !json.Valid(msg.Data()) {
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
headers := map[string]string{}
|
||||
for k := range msg.Headers() {
|
||||
headers[k] = msg.Headers().Get(k)
|
||||
}
|
||||
// Kept alive while the bundle works on it, however long that takes: the ack wait is for a holder
|
||||
// that died, not one that is slow.
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
tick := time.NewTicker(WorkHeartbeat)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
_ = msg.InProgress()
|
||||
}
|
||||
}
|
||||
}()
|
||||
err := deliver(Work{Worker: w.Consumer, Subject: msg.Subject(), Body: json.RawMessage(msg.Data()), Headers: headers})
|
||||
close(done)
|
||||
if err != nil {
|
||||
// Offered again later and later, never given up on: a channel that is away keeps its work
|
||||
// (security and correctness reviews of 2026-10-08).
|
||||
delivered := uint64(1)
|
||||
if meta, merr := msg.Metadata(); merr == nil {
|
||||
delivered = meta.NumDelivered
|
||||
}
|
||||
_ = msg.NakWithDelay(WorkBackoff(delivered))
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
}, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) {
|
||||
// A worker deleted, or the bus refusing it, is said — never a holder that silently takes nothing.
|
||||
c.Logf("[mesh-tools] %s's worker %s: %v", module, w.Consumer, err)
|
||||
}))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s's worker %s: %w", module, w.Consumer, err)
|
||||
}
|
||||
var once sync.Once
|
||||
return func() { once.Do(reading.Stop) }, nil
|
||||
}
|
||||
|
||||
// SeatRecord reads one key of a record a seat keeps for the module, under its own name: the bucket's
|
||||
// current value of `<module>.<key>`, or nil when there is none.
|
||||
func (c *Conn) SeatRecord(module, bucket, key string) (*StateEntry, error) {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if bucket == "" {
|
||||
// The one record its membership lists, when it lists one: an asker reads the router's asks without
|
||||
// knowing the router's name.
|
||||
var named []string
|
||||
for _, r := range t.Records {
|
||||
if b, _, ok := strings.Cut(strings.TrimPrefix(r, "$JS.API.DIRECT.GET.KV_"), "."); ok {
|
||||
named = append(named, b)
|
||||
}
|
||||
}
|
||||
if len(named) != 1 {
|
||||
return nil, fmt.Errorf("%s reads %d records under its name; name the one meant", module, len(named))
|
||||
}
|
||||
bucket = named[0]
|
||||
}
|
||||
subject := "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + module + "." + key
|
||||
if !listed(t.Records, subject) {
|
||||
return nil, fmt.Errorf("%s reads no record of %s under its name: its membership lists %s (novox/hq ADR 0259)",
|
||||
module, bucket, strings.Join(t.Records, ", "))
|
||||
}
|
||||
reply, err := c.nc.Request(subject, nil, StateTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status := reply.Header.Get("Status"); status == "404" {
|
||||
return nil, nil
|
||||
} else if status != "" {
|
||||
return nil, fmt.Errorf("the record %s.%s could not be read: %s %s", module, key, status, reply.Header.Get("Description"))
|
||||
}
|
||||
if op := reply.Header.Get("KV-Operation"); op == "DEL" || op == "PURGE" {
|
||||
return nil, nil
|
||||
}
|
||||
var revision uint64
|
||||
fmt.Sscan(reply.Header.Get("Nats-Sequence"), &revision)
|
||||
return &StateEntry{Key: module + "." + key, Value: valueOf(reply.Data), Revision: revision}, nil
|
||||
}
|
||||
|
||||
// StateCreate writes one key only when it has no value, and answers its revision; a key that has one is
|
||||
// refused, naming it, so two writers racing to make it are one winner and one refusal.
|
||||
func (c *Conn) StateCreate(module, name, key string, value json.RawMessage) (uint64, error) {
|
||||
return c.stateWrite(module, name, key, value, func(ctx context.Context, kv jetstream.KeyValue) (uint64, error) {
|
||||
return kv.Create(ctx, key, value)
|
||||
})
|
||||
}
|
||||
|
||||
// StateUpdate writes one key only when its revision is still the one given, and answers the new revision;
|
||||
// a key changed meanwhile is refused, so of two writers that read one value only the first writes.
|
||||
func (c *Conn) StateUpdate(module, name, key string, value json.RawMessage, revision uint64) (uint64, error) {
|
||||
if revision == 0 {
|
||||
// Revision zero would be read by the server as "the key is new": an update names the value it read.
|
||||
return 0, fmt.Errorf("an update names the revision it read; zero is none — create the key instead")
|
||||
}
|
||||
return c.stateWrite(module, name, key, value, func(ctx context.Context, kv jetstream.KeyValue) (uint64, error) {
|
||||
return kv.Update(ctx, key, value, revision)
|
||||
})
|
||||
}
|
||||
|
||||
// ErrStateChanged is a compare-and-set that lost: the key was made or changed by another writer first.
|
||||
var ErrStateChanged = errors.New("the key was written by another writer first")
|
||||
|
||||
// changed is ErrStateChanged naming the key, with the code a bundle is answered it by.
|
||||
type changed struct{ key string }
|
||||
|
||||
func (e changed) Error() string { return e.key + ": " + ErrStateChanged.Error() }
|
||||
func (e changed) Is(target error) bool { return target == ErrStateChanged }
|
||||
func (e changed) ErrorCode() int { return -32010 }
|
||||
|
||||
func (c *Conn) stateWrite(module, name, key string, value json.RawMessage,
|
||||
write func(context.Context, jetstream.KeyValue) (uint64, error)) (uint64, error) {
|
||||
s, err := c.writable(module, name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := checkKey(key); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(value) == 0 || !json.Valid(value) {
|
||||
return 0, fmt.Errorf("a state value is JSON")
|
||||
}
|
||||
if field := credentialField(value); field != "" {
|
||||
return 0, fmt.Errorf("%s's %s.%s carries a field %q, which names a credential: no secret is kept in state "+
|
||||
"(novox/hq ADR 0201)", module, name, key, field)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := write(ctx, kv)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || isWrongSequence(err) {
|
||||
return 0, changed{name + "." + key}
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
// isWrongSequence is the server's refusal of an update whose revision is not the key's last.
|
||||
func isWrongSequence(err error) bool {
|
||||
var api *jetstream.APIError
|
||||
return errors.As(err, &api) && api.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// **A holder that pulls one piece of work when it has room** (novox/hq ADR 0525 §4, §6; issue 1887 A11). SeatTake
|
||||
// hands a bundle every piece a worker holds, one at a time and prefetched; a build module must instead take one ask
|
||||
// only when it has a free slot, from its workers in their order (delivery, then check, then recheck), and decide each
|
||||
// ask's fate when its step ends. SeatPull is one fetch of at most one message from a worker the module's membership
|
||||
// lists, never a prefetch; SeatSettle gives a pulled message its fate. A message the bundle never settles (its process
|
||||
// died) is delivered again by the worker's own acknowledgement wait.
|
||||
|
||||
// MaxPullWait bounds how long one pull waits for a message: under the bundle's 30 s wait for its answer.
|
||||
var MaxPullWait = 25 * time.Second
|
||||
|
||||
// Pulled is one message a module pulled from a seat's worker, named by an id the runtime gave it.
|
||||
type Pulled struct {
|
||||
ID string `json:"id"`
|
||||
Worker string `json:"worker"`
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
Delivered uint64 `json:"delivered"`
|
||||
}
|
||||
|
||||
// pulls are the messages modules pulled and have not settled, by id; each kept with the module it was pulled for, so
|
||||
// one module never settles another's.
|
||||
type pulls struct {
|
||||
mu sync.Mutex
|
||||
held map[string]pulledMsg
|
||||
cons map[string]jetstream.Consumer // by module and consumer
|
||||
}
|
||||
|
||||
type pulledMsg struct {
|
||||
module string
|
||||
msg jetstream.Msg
|
||||
}
|
||||
|
||||
var seatPulls = &pulls{held: map[string]pulledMsg{}, cons: map[string]jetstream.Consumer{}}
|
||||
|
||||
func pullID() string {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// SeatPull fetches at most one message from consumer for module, waiting at most wait (none when 0): nil when none
|
||||
// came. The consumer must be one the module's membership lists as its worker (novox/hq ADR 0259); the runtime binds to
|
||||
// it by name and never makes one.
|
||||
func (c *Conn) SeatPull(module, consumer string, wait time.Duration) (*Pulled, error) {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var w *SeatWorker
|
||||
var names []string
|
||||
for i := range t.Workers {
|
||||
names = append(names, t.Workers[i].Consumer)
|
||||
if t.Workers[i].Consumer == consumer {
|
||||
w = &t.Workers[i]
|
||||
}
|
||||
}
|
||||
if w == nil {
|
||||
return nil, fmt.Errorf("%s pulls no work from %s: its membership lists %s (novox/hq ADR 0259)", module,
|
||||
consumer, strings.Join(names, ", "))
|
||||
}
|
||||
if wait > MaxPullWait {
|
||||
wait = MaxPullWait
|
||||
}
|
||||
bound, err := c.pullConsumer(module, *w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var batch jetstream.MessageBatch
|
||||
if wait <= 0 {
|
||||
batch, err = bound.FetchNoWait(1)
|
||||
} else {
|
||||
batch, err = bound.Fetch(1, jetstream.FetchMaxWait(wait))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pulling %s's worker %s: %w", module, w.Consumer, err)
|
||||
}
|
||||
msg, ok := <-batch.Messages()
|
||||
if !ok {
|
||||
if err := batch.Error(); err != nil {
|
||||
return nil, fmt.Errorf("pulling %s's worker %s: %w", module, w.Consumer, err)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if !json.Valid(msg.Data()) {
|
||||
_ = msg.Term()
|
||||
return nil, nil
|
||||
}
|
||||
headers := map[string]string{}
|
||||
for k := range msg.Headers() {
|
||||
headers[k] = msg.Headers().Get(k)
|
||||
}
|
||||
delivered := uint64(1)
|
||||
if meta, merr := msg.Metadata(); merr == nil && meta != nil {
|
||||
delivered = meta.NumDelivered
|
||||
}
|
||||
p := &Pulled{ID: pullID(), Worker: w.Consumer, Subject: msg.Subject(), Body: json.RawMessage(msg.Data()),
|
||||
Headers: headers, Delivered: delivered}
|
||||
seatPulls.mu.Lock()
|
||||
seatPulls.held[p.ID] = pulledMsg{module: module, msg: msg}
|
||||
seatPulls.mu.Unlock()
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (c *Conn) pullConsumer(module string, w SeatWorker) (jetstream.Consumer, error) {
|
||||
key := module + " " + w.Consumer
|
||||
seatPulls.mu.Lock()
|
||||
bound, ok := seatPulls.cons[key]
|
||||
seatPulls.mu.Unlock()
|
||||
if ok {
|
||||
return bound, nil
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
bound, err = js.Consumer(ctx, w.Stream, w.Consumer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("binding %s's worker %s on %s: %w", module, w.Consumer, w.Stream, err)
|
||||
}
|
||||
seatPulls.mu.Lock()
|
||||
seatPulls.cons[key] = bound
|
||||
seatPulls.mu.Unlock()
|
||||
return bound, nil
|
||||
}
|
||||
|
||||
// The fates SeatSettle gives a pulled message.
|
||||
const (
|
||||
SettleAck = "ack" // done: never delivered again
|
||||
SettleNak = "nak" // handed back, to be delivered again at once
|
||||
SettleTerm = "term" // dropped
|
||||
SettleProgress = "progress" // still being worked: its acknowledgement wait starts again
|
||||
)
|
||||
|
||||
// SeatSettle gives the message module pulled under id its fate. A message pulled for another module, or settled
|
||||
// already, is refused.
|
||||
func (c *Conn) SeatSettle(module, id, fate string) error {
|
||||
seatPulls.mu.Lock()
|
||||
p, ok := seatPulls.held[id]
|
||||
if ok && p.module != module {
|
||||
ok = false
|
||||
}
|
||||
if ok && fate != SettleProgress {
|
||||
delete(seatPulls.held, id)
|
||||
}
|
||||
seatPulls.mu.Unlock()
|
||||
if !ok {
|
||||
return fmt.Errorf("%s holds no pulled message %s", module, id)
|
||||
}
|
||||
switch fate {
|
||||
case SettleAck:
|
||||
return p.msg.Ack()
|
||||
case SettleNak:
|
||||
return p.msg.Nak()
|
||||
case SettleTerm:
|
||||
return p.msg.Term()
|
||||
case SettleProgress:
|
||||
return p.msg.InProgress()
|
||||
}
|
||||
seatPulls.mu.Lock()
|
||||
seatPulls.held[id] = p // an unknown fate settles nothing
|
||||
seatPulls.mu.Unlock()
|
||||
return fmt.Errorf("a pulled message's fate is ack, nak, term or progress, not %q", fate)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// 1887/A11: a holder pulls one message at a time, only from a worker its membership lists, never a prefetch; it settles
|
||||
// only what it pulled, and a message handed back comes again.
|
||||
func TestAHolderPullsOneMessageWhenItHasRoom(t *testing.T) {
|
||||
_, conn, js := seatMesh(t)
|
||||
for _, id := range []string{"a1", "a2"} {
|
||||
if _, err := conn.SeatPublish("mesh-delivery", "mesh.seat.operator-channel.accept.ask.mesh-delivery",
|
||||
json.RawMessage(`{"id":"`+id+`"}`), id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
first, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 0)
|
||||
if err != nil || first == nil || string(first.Body) != `{"id":"a1"}` || first.Delivered != 1 {
|
||||
t.Fatalf("the first pull = %+v, %v", first, err)
|
||||
}
|
||||
// One pulled, one still waiting on the stream: nothing was fetched ahead.
|
||||
info, err := js.ConsumerInfo("SEAT_OPERATOR_CHANNEL", "SEAT_OPERATOR_CHANNEL_worker")
|
||||
if err != nil || info.NumPending != 1 || info.NumAckPending != 1 {
|
||||
t.Fatalf("after one pull: pending %d, in hand %d (%v); want 1 and 1", info.NumPending, info.NumAckPending, err)
|
||||
}
|
||||
second, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 0)
|
||||
if err != nil || second == nil || string(second.Body) != `{"id":"a2"}` {
|
||||
t.Fatalf("the second pull = %+v, %v", second, err)
|
||||
}
|
||||
if none, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 0); err != nil || none != nil {
|
||||
t.Fatalf("an empty worker answered %+v, %v", none, err)
|
||||
}
|
||||
began := time.Now()
|
||||
if none, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 200*time.Millisecond); err != nil || none != nil || time.Since(began) < 150*time.Millisecond {
|
||||
t.Fatalf("a waiting pull answered %+v, %v after %v", none, err, time.Since(began))
|
||||
}
|
||||
if err := conn.SeatSettle("telegram", first.ID, bus.SettleAck); err == nil {
|
||||
t.Error("a module settled another's message")
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", first.ID, bus.SettleProgress); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", first.ID, bus.SettleAck); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", first.ID, bus.SettleAck); err == nil {
|
||||
t.Error("a message was settled twice")
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", second.ID, bus.SettleNak); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", time.Second)
|
||||
if err != nil || again == nil || string(again.Body) != `{"id":"a2"}` || again.Delivered != 2 {
|
||||
t.Fatalf("the message handed back = %+v, %v", again, err)
|
||||
}
|
||||
if _, err := conn.SeatPull("telegram", "SEAT_OPERATOR_CHANNEL_worker", 0); err == nil || !strings.Contains(err.Error(), "membership lists") {
|
||||
t.Errorf("a channel pulled the router's work: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// The seats of novox/hq ADR 0259 §3 as the controller issues them: an asker, the router, and a channel of
|
||||
// kind telegram, each with the seat traffic its membership lists.
|
||||
func seatMesh(t *testing.T) (*mt.Mesh, *bus.Conn, nats.JetStreamContext) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, _ := nc.JetStream()
|
||||
for _, s := range []*nats.StreamConfig{
|
||||
{Name: "SEAT_OPERATOR_CHANNEL", Subjects: []string{"mesh.seat.operator-channel.accept.>"}, Retention: nats.WorkQueuePolicy},
|
||||
{Name: "SEAT_CHANNEL", Subjects: []string{"mesh.seat.channel.accept.>"}, Retention: nats.WorkQueuePolicy},
|
||||
{Name: "SEAT_EVENTS", Subjects: []string{"mesh.seat.*.event.>"}},
|
||||
} {
|
||||
if _, err := js.AddStream(s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for stream, c := range map[string]*nats.ConsumerConfig{
|
||||
"SEAT_OPERATOR_CHANNEL": {Durable: "SEAT_OPERATOR_CHANNEL_worker", AckPolicy: nats.AckExplicitPolicy,
|
||||
FilterSubject: "mesh.seat.operator-channel.accept.>", AckWait: 2 * time.Second},
|
||||
"SEAT_CHANNEL": {Durable: "SEAT_CHANNEL_TELEGRAM_worker", AckPolicy: nats.AckExplicitPolicy,
|
||||
FilterSubject: "mesh.seat.channel.accept.*.telegram", AckWait: 2 * time.Second},
|
||||
} {
|
||||
if _, err := js.AddConsumer(stream, c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "messenger_asks"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asker := mt.MembershipOf("mesh-delivery", "anchor", false, nil)
|
||||
asker.SeatTraffic = &bus.SeatTraffic{
|
||||
Publish: []string{"mesh.seat.operator-channel.accept.ask.mesh-delivery"},
|
||||
Subscribe: []string{"mesh.seat.operator-channel.event.decided.mesh-delivery"},
|
||||
Records: []string{"$JS.API.DIRECT.GET.KV_messenger_asks.$KV.messenger_asks.mesh-delivery.>"},
|
||||
}
|
||||
router := mt.MembershipOf("messenger", "anchor", false, nil)
|
||||
router.SeatTraffic = &bus.SeatTraffic{
|
||||
Publish: []string{"mesh.seat.operator-channel.event.decided.*", "mesh.seat.channel.accept.show.*"},
|
||||
Answers: []string{"mesh.seat.intake.proof.code.*"},
|
||||
Workers: []bus.SeatWorker{{Stream: "SEAT_OPERATOR_CHANNEL", Consumer: "SEAT_OPERATOR_CHANNEL_worker",
|
||||
Filter: "mesh.seat.operator-channel.accept.>"}},
|
||||
}
|
||||
router.State = []bus.StateIssued{{Name: "asks", Bucket: "messenger_asks", Writes: true}}
|
||||
telegram := mt.MembershipOf("telegram", "anchor", false, nil)
|
||||
telegram.SeatTraffic = &bus.SeatTraffic{
|
||||
Publish: []string{"mesh.seat.intake.event.choice.telegram", "mesh.seat.intake.proof.code.telegram"},
|
||||
Workers: []bus.SeatWorker{{Stream: "SEAT_CHANNEL", Consumer: "SEAT_CHANNEL_TELEGRAM_worker",
|
||||
Filter: "mesh.seat.channel.accept.*.telegram"}},
|
||||
}
|
||||
for _, m := range []bus.Membership{asker, router, telegram} {
|
||||
mesh.Issue(t, m)
|
||||
}
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
for _, m := range []string{"mesh-delivery", "messenger", "telegram"} {
|
||||
conn.Follow(m)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
for _, m := range []string{"mesh-delivery", "messenger", "telegram"} {
|
||||
if conn.Membership(m) == nil {
|
||||
return errors.New(m + " not issued yet")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return mesh, conn, js
|
||||
}
|
||||
|
||||
func TestAModuleSubmitsAndSaysOnlyUnderItsOwnNameAndKind(t *testing.T) {
|
||||
_, conn, _ := seatMesh(t)
|
||||
body := json.RawMessage(`{"id":"a1"}`)
|
||||
if _, err := conn.SeatPublish("mesh-delivery", "mesh.seat.operator-channel.accept.ask.mesh-delivery", body, "a1"); err != nil {
|
||||
t.Fatalf("an asker could not ask under its own name: %v", err)
|
||||
}
|
||||
for module, subject := range map[string]string{
|
||||
"mesh-delivery": "mesh.seat.operator-channel.accept.ask.mesh-controller",
|
||||
"telegram": "mesh.seat.intake.event.choice.desktop",
|
||||
"messenger": "mesh.seat.intake.event.choice.telegram",
|
||||
} {
|
||||
if _, err := conn.SeatPublish(module, subject, body, ""); err == nil || !strings.Contains(err.Error(), "may not publish") {
|
||||
t.Errorf("%s published %s: %v", module, subject, err)
|
||||
}
|
||||
}
|
||||
if _, err := conn.SeatPublish("messenger", "mesh.seat.operator-channel.event.decided.*", body, ""); err == nil {
|
||||
t.Error("a wildcard was published")
|
||||
}
|
||||
if _, err := conn.SeatPublish("telegram", "mesh.seat.intake.proof.code.telegram", body, ""); err == nil ||
|
||||
!strings.Contains(err.Error(), "never kept") {
|
||||
t.Errorf("a proof was published onto a stream: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheHolderTakesItsWorkOnceAndAFailureComesBack(t *testing.T) {
|
||||
_, conn, _ := seatMesh(t)
|
||||
if _, err := conn.SeatPublish("mesh-delivery", "mesh.seat.operator-channel.accept.ask.mesh-delivery",
|
||||
json.RawMessage(`{"id":"a1"}`), "a1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The same id again is one ask: a retry after a lost answer does not ask twice.
|
||||
if _, err := conn.SeatPublish("mesh-delivery", "mesh.seat.operator-channel.accept.ask.mesh-delivery",
|
||||
json.RawMessage(`{"id":"a1"}`), "a1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var got []string
|
||||
fail := true
|
||||
stop, err := conn.SeatTake("messenger", "SEAT_OPERATOR_CHANNEL_worker", func(w bus.Work) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
got = append(got, w.Subject+" from "+w.Headers["x-source"])
|
||||
if fail {
|
||||
fail = false
|
||||
return errors.New("not recorded")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for {
|
||||
mu.Lock()
|
||||
n := len(got)
|
||||
mu.Unlock()
|
||||
if n >= 2 || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(got) != 2 || got[0] != "mesh.seat.operator-channel.accept.ask.mesh-delivery from mesh-delivery" || got[0] != got[1] {
|
||||
t.Fatalf("want the one ask, offered again after the failure: %v", got)
|
||||
}
|
||||
if _, err := conn.SeatTake("telegram", "SEAT_OPERATOR_CHANNEL_worker", func(bus.Work) error { return nil }); err == nil {
|
||||
t.Error("a channel took the router's work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProofIsAskedAndAnsweredAndNeverKept(t *testing.T) {
|
||||
_, conn, js := seatMesh(t)
|
||||
stop, err := conn.SeatAnswer("messenger", "mesh.seat.intake.proof.code.*", func(subject string, body json.RawMessage) (json.RawMessage, error) {
|
||||
if !strings.HasSuffix(subject, ".telegram") {
|
||||
return nil, errors.New("asked by another kind")
|
||||
}
|
||||
return json.RawMessage(`{"linked":true}`), nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
var answer json.RawMessage
|
||||
mt.Until(t, func() error {
|
||||
answer, err = conn.SeatProve("telegram", "mesh.seat.intake.proof.code.telegram", json.RawMessage(`{"code":"123456"}`))
|
||||
return err
|
||||
})
|
||||
if string(answer) != `{"linked":true}` {
|
||||
t.Fatalf("the proof was answered %s", answer)
|
||||
}
|
||||
if _, err := conn.SeatProve("telegram", "mesh.seat.intake.proof.code.desktop", json.RawMessage(`{}`)); err == nil {
|
||||
t.Error("a channel asked a proof as another kind")
|
||||
}
|
||||
if _, err := conn.SeatAnswer("telegram", "mesh.seat.intake.proof.code.*", nil); err == nil {
|
||||
t.Error("a channel answers proofs")
|
||||
}
|
||||
for name := range js.StreamNames() {
|
||||
info, _ := js.StreamInfo(name)
|
||||
for _, s := range info.Config.Subjects {
|
||||
if bus.SubjectMatches(s, "mesh.seat.intake.proof.code.telegram") {
|
||||
t.Errorf("%s keeps proofs", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAskerReadsItsOwnRecordAndNoOther(t *testing.T) {
|
||||
_, conn, js := seatMesh(t)
|
||||
kv, _ := js.KeyValue("messenger_asks")
|
||||
if _, err := kv.Put("mesh-delivery.a1", []byte(`{"state":"open"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put("mesh-controller.a1", []byte(`{"state":"open"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := conn.SeatRecord("mesh-delivery", "messenger_asks", "a1")
|
||||
if err != nil || got == nil || string(got.Value) != `{"state":"open"}` {
|
||||
t.Fatalf("the asker could not read its own ask: %v %v", got, err)
|
||||
}
|
||||
if missing, err := conn.SeatRecord("mesh-delivery", "messenger_asks", "a2"); err != nil || missing != nil {
|
||||
t.Fatalf("a missing ask read as %v, %v", missing, err)
|
||||
}
|
||||
if _, err := conn.SeatRecord("telegram", "messenger_asks", "a1"); err == nil {
|
||||
t.Error("a module read records it is not given")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfTwoAnswersToOneAskOnlyTheFirstIsWritten(t *testing.T) {
|
||||
_, conn, _ := seatMesh(t)
|
||||
rev, err := conn.StateCreate("messenger", "asks", "mesh-delivery.a1", json.RawMessage(`{"state":"open"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := conn.StateCreate("messenger", "asks", "mesh-delivery.a1", json.RawMessage(`{"state":"open"}`)); !errors.Is(err, bus.ErrStateChanged) {
|
||||
t.Errorf("a second create stood: %v", err)
|
||||
}
|
||||
if _, err := conn.StateUpdate("messenger", "asks", "mesh-delivery.a1", json.RawMessage(`{"state":"answered"}`), rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := conn.StateUpdate("messenger", "asks", "mesh-delivery.a1", json.RawMessage(`{"state":"answered"}`), rev); !errors.Is(err, bus.ErrStateChanged) {
|
||||
t.Errorf("a second answer from the same revision stood: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's state on the bus (novox/hq ADR 0201): key-value buckets the controller creates from what
|
||||
// the module declared, and issues to each assignment in its membership by the name the module uses —
|
||||
// its own state by the local name, another module's as `<module>.<name>`.
|
||||
//
|
||||
// **The runtime keeps each module to its own buckets.** One account per machine carries every module
|
||||
// on it, so the bus enforces only the union; and a write the bus refuses reaches the writer as a
|
||||
// timeout, not a refusal (measured, novox/hq research 024). So what a module may reach is decided here,
|
||||
// from its membership, and refused with the reason before anything is sent.
|
||||
|
||||
// StateIssued is one bucket an assignment may reach (ADR 0201).
|
||||
type StateIssued struct {
|
||||
Name string `json:"name"`
|
||||
Bucket string `json:"bucket"`
|
||||
Writes bool `json:"writes,omitempty"`
|
||||
}
|
||||
|
||||
// StateEntry is one key's current value.
|
||||
type StateEntry struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
|
||||
// StateChange is one change a watch delivers: a key put or deleted.
|
||||
type StateChange struct {
|
||||
State string `json:"state"`
|
||||
Key string `json:"key"`
|
||||
Op string `json:"op"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
Revision uint64 `json:"revision"`
|
||||
// Current is true for a value that was there when the watch began, false for a change since.
|
||||
Current bool `json:"current"`
|
||||
}
|
||||
|
||||
// StateTimeout bounds one state operation on the bus.
|
||||
var StateTimeout = 10 * time.Second
|
||||
|
||||
// stateKey is a key the bus can hold: letters, digits and `-/_=.`, no leading or trailing dot. A
|
||||
// module's convention for naming a machine in a key (`all.<server>`, `<machine>.<server>`) fits it.
|
||||
var stateKey = regexp.MustCompile(`^[-/_=a-zA-Z0-9]+(\.[-/_=a-zA-Z0-9]+)*$`)
|
||||
|
||||
// issuedState is the bucket a module may reach by a name, and whether it may write it — or the reason
|
||||
// it may not reach it at all.
|
||||
func (c *Conn) issuedState(module, name string) (StateIssued, error) {
|
||||
m := c.Membership(module)
|
||||
if m == nil {
|
||||
return StateIssued{}, fmt.Errorf("%s has no membership issued on %s yet, so no state of it is reachable "+
|
||||
"until the mesh issues one (novox/hq ADR 0201)", module, c.node)
|
||||
}
|
||||
var names []string
|
||||
for _, s := range m.State {
|
||||
if s.Name == name {
|
||||
return s, nil
|
||||
}
|
||||
names = append(names, s.Name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
issued := "none"
|
||||
if len(names) > 0 {
|
||||
issued = strings.Join(names, ", ")
|
||||
}
|
||||
return StateIssued{}, fmt.Errorf("%s keeps and reads no state called %q: it declares what it keeps under "+
|
||||
"`state` and what it reads under `reads` as <module>.<name>, and was issued: %s (novox/hq ADR 0201)",
|
||||
module, name, issued)
|
||||
}
|
||||
|
||||
func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, error) {
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, s.Bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the controller "+
|
||||
"creates it from the catalogue on its next raise", s.Name)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
func checkKey(key string) error {
|
||||
if !stateKey.MatchString(key) {
|
||||
return fmt.Errorf("%q is not a key the bus can hold: letters, digits and -/_=, in dot-separated "+
|
||||
"names", key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StateGet is one key's current value, or nil when it has none.
|
||||
func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) {
|
||||
s, err := c.issuedState(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &StateEntry{Key: e.Key(), Value: valueOf(e.Value()), Revision: e.Revision()}, nil
|
||||
}
|
||||
|
||||
// StateKept is one value a bucket still keeps for a key: a put, or a delete (a purge reads as one).
|
||||
type StateKept struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Op string `json:"op"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// StateHistory is every value the bucket still keeps for one key, oldest first, or none (novox/hq ADR
|
||||
// 0307). It is a read like StateGet: served to the owner and to every reader, and writes nothing.
|
||||
func (c *Conn) StateHistory(module, name, key string) ([]StateKept, error) {
|
||||
s, err := c.issuedState(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := kv.History(ctx, key)
|
||||
kept := []StateKept{}
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return kept, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
k := StateKept{Key: e.Key(), Revision: e.Revision(), Op: "delete", At: e.Created().UTC()}
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
k.Op = "put"
|
||||
k.Value = valueOf(e.Value())
|
||||
}
|
||||
kept = append(kept, k)
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// StatePut writes one key, as the module, where the module keeps the state. It answers the revision.
|
||||
func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64, error) {
|
||||
s, err := c.writable(module, name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := checkKey(key); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(value) == 0 || !json.Valid(value) {
|
||||
return 0, fmt.Errorf("a state value is JSON")
|
||||
}
|
||||
if field := credentialField(value); field != "" {
|
||||
return 0, fmt.Errorf("%s's %s.%s carries a field %q, which names a credential: no secret is kept in state, "+
|
||||
"sealed or not — a bucket is a stream, and a machine joining a year later reads it whole. Name the "+
|
||||
"secret and fetch it on request/reply (novox/hq ADR 0201, design 32 §10)", module, name, key, field)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return kv.Put(ctx, key, value)
|
||||
}
|
||||
|
||||
// StateDelete removes one key, as the module, where the module keeps the state. A key that was not
|
||||
// there is not an error: what is asked for is that it is gone.
|
||||
func (c *Conn) StateDelete(module, name, key string) error {
|
||||
s, err := c.writable(module, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkKey(key); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return kv.Delete(ctx, key)
|
||||
}
|
||||
|
||||
// StateKeys is every key with a current value, sorted.
|
||||
func (c *Conn) StateKeys(module, name string) ([]string, error) {
|
||||
s, err := c.issuedState(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = lister.Stop() }()
|
||||
keys := []string{}
|
||||
for k := range lister.Keys() {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (c *Conn) writable(module, name string) (StateIssued, error) {
|
||||
s, err := c.issuedState(module, name)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if !s.Writes {
|
||||
owner := name
|
||||
if dot := strings.LastIndex(name, "."); dot > 0 {
|
||||
owner = name[:dot]
|
||||
}
|
||||
return s, fmt.Errorf("%s reads %s and does not keep it: only %s's own instances write it (novox/hq ADR 0201)",
|
||||
module, name, owner)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// StateWatch hands deliver the current value of every key matching the pattern — none that is
|
||||
// deleted — and then every change, in order (ADR 0201). It returns once the current values are
|
||||
// delivered. **StateTimeout bounds the time without progress, not the whole set** (novox/hq issue
|
||||
// 1726): each current value is a round trip to the bundle, so a state of a few hundred keys outlasts
|
||||
// any fixed bound for all of them; the watch then failed at every attempt, the bundle dropped its
|
||||
// handler, and the newest keys, handed over last, were never taken (the checker's owed, whose newest
|
||||
// verdicts never reached mesh/check). It fails only when no current value arrives, or one is not
|
||||
// taken, within StateTimeout; deliver is called from one goroutine, one change at a time, and an error from it is said
|
||||
// and the watch goes on: state is not a queue, and the next change, or the next start, reads it again.
|
||||
// The pattern is a key, with `*` for one name and `**` for the rest; empty is every key.
|
||||
func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange) error) (stop func(), err error) {
|
||||
s, err := c.issuedState(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter := ">"
|
||||
if pattern != "" {
|
||||
parts := strings.Split(pattern, ".")
|
||||
for i, p := range parts {
|
||||
if p == "**" {
|
||||
if i != len(parts)-1 {
|
||||
return nil, fmt.Errorf("%q: `**` stands for the rest of a key, so it comes last", pattern)
|
||||
}
|
||||
parts[i] = ">"
|
||||
continue
|
||||
}
|
||||
if p != "*" && !stateKey.MatchString(p) {
|
||||
return nil, fmt.Errorf("%q is not a key pattern: names, `*` for one and `**` for the rest", pattern)
|
||||
}
|
||||
}
|
||||
filter = strings.Join(parts, ".")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
kv, err := c.bucket(ctx, s)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
watching, stopWatching := context.WithCancel(context.Background())
|
||||
w, err := kv.Watch(watching, filter)
|
||||
if err != nil {
|
||||
stopWatching()
|
||||
return nil, err
|
||||
}
|
||||
var once sync.Once
|
||||
stop = func() {
|
||||
once.Do(func() {
|
||||
_ = w.Stop()
|
||||
stopWatching()
|
||||
})
|
||||
}
|
||||
current := make(chan struct{})
|
||||
// handed says a current value was handed over or passed by: progress, which restarts the bound.
|
||||
handed := make(chan struct{}, 1)
|
||||
progress := func() {
|
||||
select {
|
||||
case handed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
initial := true
|
||||
for e := range w.Updates() {
|
||||
if e == nil {
|
||||
// The end of what was there when the watch began.
|
||||
if initial {
|
||||
initial = false
|
||||
close(current)
|
||||
}
|
||||
continue
|
||||
}
|
||||
change := StateChange{State: name, Key: e.Key(), Revision: e.Revision(), Current: initial}
|
||||
switch e.Operation() {
|
||||
case jetstream.KeyValuePut:
|
||||
change.Op = "put"
|
||||
change.Value = valueOf(e.Value())
|
||||
default:
|
||||
// A deletion among the current values is a key that is not there: not handed over
|
||||
// (measured, research 024 — the server sends its marker among the initial values).
|
||||
if initial {
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
change.Op = "delete"
|
||||
}
|
||||
if err := deliver(change); err != nil {
|
||||
c.Logf("[mesh-tools] %s did not take %s %s of %s: %v; its next change, or its next start, reads it again",
|
||||
module, change.Op, change.Key, name, err)
|
||||
}
|
||||
if initial {
|
||||
progress()
|
||||
}
|
||||
}
|
||||
if initial {
|
||||
close(current)
|
||||
}
|
||||
}()
|
||||
bound := time.NewTimer(StateTimeout)
|
||||
defer bound.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-current:
|
||||
return stop, nil
|
||||
case <-handed:
|
||||
if !bound.Stop() {
|
||||
select {
|
||||
case <-bound.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
bound.Reset(StateTimeout)
|
||||
case <-bound.C:
|
||||
stop()
|
||||
return nil, fmt.Errorf("the current values of %s did not arrive in %s: none was handed over in that time", name, StateTimeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func valueOf(raw []byte) json.RawMessage {
|
||||
if len(raw) == 0 || !json.Valid(raw) {
|
||||
b, _ := json.Marshal(string(raw))
|
||||
return b
|
||||
}
|
||||
return json.RawMessage(raw)
|
||||
}
|
||||
|
||||
// credentialEndings are the ends of a field name that say its value is a credential. A guard against
|
||||
// the ordinary mistake, not a determined one: a sealed value is plain text to anything inspecting it,
|
||||
// so the rule is checked where it can be and said to be partial (ADR 0201).
|
||||
var credentialEndings = []string{"password", "passwd", "secret", "token", "credential", "credentials",
|
||||
"authorization", "apikey", "privatekey", "accesskey", "cookie"}
|
||||
|
||||
// credentialField is the first field anywhere in a JSON value whose name says it is a credential, or "". The one shape a
|
||||
// credential-shaped name may hold is a provider reference, an object of exactly `node` and `module`, both strings: a
|
||||
// placement's binding of the provision "secret" to the vault (issue 1126) names who provides it, never its value. Any
|
||||
// other shape under such a name is refused.
|
||||
func credentialField(value json.RawMessage) string {
|
||||
var v any
|
||||
if json.Unmarshal(value, &v) != nil {
|
||||
return ""
|
||||
}
|
||||
var walk func(any) string
|
||||
walk = func(v any) string {
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(t))
|
||||
for k := range t {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
norm := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(k))
|
||||
for _, end := range credentialEndings {
|
||||
if strings.HasSuffix(norm, end) && !providerReference(t[k]) {
|
||||
return k
|
||||
}
|
||||
}
|
||||
if found := walk(t[k]); found != "" {
|
||||
return found
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, x := range t {
|
||||
if found := walk(x); found != "" {
|
||||
return found
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return walk(v)
|
||||
}
|
||||
|
||||
// providerReference says whether v is exactly {"node": <name>, "module": <name>}, each a name as mesh-sdk names them.
|
||||
func providerReference(v any) bool {
|
||||
m, ok := v.(map[string]any)
|
||||
if !ok || len(m) != 2 {
|
||||
return false
|
||||
}
|
||||
node, okN := m["node"].(string)
|
||||
module, okM := m["module"].(string)
|
||||
return okN && okM && referenceName.MatchString(node) && referenceName.MatchString(module)
|
||||
}
|
||||
|
||||
// referenceName is a node's or a module's name as mesh-sdk names them: lower case, digits and dashes.
|
||||
var referenceName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
@@ -0,0 +1,31 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A value naming a credential anywhere in it is found (novox/hq ADR 0201) — the guard against the
|
||||
// ordinary mistake — and a value that only mentions tokens as a count is not.
|
||||
func TestACredentialNamedFieldIsFoundAnywhereInAValue(t *testing.T) {
|
||||
for value, want := range map[string]string{
|
||||
`{"url":"http://x","headers":{"Authorization":"Bearer s"}}`: "Authorization",
|
||||
`[{"name":"a","env":{"API_KEY":"s"}}]`: "API_KEY",
|
||||
`{"refresh_token":"s"}`: "refresh_token",
|
||||
`{"clientSecret":"s"}`: "clientSecret",
|
||||
`{"maxTokens":4096,"name":"a","generation":3}`: "",
|
||||
`"just a string"`: "",
|
||||
// Issue 1126: a placement binds the provision "secret" to the vault; only an exact provider reference passes.
|
||||
`{"bindings":{"secret":{"node":"anchor","module":"mesh-vault"}}}`: "",
|
||||
`{"secret":{"value":"s"}}`: "secret",
|
||||
`{"secret":{"node":"anchor","module":"mesh-vault","value":"s"}}`: "secret",
|
||||
`{"secret":{"node":"anchor","module":7}}`: "secret",
|
||||
`{"secret":{"node":"anchor","module":"hunter2 Bearer s3cr3t"}}`: "secret",
|
||||
`{"secret":["s"]}`: "secret",
|
||||
`{"bindings":{"secret":{"token":"s"}}}`: "secret",
|
||||
} {
|
||||
if got := credentialField(json.RawMessage(value)); got != want {
|
||||
t.Errorf("%s: found %q, want %q", value, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//go:build integration
|
||||
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// **A watch whose current values take longer than the bound to hand over still holds** (novox/hq issue
|
||||
// 1726). The gitea module watches the checker's owed: one key per pull request head, each a round trip
|
||||
// to the bundle. With a few hundred keys the hand-over outlasted StateTimeout at every attempt, the
|
||||
// watch was refused, and the newest keys, handed over last, never reached mesh/check: a group's passing
|
||||
// verdict was never written on its members' heads. The bound is on progress: a watch fails only when
|
||||
// no current value is handed over within it.
|
||||
func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) {
|
||||
was := bus.StateTimeout
|
||||
bus.StateTimeout = 400 * time.Millisecond
|
||||
t.Cleanup(func() { bus.StateTimeout = was })
|
||||
|
||||
conn, kv := owedWatcher(t)
|
||||
// A group of three pull requests judged last, after many older heads: their keys come last.
|
||||
const older = 30
|
||||
var keys []string
|
||||
for i := 0; i < older; i++ {
|
||||
keys = append(keys, fmt.Sprintf("novox/mesh-catalog.%d.%040x.status=mesh/check", 400+i, i))
|
||||
}
|
||||
for _, member := range []string{"novox/mesh-sdk.76.34f210e5", "novox/mesh-controller.453.c06f75d0", "novox/mesh-delivery.33.28ee22ef"} {
|
||||
keys = append(keys, member+strings.Repeat("0", 32)+".status=mesh/check")
|
||||
}
|
||||
for _, k := range keys {
|
||||
if _, err := kv.Put(k, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"success"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var current, live []string
|
||||
// Each value takes 40 ms to be taken: 33 values take about 1.3 s, three times the bound.
|
||||
stop, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(ch bus.StateChange) error {
|
||||
if ch.Current {
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if ch.Current {
|
||||
current = append(current, ch.Key)
|
||||
} else {
|
||||
live = append(live, ch.Key)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a watch whose current values kept arriving was refused: %v", err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mu.Lock()
|
||||
got := append([]string(nil), current...)
|
||||
mu.Unlock()
|
||||
if len(got) != len(keys) {
|
||||
t.Fatalf("the watch returned after %d of %d current values", len(got), len(keys))
|
||||
}
|
||||
for i, k := range keys {
|
||||
if got[i] != k {
|
||||
t.Fatalf("current value %d is %s, want %s", i, got[i], k)
|
||||
}
|
||||
}
|
||||
|
||||
// And it holds: a verdict written afterwards reaches it as a change.
|
||||
next := "novox/mesh-sdk.77." + strings.Repeat("a", 40) + ".status=mesh/check"
|
||||
if _, err := kv.Put(next, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"pending"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(live) != 1 || live[0] != next {
|
||||
return fmt.Errorf("live changes so far: %v", live)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// A watch whose bundle takes nothing for longer than the bound is still refused, so a bundle that hangs
|
||||
// is said, not waited on for ever.
|
||||
func TestAWatchIsRefusedWhenNoCurrentValueIsTakenWithinTheBound(t *testing.T) {
|
||||
was := bus.StateTimeout
|
||||
bus.StateTimeout = 300 * time.Millisecond
|
||||
t.Cleanup(func() { bus.StateTimeout = was })
|
||||
|
||||
conn, kv := owedWatcher(t)
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
release := make(chan struct{})
|
||||
t.Cleanup(func() { close(release) })
|
||||
began := time.Now()
|
||||
_, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(bus.StateChange) error {
|
||||
<-release
|
||||
return nil
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "did not arrive in") {
|
||||
t.Fatalf("a watch whose first value was never taken: want refused, got %v", err)
|
||||
}
|
||||
if took := time.Since(began); took > 3*time.Second {
|
||||
t.Fatalf("the refusal took %s, far past the bound", took)
|
||||
}
|
||||
}
|
||||
|
||||
// owedWatcher is the gitea module's view of the checker's owed: the bucket, and a runtime connection
|
||||
// that serves gitea with the owed issued for reading.
|
||||
func owedWatcher(t *testing.T) (*bus.Conn, interface {
|
||||
Put(string, []byte) (uint64, error)
|
||||
}) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
kv := mesh.Bucket(t, "mesh-delivery-checker_owed")
|
||||
gitea := mt.MembershipOf("gitea", "anchor", false, nil)
|
||||
gitea.State = []bus.StateIssued{{Name: "mesh-delivery-checker.owed", Bucket: "mesh-delivery-checker_owed"}}
|
||||
mesh.Issue(t, gitea)
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("gitea")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("gitea") == nil {
|
||||
return errors.New("gitea not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return conn, kv
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A module reading its own stream back (novox/hq ADR 0363; the operator, 2026-10-30: a build step's whole failure output
|
||||
// is said on its build module's stream, and the module's tools read it there). As a log's read: one batch direct get,
|
||||
// no consumer, and only a stream the module's own membership names (`streams`); the bus grants its owner that stream's
|
||||
// info and direct get and no other's (mesh-sdk buscompose ownStreamGrants).
|
||||
|
||||
// StreamEntry is one event of a module's own stream: its sequence, its event (the subject after the stream's
|
||||
// `…event.`), its body and when the bus took it.
|
||||
type StreamEntry struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Event string `json:"event"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// StreamPage is one page of a stream's read: Next is the sequence to continue from, 0 when nothing follows; Last the
|
||||
// stream's last sequence.
|
||||
type StreamPage struct {
|
||||
Entries []StreamEntry `json:"entries"`
|
||||
Next uint64 `json:"next"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
// streamBusName is a stream's local name as the bus holds it (mesh-sdk catalogue.StreamBusName): capitals, a hyphen an
|
||||
// underscore; a bus name is itself.
|
||||
func streamBusName(name string) string { return strings.ToUpper(strings.ReplaceAll(name, "-", "_")) }
|
||||
|
||||
// issuedStream is the stream of its own a module may read by a name, local or the bus's, or the reason it may not.
|
||||
func (c *Conn) issuedStream(module, name string) (StreamIssued, error) {
|
||||
m := c.Membership(module)
|
||||
if m == nil {
|
||||
return StreamIssued{}, fmt.Errorf("%s has no membership issued on %s yet, so no stream of it is readable "+
|
||||
"until the mesh issues one (novox/hq ADR 0363)", module, c.node)
|
||||
}
|
||||
var names []string
|
||||
for _, s := range m.Streams {
|
||||
if s.Stream == streamBusName(name) && strings.Contains(s.Subject, "{event}") {
|
||||
return s, nil
|
||||
}
|
||||
names = append(names, s.Stream)
|
||||
}
|
||||
sort.Strings(names)
|
||||
issued := "none"
|
||||
if len(names) > 0 {
|
||||
issued = strings.Join(names, ", ")
|
||||
}
|
||||
return StreamIssued{}, fmt.Errorf("%s keeps no stream called %q: a module reads the streams it declares under `streams`, "+
|
||||
"and no other's; it was issued: %s (novox/hq ADR 0363)", module, name, issued)
|
||||
}
|
||||
|
||||
// StreamRead is a module's own stream's events from a sequence on, whose event names match a pattern (`*` one name, `>`
|
||||
// the rest, last; empty is every event), at most limit (LogReadDefault when 0, at most LogReadMax) and about LogPageBytes.
|
||||
func (c *Conn) StreamRead(module, name, events string, from uint64, limit int) (*StreamPage, error) {
|
||||
s, err := c.issuedStream(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case limit == 0:
|
||||
limit = LogReadDefault
|
||||
case limit < 0 || limit > LogReadMax:
|
||||
return nil, fmt.Errorf("a read's limit is 1 to %d events, %d when none is given, not %d", LogReadMax, LogReadDefault, limit)
|
||||
}
|
||||
if events == "" {
|
||||
events = ">"
|
||||
}
|
||||
parts := strings.Split(events, ".")
|
||||
for i, p := range parts {
|
||||
if !(p == ">" && i == len(parts)-1) && p != "*" && !logToken.MatchString(p) {
|
||||
return nil, fmt.Errorf("%q is not an event pattern: names, `*` for one and `>` for the rest, last", events)
|
||||
}
|
||||
}
|
||||
prefix, _, _ := strings.Cut(s.Subject, "{event}")
|
||||
missing := func() error {
|
||||
return fmt.Errorf("the stream %s is issued and not on the bus yet: the controller creates it from the catalogue "+
|
||||
"on its next raise, and reads it directly once it allows that (novox/hq ADR 0363)", s.Stream)
|
||||
}
|
||||
msgs, next, err := c.directBatch(s.Stream, prefix+events, from, limit, s.Stream, missing)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page := &StreamPage{Entries: []StreamEntry{}, Next: next}
|
||||
for _, msg := range msgs {
|
||||
e, err := streamEntryOf(s.Stream, prefix, msg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page.Entries = append(page.Entries, *e)
|
||||
}
|
||||
info, err := c.logInfo(LogIssued{Name: s.Stream, Stream: s.Stream})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page.Last = info.Last
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// streamEntryOf is an event a direct get answered: its event is its subject after prefix.
|
||||
func streamEntryOf(stream, prefix string, msg *nats.Msg) (*StreamEntry, error) {
|
||||
subject := msg.Header.Get("Nats-Subject")
|
||||
event, ok := strings.CutPrefix(subject, prefix)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the bus answered an event of %s on %q, which is not under %s", stream, subject, prefix)
|
||||
}
|
||||
seq, err := strconv.ParseUint(msg.Header.Get("Nats-Sequence"), 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus answered an event of %s with no sequence", stream)
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339Nano, msg.Header.Get("Nats-Time-Stamp"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus answered event %d of %s with no time: %v", seq, stream, err)
|
||||
}
|
||||
return &StreamEntry{Sequence: seq, Event: event, Body: valueOf(msg.Data), At: at.UTC()}, nil
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's own stream, as the runtime publishes to it and hears it (novox/hq ADR 0363 §8, issue 725).
|
||||
//
|
||||
// A module may declare a stream of its own; the controller creates it and says, in the module's
|
||||
// membership, which of its events go there and on which subject (`streams`), and, in the membership of
|
||||
// each module granted to hear it, which durable consumer on it to bind (`hears`). The runtime publishes and
|
||||
// binds exactly as the membership says. A membership naming neither is what every membership was before:
|
||||
// every event on EVENTS, and only the consumer on EVENTS bound.
|
||||
|
||||
// StreamIssued is one of a module's own streams as its membership names it: the events it carries, as the
|
||||
// bus spells them (`*` one token, `>` the rest), and the subject an event of them is published on, `{event}`
|
||||
// standing for the event.
|
||||
type StreamIssued struct {
|
||||
Stream string `json:"stream"`
|
||||
Events []string `json:"events"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
|
||||
// HeardIssued is one module stream an assignment hears: the stream and the durable consumer on it the
|
||||
// runtime binds. The consumer is the controller's to make, to filter and to delete.
|
||||
type HeardIssued struct {
|
||||
Stream string `json:"stream"`
|
||||
Consumer string `json:"consumer"`
|
||||
}
|
||||
|
||||
// eventSubject is where a module's event is published: on the subject of the first of its membership's
|
||||
// streams that carries it, else on EVENTS — `mesh.mod.<module>.event.<event>`, as every event was before.
|
||||
func eventSubject(m *Membership, module, event string) string {
|
||||
if m != nil {
|
||||
for _, s := range m.Streams {
|
||||
if s.Subject == "" || !strings.Contains(s.Subject, "{event}") {
|
||||
continue
|
||||
}
|
||||
for _, pattern := range s.Events {
|
||||
if eventMatches(pattern, event) {
|
||||
return strings.ReplaceAll(s.Subject, "{event}", event)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return "mesh.mod." + module + ".event." + event
|
||||
}
|
||||
|
||||
// eventMatches says whether an event's name falls under a pattern as the bus matches subjects: `*` is one
|
||||
// token, `>` (or `**`, the manifest's spelling) the one or more tokens left.
|
||||
func eventMatches(pattern, event string) bool {
|
||||
if pattern == "" || event == "" {
|
||||
return false
|
||||
}
|
||||
ps, es := strings.Split(pattern, "."), strings.Split(event, ".")
|
||||
for i, p := range ps {
|
||||
if p == ">" || p == "**" {
|
||||
return i == len(ps)-1 && len(es) > i
|
||||
}
|
||||
if i >= len(es) || (p != "*" && p != es[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(ps) == len(es)
|
||||
}
|
||||
|
||||
// RebindEvery is how often a consumer the membership names and the bus did not let the runtime bind is
|
||||
// tried again: the controller makes the consumer and sends the user list carrying its grant in the
|
||||
// composition that issues the membership, so one may arrive before the other (ADR 0363 §6).
|
||||
var RebindEvery = 30 * time.Second
|
||||
|
||||
// hearing is one module's consumers: the one on EVENTS, and each on a module stream its membership names.
|
||||
// Every event of all of them is handed to the module one at a time, as the consumer on EVENTS alone did.
|
||||
type hearing struct {
|
||||
c *Conn
|
||||
js jetstream.JetStream
|
||||
module string
|
||||
deliver func(Envelope) error
|
||||
// serial hands the module one event at a time across its consumers (novox/hq issue 387).
|
||||
serial sync.Mutex
|
||||
|
||||
// halted is stopped, read by handle without mu: the bus's loops hand events while mu is held.
|
||||
halted atomic.Bool
|
||||
|
||||
mu sync.Mutex
|
||||
stopped bool
|
||||
bound map[HeardIssued]jetstream.ConsumeContext
|
||||
// made is when the bus made each bound consumer: a consumer deleted and made again under its name has
|
||||
// another time, and a loop bound to the old one is not reading the new one.
|
||||
made map[HeardIssued]time.Time
|
||||
said map[HeardIssued]bool
|
||||
// why is the last error the bus gave a consumer's loop, said when the loop ends on it.
|
||||
// Guarded by whyMu, not mu: the bus hands errors from its own goroutines, possibly while mu is held.
|
||||
whyMu sync.Mutex
|
||||
why map[HeardIssued]error
|
||||
kick chan struct{}
|
||||
done chan struct{}
|
||||
unhear func()
|
||||
}
|
||||
|
||||
func newHearing(c *Conn, js jetstream.JetStream, module string, deliver func(Envelope) error) *hearing {
|
||||
return &hearing{c: c, js: js, module: module, deliver: deliver,
|
||||
bound: map[HeardIssued]jetstream.ConsumeContext{}, made: map[HeardIssued]time.Time{}, said: map[HeardIssued]bool{}, why: map[HeardIssued]error{},
|
||||
kick: make(chan struct{}, 1), done: make(chan struct{})}
|
||||
}
|
||||
|
||||
// bind reads one durable consumer by name, and on failure says what it was doing. Errors the bus gives
|
||||
// the loop afterwards are handed to onErr, when given.
|
||||
func (h *hearing) bind(stream, durable string, onErr func(error)) (jetstream.ConsumeContext, time.Time, string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
consumer, err := h.js.Consumer(ctx, stream, durable)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, "binding", err
|
||||
}
|
||||
var created time.Time
|
||||
if info := consumer.CachedInfo(); info != nil {
|
||||
created = info.Created
|
||||
}
|
||||
opts := []jetstream.PullConsumeOpt{jetstream.PullMaxMessages(1)}
|
||||
if onErr != nil {
|
||||
opts = append(opts, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { onErr(err) }))
|
||||
}
|
||||
reading, err := consumer.Consume(h.handle, opts...)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, "reading", err
|
||||
}
|
||||
return reading, created, "", nil
|
||||
}
|
||||
|
||||
// handle hands one delivered event to the module: acknowledged when it took it, offered again after NakDelay
|
||||
// when it did not, terminated when it is not an event at all.
|
||||
func (h *hearing) handle(msg jetstream.Msg) {
|
||||
env, ok := envelopeOf(msg.Subject(), msg.Headers(), msg.Data())
|
||||
if !ok {
|
||||
// Unparseable: redelivering bytes no version of a handler can read is a loop.
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
// Kept alive while the bundle works on it, as a worker's work is (seat.go): the consumer's ack wait
|
||||
// (30 seconds) is for a runtime that died, not a handler that is slow — the runtime gives a handler
|
||||
// launch.EventTimeout. Without it an event handled longer than the ack wait was offered again while
|
||||
// it was handled, and counted toward max-deliver (novox/hq issue 387). Kept alive too while it waits
|
||||
// for an event from another of the module's consumers to be handled.
|
||||
done, stopped := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(stopped)
|
||||
tick := time.NewTicker(EventHeartbeat)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
_ = msg.InProgress()
|
||||
}
|
||||
}
|
||||
}()
|
||||
h.serial.Lock()
|
||||
halted := h.halted.Load()
|
||||
var err error
|
||||
if !halted {
|
||||
err = h.deliver(env)
|
||||
}
|
||||
h.serial.Unlock()
|
||||
close(done)
|
||||
// No in-progress after the answer: one after a Nak would stretch its delay to the whole ack wait.
|
||||
<-stopped
|
||||
if halted {
|
||||
// **A hearing stopped hands nothing on** (issue 1945): the module's bundle that subscribed may be
|
||||
// gone, loaded again in its place; offered again at once, to the consumer bound after it.
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
_ = msg.NakWithDelay(NakDelay)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
}
|
||||
|
||||
// start binds what the membership names now, and follows it: a new membership binds what it adds and stops
|
||||
// what it no longer names, and a consumer not bound yet is tried again every RebindEvery.
|
||||
func (h *hearing) start() {
|
||||
h.reconcile()
|
||||
h.unhear = h.c.onMembership(func(m Membership) {
|
||||
if m.Module != h.module {
|
||||
return
|
||||
}
|
||||
h.nudge()
|
||||
})
|
||||
go func() {
|
||||
tick := time.NewTicker(RebindEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-h.done:
|
||||
return
|
||||
case <-h.kick:
|
||||
case <-tick.C:
|
||||
}
|
||||
h.reconcile()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// forget drops the last error a consumer's loop was given.
|
||||
func (h *hearing) forget(heard HeardIssued) {
|
||||
h.whyMu.Lock()
|
||||
delete(h.why, heard)
|
||||
h.whyMu.Unlock()
|
||||
}
|
||||
|
||||
// nudge asks the loop to reconcile now.
|
||||
func (h *hearing) nudge() {
|
||||
select {
|
||||
case h.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// stop stops every consumer on a module stream this module reads; the one on EVENTS is its caller's.
|
||||
func (h *hearing) stop() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.stopped {
|
||||
return
|
||||
}
|
||||
h.stopped = true
|
||||
h.halted.Store(true)
|
||||
close(h.done)
|
||||
if h.unhear != nil {
|
||||
h.unhear()
|
||||
}
|
||||
for heard, reading := range h.bound {
|
||||
reading.Stop()
|
||||
delete(h.bound, heard)
|
||||
delete(h.made, heard)
|
||||
}
|
||||
}
|
||||
|
||||
// replaced says whether the consumer a loop is bound to is gone from the bus or was made again since. A bus
|
||||
// that cannot be asked right now says no: the loop is left as it is.
|
||||
func (h *hearing) replaced(heard HeardIssued) bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
consumer, err := h.js.Consumer(ctx, heard.Stream, heard.Consumer)
|
||||
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
return true
|
||||
}
|
||||
if err != nil || consumer.CachedInfo() == nil {
|
||||
return false
|
||||
}
|
||||
created := h.made[heard]
|
||||
return !created.IsZero() && !consumer.CachedInfo().Created.Equal(created)
|
||||
}
|
||||
|
||||
// reconcile makes what is bound what the module's membership names now.
|
||||
func (h *hearing) reconcile() {
|
||||
want := map[HeardIssued]bool{}
|
||||
if m := h.c.Membership(h.module); m != nil {
|
||||
for _, heard := range m.Hears {
|
||||
if heard.Stream != "" && heard.Consumer != "" {
|
||||
want[heard] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.stopped {
|
||||
return
|
||||
}
|
||||
for heard, reading := range h.bound {
|
||||
if !want[heard] {
|
||||
reading.Stop()
|
||||
delete(h.bound, heard)
|
||||
delete(h.made, heard)
|
||||
h.c.Logf("[mesh-tools] %s no longer hears %s: its membership stopped naming its consumer %s",
|
||||
h.module, heard.Stream, heard.Consumer)
|
||||
}
|
||||
}
|
||||
for heard := range h.said {
|
||||
if !want[heard] {
|
||||
delete(h.said, heard)
|
||||
h.forget(heard)
|
||||
}
|
||||
}
|
||||
// **A loop the bus ended is no longer reading** (nats.go ends a pull loop on its own when its consumer is
|
||||
// deleted): the controller deletes and makes again under the same name a restricted consumer it cannot
|
||||
// narrow (ADR 0363 §6), and the membership, unchanged, says nothing — so it is bound again here.
|
||||
// **A loop the bus did not end is asked about**: nats.go learns of a deleted consumer only from the answer
|
||||
// to a pull it has pending, and a consumer deleted and made again between two pulls, or while none is
|
||||
// pending, leaves the loop waiting out its heartbeat (30 seconds) on a consumer that is gone. So each
|
||||
// bound consumer is looked up by name: gone, or made at another time, ends its loop here.
|
||||
for heard, reading := range h.bound {
|
||||
if h.replaced(heard) {
|
||||
h.whyMu.Lock()
|
||||
if h.why[heard] == nil {
|
||||
h.why[heard] = errors.New("the consumer was deleted or made again")
|
||||
}
|
||||
h.whyMu.Unlock()
|
||||
reading.Stop()
|
||||
}
|
||||
}
|
||||
for heard, reading := range h.bound {
|
||||
select {
|
||||
case <-reading.Closed():
|
||||
delete(h.bound, heard)
|
||||
delete(h.made, heard)
|
||||
h.whyMu.Lock()
|
||||
why := h.why[heard]
|
||||
h.whyMu.Unlock()
|
||||
if why == nil {
|
||||
why = errors.New("no reason given")
|
||||
}
|
||||
h.c.Logf("[mesh-tools] %s stopped hearing %s: the bus ended reading its consumer %s (%v); bound again as it exists",
|
||||
h.module, heard.Stream, heard.Consumer, why)
|
||||
h.said[heard] = true
|
||||
default:
|
||||
}
|
||||
}
|
||||
missing := make([]HeardIssued, 0, len(want))
|
||||
for heard := range want {
|
||||
if h.bound[heard] == nil {
|
||||
missing = append(missing, heard)
|
||||
}
|
||||
}
|
||||
sort.Slice(missing, func(i, j int) bool { return missing[i].Stream < missing[j].Stream })
|
||||
for _, heard := range missing {
|
||||
heard := heard
|
||||
h.forget(heard)
|
||||
reading, created, doing, err := h.bind(heard.Stream, heard.Consumer, func(err error) {
|
||||
h.whyMu.Lock()
|
||||
h.why[heard] = err
|
||||
h.whyMu.Unlock()
|
||||
})
|
||||
if err != nil {
|
||||
// Said once, not at every attempt; said again when it is bound.
|
||||
if !h.said[heard] {
|
||||
h.c.Logf("[mesh-tools] %s does not hear %s yet: %s its consumer %s failed: %v; tried again every %s",
|
||||
h.module, heard.Stream, doing, heard.Consumer, err, RebindEvery)
|
||||
h.said[heard] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
h.bound[heard] = reading
|
||||
h.made[heard] = created
|
||||
delete(h.said, heard)
|
||||
h.c.Logf("[mesh-tools] %s hears %s on its consumer %s", h.module, heard.Stream, heard.Consumer)
|
||||
// When the bus ends the loop, look again at once rather than at the next tick.
|
||||
go func() {
|
||||
select {
|
||||
case <-reading.Closed():
|
||||
h.nudge()
|
||||
case <-h.done:
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package bus
|
||||
|
||||
import "testing"
|
||||
|
||||
// novox/hq ADR 0363 §2, §3: an event goes to the stream whose pattern carries it, as the bus matches subjects.
|
||||
func TestAnEventFallsUnderAStreamsPatternAsTheBusMatchesIt(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
pattern, event string
|
||||
want bool
|
||||
}{
|
||||
{"mail.*.message.added", "mail.home.message.added", true},
|
||||
{"mail.*.message.added", "mail.home.message.removed", false},
|
||||
{"mail.*.message.added", "mail.home.work.message.added", false},
|
||||
{"mail.>", "mail.home.message.added", true},
|
||||
{"mail.**", "mail.home", true},
|
||||
{"mail.>", "mail", false},
|
||||
{"status", "status", true},
|
||||
{"status", "status.changed", false},
|
||||
{"", "status", false},
|
||||
} {
|
||||
if got := eventMatches(c.pattern, c.event); got != c.want {
|
||||
t.Errorf("%q under %q: %v, want %v", c.event, c.pattern, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A membership naming no stream publishes every event on EVENTS, exactly as before (issue 725).
|
||||
func TestAnEventNoStreamCarriesIsPublishedOnEvents(t *testing.T) {
|
||||
mail := &Membership{Module: "mail", Streams: []StreamIssued{{Stream: "MAIL", Events: []string{"mail.*.message.added"},
|
||||
Subject: "mesh.stream.mail.event.{event}"}}}
|
||||
for _, c := range []struct {
|
||||
m *Membership
|
||||
event string
|
||||
want string
|
||||
}{
|
||||
{nil, "mail.home.message.added", "mesh.mod.mail.event.mail.home.message.added"},
|
||||
{&Membership{Module: "mail"}, "mail.home.message.added", "mesh.mod.mail.event.mail.home.message.added"},
|
||||
{mail, "mail.home.message.added", "mesh.stream.mail.event.mail.home.message.added"},
|
||||
{mail, "status", "mesh.mod.mail.event.status"},
|
||||
} {
|
||||
if got := eventSubject(c.m, "mail", c.event); got != c.want {
|
||||
t.Errorf("%s: %s, want %s", c.event, got, c.want)
|
||||
}
|
||||
}
|
||||
if got := keyOf("mesh.stream.mail.event.mail.home.message.added"); got != "mail.mail.home.message.added" {
|
||||
t.Errorf("a stream's event reaches its consumer as %q, not as on EVENTS", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// streamMesh is a mesh with the mail module's own stream MAIL, and a connection on anchor following the
|
||||
// memberships of the modules named.
|
||||
func streamMesh(t *testing.T, memberships ...bus.Membership) (*mt.Mesh, *bus.Conn) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
mesh.ModuleStream(t, "mail", "MAIL")
|
||||
for _, m := range memberships {
|
||||
mesh.Issue(t, m)
|
||||
}
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
for _, m := range memberships {
|
||||
conn.Follow(m.Module)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
for _, m := range memberships {
|
||||
if conn.Membership(m.Module) == nil {
|
||||
return errors.New(m.Module + " not issued yet")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return mesh, conn
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §2, §8 (issue 725): an event one of the module's streams carries is published on that
|
||||
// stream's subject and kept by that stream alone; any other event of it goes to EVENTS as before.
|
||||
func TestAnEventADeclaredStreamCarriesIsPublishedOnItsSubjectAlone(t *testing.T) {
|
||||
mail := mt.MembershipOf("mail", "anchor", true, nil)
|
||||
mail.Streams = []bus.StreamIssued{{Stream: "MAIL", Events: []string{"mail.*.message.added"},
|
||||
Subject: "mesh.stream.mail.event.{event}"}}
|
||||
mesh, conn := streamMesh(t, mail)
|
||||
|
||||
landed := mesh.NextEvent(t, "mesh.stream.mail.event.>")
|
||||
if err := conn.PublishAs("mail", bus.Envelope{Key: "mail.home.message.added", Body: json.RawMessage(`{"id":1}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case got := <-landed:
|
||||
if got != "mesh.stream.mail.event.mail.home.message.added" {
|
||||
t.Errorf("published on %s", got)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the event did not land on its stream's subject")
|
||||
}
|
||||
if err := conn.PublishAs("mail", bus.Envelope{Key: "status", Body: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := mesh.Messages(t, "MAIL"); n != 1 {
|
||||
t.Errorf("MAIL holds %d messages, want the one event it carries", n)
|
||||
}
|
||||
if n := mesh.Messages(t, "EVENTS"); n != 1 {
|
||||
t.Errorf("EVENTS holds %d messages, want only the event no stream carries", n)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §8 (issue 725): a module hears, beside EVENTS, every module stream its membership names,
|
||||
// one event at a time, with the key it reads on EVENTS; a membership naming none hears EVENTS alone.
|
||||
func TestAModuleHearsEveryStreamItsMembershipNames(t *testing.T) {
|
||||
reader := mt.MembershipOf("reader", "anchor", false, nil)
|
||||
mesh, conn := streamMesh(t, reader)
|
||||
mesh.Consumer(t, "anchor", "reader", []string{"mesh.mod.alpha.event.>"}, 5*time.Second)
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.mail.home.>"}, 5*time.Second)
|
||||
|
||||
var mu sync.Mutex
|
||||
heard := map[string]int{}
|
||||
handling := 0
|
||||
stop, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||
mu.Lock()
|
||||
handling++
|
||||
if handling > 1 {
|
||||
t.Error("two events were handed to the module at once")
|
||||
}
|
||||
mu.Unlock()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
mu.Lock()
|
||||
handling--
|
||||
heard[env.Key]++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
count := func(key string) int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return heard[key]
|
||||
}
|
||||
|
||||
// Its membership names no stream: EVENTS alone, as before.
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 1})
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error {
|
||||
if count("alpha.happened") != 1 {
|
||||
return errors.New("the event on EVENTS was not heard")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if _, notYet := mesh.PendingOn(t, "MAIL", "anchor", "reader"); notYet != 1 || count("mail.mail.home.message.added") != 0 {
|
||||
t.Fatalf("a module whose membership names no stream heard MAIL (%d not yet delivered)", notYet)
|
||||
}
|
||||
|
||||
// Its membership now names its consumer on MAIL: bound, and what waited there is heard.
|
||||
reader.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "anchor_reader"}}
|
||||
mesh.Issue(t, reader)
|
||||
mt.Until(t, func() error {
|
||||
if count("mail.mail.home.message.added") != 1 {
|
||||
return errors.New("the event on MAIL was not heard")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for n := 2; n <= 4; n++ {
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": n})
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": n})
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
if a, m := count("alpha.happened"), count("mail.mail.home.message.added"); a != 4 || m != 4 {
|
||||
return fmt.Errorf("heard %d on EVENTS and %d on MAIL, want 4 and 4", a, m)
|
||||
}
|
||||
for _, stream := range []string{"EVENTS", "MAIL"} {
|
||||
if ack, notYet := mesh.PendingOn(t, stream, "anchor", "reader"); ack != 0 || notYet != 0 {
|
||||
return fmt.Errorf("%s still holds %d unacknowledged, %d undelivered", stream, ack, notYet)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// A membership that stops naming it stops what it reads there.
|
||||
reader.Hears = nil
|
||||
mesh.Issue(t, reader)
|
||||
mt.Until(t, func() error {
|
||||
if mesh.Consumers(t, "MAIL") != 1 {
|
||||
return errors.New("MAIL's consumer is gone")
|
||||
}
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 0})
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if _, notYet := mesh.PendingOn(t, "MAIL", "anchor", "reader"); notYet == 0 {
|
||||
return errors.New("MAIL is still read")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §6 (issue 725): a consumer the membership names before the controller made it is bound once
|
||||
// it exists, without the module's events on EVENTS waiting for it.
|
||||
func TestAConsumerNamedBeforeItExistsIsBoundOnceItDoes(t *testing.T) {
|
||||
restore := bus.RebindEvery
|
||||
bus.RebindEvery = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.RebindEvery = restore })
|
||||
reader := mt.MembershipOf("reader", "anchor", false, nil)
|
||||
reader.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "anchor_reader"}}
|
||||
mesh, conn := streamMesh(t, reader)
|
||||
mesh.Consumer(t, "anchor", "reader", []string{"mesh.mod.alpha.event.>"}, 5*time.Second)
|
||||
|
||||
var mu sync.Mutex
|
||||
heard := map[string]int{}
|
||||
stop, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||
mu.Lock()
|
||||
heard[env.Key]++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a consumer on a module stream not made yet kept the module from hearing EVENTS: %v", err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{})
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.>"}, 5*time.Second)
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{})
|
||||
mt.Until(t, func() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if heard["alpha.happened"] != 1 || heard["mail.mail.home.message.added"] != 1 {
|
||||
return fmt.Errorf("heard %v", heard)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §6 (issue 725): a consumer the controller deletes and makes again under the same name, its
|
||||
// membership unchanged, is bound again, and what is published after is heard — the module does not go deaf.
|
||||
func TestAConsumerDeletedAndMadeAgainIsHeardAgain(t *testing.T) {
|
||||
restore := bus.RebindEvery
|
||||
bus.RebindEvery = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.RebindEvery = restore })
|
||||
reader := mt.MembershipOf("reader", "anchor", false, nil)
|
||||
reader.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "anchor_reader"}}
|
||||
mesh, conn := streamMesh(t, reader)
|
||||
mesh.Consumer(t, "anchor", "reader", []string{"mesh.mod.alpha.event.>"}, 5*time.Second)
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.>"}, 5*time.Second)
|
||||
logs := &syncLogs{}
|
||||
conn.Logf = logs.Logf
|
||||
|
||||
var mu sync.Mutex
|
||||
heard := 0
|
||||
stop, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||
if env.Key == "mail.mail.home.message.added" {
|
||||
mu.Lock()
|
||||
heard++
|
||||
mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
count := func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return heard
|
||||
}
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error {
|
||||
if count() != 1 {
|
||||
return errors.New("the first event on MAIL was not heard")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
mesh.RemoveConsumer(t, "MAIL", "anchor", "reader")
|
||||
mt.Until(t, func() error {
|
||||
if !logs.Has("stopped hearing MAIL") {
|
||||
return errors.New("the end of the loop was not said")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.>"}, 5*time.Second)
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 2})
|
||||
mt.Until(t, func() error {
|
||||
if count() != 2 {
|
||||
return fmt.Errorf("heard %d events on MAIL after its consumer was made again, want 2", count())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// syncLogs keeps what the runtime says, from any goroutine.
|
||||
type syncLogs struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (l *syncLogs) Logf(format string, args ...any) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.lines = append(l.lines, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *syncLogs) Has(fragment string) bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
for _, line := range l.lines {
|
||||
if strings.Contains(line, fragment) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// Waiting on the bus for one event (novox/hq ADR 0367): what the mesh MCP server's `mesh_wait` does.
|
||||
//
|
||||
// **A subscription for the call, and nothing kept** (§2): no durable consumer, no queue, no stream. It
|
||||
// subscribes the subjects when called and unsubscribes them before it answers, so an event published
|
||||
// between two calls is not heard — a session reads the state from its owner after every wake (§4).
|
||||
|
||||
// WaitIssued is one event family the mesh issued this account to wait on (ADR 0367 §2): the name a session
|
||||
// asks for, and the subject it lands on. Only the tool runner's membership carries any.
|
||||
type WaitIssued struct {
|
||||
Event string `json:"event"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
|
||||
// Heard is one event a wait heard: the subject it arrived on, its body and when it arrived.
|
||||
type Heard struct {
|
||||
Subject string
|
||||
Body json.RawMessage
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// HearFirst subscribes the subjects for this call alone and answers the first message whose body match
|
||||
// accepts (every one, when match is nil), or ok false once `within` has passed. Every subscription is
|
||||
// closed before it returns. A subscription the bus refuses is said as an error naming the subject, at once;
|
||||
// a caller gone — ctx done — ends the wait at once with ctx's error, so nothing waits on for nobody.
|
||||
func (c *Conn) HearFirst(ctx context.Context, subjects []string, within time.Duration, match func(json.RawMessage) bool) (heard Heard, ok bool, err error) {
|
||||
if len(subjects) == 0 {
|
||||
return Heard{}, false, fmt.Errorf("a wait names at least one subject")
|
||||
}
|
||||
got := make(chan Heard, 1)
|
||||
refused := make(chan string, len(subjects))
|
||||
var subs []*nats.Subscription
|
||||
defer func() {
|
||||
for _, s := range subs {
|
||||
_ = s.Unsubscribe()
|
||||
}
|
||||
c.mu.Lock()
|
||||
for _, subject := range subjects {
|
||||
delete(c.waits[subject], refused)
|
||||
if len(c.waits[subject]) == 0 {
|
||||
delete(c.waits, subject)
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}()
|
||||
c.mu.Lock()
|
||||
if c.waits == nil {
|
||||
c.waits = map[string]map[chan string]bool{}
|
||||
}
|
||||
for _, subject := range subjects {
|
||||
if c.waits[subject] == nil {
|
||||
c.waits[subject] = map[chan string]bool{}
|
||||
}
|
||||
c.waits[subject][refused] = true
|
||||
}
|
||||
c.mu.Unlock()
|
||||
for _, subject := range subjects {
|
||||
sub, err := c.nc.Subscribe(subject, func(msg *nats.Msg) {
|
||||
body := json.RawMessage(msg.Data)
|
||||
if !json.Valid(body) {
|
||||
quoted, _ := json.Marshal(string(msg.Data))
|
||||
body = quoted
|
||||
}
|
||||
if match != nil && !match(body) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case got <- Heard{Subject: msg.Subject, Body: body, At: time.Now().UTC()}:
|
||||
default:
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return Heard{}, false, fmt.Errorf("could not wait on %s: %w", subject, err)
|
||||
}
|
||||
subs = append(subs, sub)
|
||||
}
|
||||
// The bus has every subscription — or has said which it refused — once this returns.
|
||||
if err := c.nc.FlushTimeout(5 * time.Second); err != nil {
|
||||
return Heard{}, false, fmt.Errorf("the bus did not take the wait: %w", err)
|
||||
}
|
||||
timer := time.NewTimer(within)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case h := <-got:
|
||||
return h, true, nil
|
||||
case subject := <-refused:
|
||||
return Heard{}, false, fmt.Errorf("the bus refused this account a subscription on %s: it is not in the "+
|
||||
"account's wait grant (novox/hq ADR 0367 §2)", subject)
|
||||
case <-timer.C:
|
||||
return Heard{}, false, nil
|
||||
case <-ctx.Done():
|
||||
return Heard{}, false, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// waitRefused tells every wait on a subject the bus refused that it was refused; true when one was waiting.
|
||||
func (c *Conn) waitRefused(subject string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
told := false
|
||||
for ch := range c.waits[subject] {
|
||||
select {
|
||||
case ch <- subject:
|
||||
default:
|
||||
}
|
||||
told = true
|
||||
}
|
||||
return told
|
||||
}
|
||||
|
||||
// Subscriptions is how many subscriptions this connection holds now: what a wait leaves behind is none.
|
||||
func (c *Conn) Subscriptions() int { return c.nc.NumSubscriptions() }
|
||||
|
||||
// subjectRefused is the subject a subscription refusal names, or "".
|
||||
func subjectRefused(err error) string {
|
||||
m := refusedSubject.FindStringSubmatch(err.Error())
|
||||
if len(m) < 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.Trim(m[1], `"`)
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0367 §2, §5: the bus refuses the tool runner's account any subject its wait grant does not
|
||||
// name — another asker's `decided` among them — and a wait on one says the bus refused it, at once.
|
||||
func TestTheBusRefusesAWaitOutsideTheGrant(t *testing.T) {
|
||||
granted := "mesh.seat.operator-channel.event.decided.claude-code"
|
||||
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, NoLog: true, NoSigs: true,
|
||||
Users: []*server.User{{Username: "desk.node-tools", Password: "pw", Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: []string{"$JS.API.DIRECT.GET.>", "mesh.>"}},
|
||||
Subscribe: &server.SubjectPermission{Allow: []string{granted, "_INBOX.desk.node-tools.>", "mesh.assignment.desk.*"}},
|
||||
}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
c := dial(t, s.ClientURL(), "node-tools", "desk", "desk.node-tools")
|
||||
|
||||
start := time.Now()
|
||||
_, ok, err := c.HearFirst(context.Background(), []string{"mesh.seat.operator-channel.event.decided.shop"}, 5*time.Second, nil)
|
||||
if ok || err == nil || !strings.Contains(err.Error(), "refused") {
|
||||
t.Errorf("a wait on another asker's decided: heard %v, %v", ok, err)
|
||||
}
|
||||
if time.Since(start) > 3*time.Second {
|
||||
t.Errorf("the refusal took %s to be said", time.Since(start))
|
||||
}
|
||||
|
||||
done := make(chan bus.Heard, 1)
|
||||
go func() {
|
||||
h, ok, err := c.HearFirst(context.Background(), []string{granted}, 5*time.Second, func(body json.RawMessage) bool { return true })
|
||||
if ok && err == nil {
|
||||
done <- h
|
||||
}
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if err := c.Say(granted, []byte(`{"ask":"a1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case h := <-done:
|
||||
if h.Subject != granted || string(h.Body) != `{"ask":"a1"}` {
|
||||
t.Errorf("heard %+v", h)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a wait on the granted subject heard nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// A wait whose caller is gone ends at once, its subscription closed (the review of issue 745).
|
||||
func TestAWaitEndsWhenItsCallerIsGone(t *testing.T) {
|
||||
c := dial(t, mt.URL(t), "node-tools", "desk", "")
|
||||
before := c.Subscriptions()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
time.AfterFunc(200*time.Millisecond, cancel)
|
||||
start := time.Now()
|
||||
_, ok, err := c.HearFirst(ctx, []string{"mesh.mod.mesh-knowledge.event.reviewed"}, time.Minute, nil)
|
||||
if ok || !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("a cancelled wait answered %v, %v", ok, err)
|
||||
}
|
||||
if took := time.Since(start); took > 2*time.Second {
|
||||
t.Errorf("a cancelled wait ended after %s", took)
|
||||
}
|
||||
if after := c.Subscriptions(); after != before {
|
||||
t.Errorf("%d subscriptions before, %d after", before, after)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,244 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
// text is a tool result's first text, and whether it was an error.
|
||||
func text(t *testing.T, reply map[string]any) (string, bool) {
|
||||
t.Helper()
|
||||
result, ok := reply["result"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("no result: %v", reply)
|
||||
}
|
||||
content := result["content"].([]any)
|
||||
isErr, _ := result["isError"].(bool)
|
||||
var parts []string
|
||||
for _, c := range content {
|
||||
parts = append(parts, c.(map[string]any)["text"].(string))
|
||||
}
|
||||
return strings.Join(parts, "\n"), isErr
|
||||
}
|
||||
|
||||
func call(t *testing.T, endpoint, tool string, args map[string]any) (string, bool) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 9, "method": "tools/call",
|
||||
"params": map[string]any{"name": tool, "arguments": args}})
|
||||
return text(t, post(t, endpoint, string(body)))
|
||||
}
|
||||
|
||||
// novox/hq ADR 0195 and ADR 0367: the console announces seven tools, and everything the mesh answers is reached
|
||||
// through them by one address per layer.
|
||||
func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
was := IndexKept
|
||||
IndexKept = 0 // every discovery asks the mesh, so a module arriving mid-test is found
|
||||
t.Cleanup(func() { IndexKept = was })
|
||||
|
||||
mesh := mt.New(t)
|
||||
// alpha: interchangeable (the mesh issued it a plain subject); beta: state on its machine, holds
|
||||
// the node-shelf seat there.
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "desk", true, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "desk", false, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
nodeTools := connect(t, "node-tools", "desk")
|
||||
stop, err := runtime.Run(nodeTools, []runtime.Served{
|
||||
{Module: "alpha", Entrypoints: []string{mt.Fixture("many-alpha.serve.mjs")}},
|
||||
{Module: "beta", Entrypoints: []string{mt.Fixture("many-beta.serve.mjs")}},
|
||||
}, nil, (&mt.Logs{}).Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
|
||||
// Nothing may be asked for a roster or a module's `tools` any more (ADR 0197): counted.
|
||||
var asked atomic.Int32
|
||||
watcher := connect(t, "watcher", "")
|
||||
for _, subject := range []string{"mesh.mod.*.tool.tools", "mesh.mod.*.tool.tools.*", "mesh.mod.mesh-catalog.>"} {
|
||||
stop, err := watcher.Raw(subject, func(string, []byte) []byte { asked.Add(1); return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
|
||||
// The controller: its records as JSON, as its seat verbs answer them, and its own seat's verbs
|
||||
// announced on the bus like every runtime's.
|
||||
controller := connect(t, "mesh-controller", "bench")
|
||||
out := func(s string) map[string]any { return map[string]any{"output": s, "ok": true} }
|
||||
serve := func(verb string, answer func() any) {
|
||||
stop, err := controller.HandleSubject("mesh.seat.mesh-controller.tool."+verb, func(json.RawMessage) (any, error) {
|
||||
return answer(), nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
serve("nodes", func() any {
|
||||
return out("the bus's user list leaves out 2 user(s)\n" +
|
||||
`[{"name":"bench","heard":"3m ago","mode":"converged","id":"1f2e"},{"name":"desk","heard":"here","mode":"converged","id":"9a8b"}]`)
|
||||
})
|
||||
var gammaOn atomic.Value
|
||||
gammaOn.Store("[]")
|
||||
serve("modules", func() any {
|
||||
return out(fmt.Sprintf(`[{"module":"alpha","on":["desk"],"tools":true},{"module":"beta","on":["desk"],"tools":true},`+
|
||||
`{"module":"gamma","on":%s,"tools":true},{"module":"delta","on":["desk"],"tools":false},`+
|
||||
`{"module":"epsilon","on":["bench"],"tools":true},{"module":"mesh-controller","on":["bench"],"tools":true}]`, gammaOn.Load().(string)))
|
||||
})
|
||||
stopAnn, err := announce.Serve(controller, announce.Service{Name: "mesh-controller", ID: "bench"}, func() []announce.Endpoint {
|
||||
return []announce.Endpoint{{Kind: announce.KindSeat, Module: "mesh-controller", Seat: "mesh-controller", Scope: "mesh",
|
||||
// As the live controller announces: a mesh seat's holder names no machine.
|
||||
Tool: "nodes", Node: "", Description: "Every machine the mesh knows.", Schema: json.RawMessage(`{}`),
|
||||
Subject: "mesh.seat.mesh-controller.tool.nodes"}}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopAnn)
|
||||
controller.Flush()
|
||||
watcher.Flush()
|
||||
|
||||
up, err := Serve(NewSurface(nodeTools, "desk.node-tools"), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer up.Close()
|
||||
endpoint := "http://" + up.Address + "/mcp"
|
||||
|
||||
// Seven tools, nothing else (ADR 0367).
|
||||
listed := post(t, endpoint, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`)["result"].(map[string]any)
|
||||
var names []string
|
||||
for _, x := range listed["tools"].([]any) {
|
||||
name := x.(map[string]any)["name"].(string)
|
||||
names = append(names, name)
|
||||
for _, r := range name {
|
||||
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-') || len(name) > 64 {
|
||||
t.Errorf("%q is not a name the API takes", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if got := strings.Join(names, ","); got != "mesh_overview,mesh_machine,mesh_search,mesh_describe,mesh_call,mesh_runtimes,mesh_wait" {
|
||||
t.Errorf("announced %s", got)
|
||||
}
|
||||
|
||||
// The overview names the mesh's seats, the machines' seats and the machines.
|
||||
overview, isErr := call(t, endpoint, "mesh_overview", nil)
|
||||
if isErr || !strings.Contains(overview, "mesh-controller.nodes") || !strings.Contains(overview, "<node>/node-shelf.list") ||
|
||||
!strings.Contains(overview, `"bench"`) || !strings.Contains(overview, `"desk"`) {
|
||||
t.Errorf("overview: %s", overview)
|
||||
}
|
||||
// Silence is named only where tools should have answered: epsilon declares tools on bench and
|
||||
// nothing there announced it; delta declares none and is never a name.
|
||||
if strings.Contains(overview, "mesh-controller on bench") {
|
||||
t.Errorf("a mesh seat's holder that announced no machine is called silent on its machine:\n%s", overview)
|
||||
}
|
||||
if !strings.Contains(overview, "epsilon on bench") || strings.Contains(overview, "delta") {
|
||||
t.Errorf("not answering: %s", overview)
|
||||
}
|
||||
machine, isErr := call(t, endpoint, "mesh_machine", map[string]any{"node": "desk"})
|
||||
if isErr || !strings.Contains(machine, "desk/node-shelf.list") || !strings.Contains(machine, "desk/beta.three") ||
|
||||
!strings.Contains(machine, "desk/alpha.one") {
|
||||
t.Errorf("machine: %s", machine)
|
||||
}
|
||||
|
||||
// A mesh seat, a node seat, an assignment and an interchangeable module, each by address.
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "mesh-controller.nodes"}); isErr || !strings.Contains(got, "converged") {
|
||||
t.Errorf("mesh seat: %s", got)
|
||||
}
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "desk/node-shelf.list"}); isErr || !strings.Contains(got, `"a"`) {
|
||||
t.Errorf("node seat: %s", got)
|
||||
}
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "desk/beta.three"}); isErr ||
|
||||
!strings.Contains(got, `"beta": 3`) || !strings.Contains(got, "answered by desk") {
|
||||
t.Errorf("assignment: %s", got)
|
||||
}
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "alpha.one"}); isErr || !strings.Contains(got, `"alpha": 1`) {
|
||||
t.Errorf("interchangeable module: %s", got)
|
||||
}
|
||||
|
||||
// Refused, by name, where the address does not say enough or says the wrong thing.
|
||||
for address, want := range map[string]string{
|
||||
"beta.three": "keeps state on each machine it runs on, so a call names the machine: write <node>/beta.three — it runs on desk",
|
||||
"node-shelf.list": "held once per machine: write <node>/node-shelf.list — it is held on desk",
|
||||
"desk/mesh-controller.nodes": "held once for the whole mesh",
|
||||
"bench/beta.three": "beta does not run on bench; it runs on desk",
|
||||
"nonsense": "is not an address",
|
||||
} {
|
||||
got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": address})
|
||||
if !isErr || !strings.Contains(got, want) {
|
||||
t.Errorf("%s: %s (want %q)", address, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Discovery itself asked nothing but the bus.
|
||||
if n := asked.Load(); n != 0 {
|
||||
t.Errorf("discovery asked a roster or a module's tools %d time(s); it asks the bus", n)
|
||||
}
|
||||
|
||||
// Described without `node`: the address carries the machine.
|
||||
described, isErr := call(t, endpoint, "mesh_describe", map[string]any{"address": "desk/beta.three"})
|
||||
if isErr || strings.Contains(described, `"node"`) || !strings.Contains(described, `"address": "desk/beta.three"`) {
|
||||
t.Errorf("describe: %s", described)
|
||||
}
|
||||
// Its schema, which discovery no longer carries for a module's tool, asked of the runtime serving it.
|
||||
if !strings.Contains(described, `"verbose"`) {
|
||||
t.Errorf("describe lost the tool's arguments: %s", described)
|
||||
}
|
||||
|
||||
// Search finds across the layers; a module that starts serving after the first answer is found.
|
||||
if got, _ := call(t, endpoint, "mesh_search", map[string]any{"query": "shelf"}); !strings.Contains(got, "desk/node-shelf.list") {
|
||||
t.Errorf("search a node seat: %s", got)
|
||||
}
|
||||
if got, _ := call(t, endpoint, "mesh_search", map[string]any{"query": "gamma"}); strings.Contains(got, "gamma.given") {
|
||||
t.Fatalf("gamma was found before it served: %s", got)
|
||||
}
|
||||
mesh.Issue(t, mt.MembershipOf("gamma", "desk", false, nil))
|
||||
gammaOn.Store(`["desk"]`)
|
||||
late := connect(t, "node-tools", "desk")
|
||||
stopLate, err := runtime.Run(late, []runtime.Served{{Module: "gamma", Entrypoints: []string{mt.Fixture("env-gamma.serve.mjs")}}},
|
||||
nil, (&mt.Logs{}).Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stopLate()
|
||||
var found string
|
||||
for i := 0; i < 30; i++ {
|
||||
found, _ = call(t, endpoint, "mesh_search", map[string]any{"query": "gamma given"})
|
||||
if strings.Contains(found, "desk/gamma.given") {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if !strings.Contains(found, "desk/gamma.given") {
|
||||
t.Errorf("a module that arrived later was not found: %s", found)
|
||||
}
|
||||
|
||||
// Only describing a module's tool asks its runtime — once, for the schema discovery no longer carries
|
||||
// (2026-10-04: every tool's schema made a machine's discovery answer outgrow the bus).
|
||||
if n := asked.Load(); n != 1 {
|
||||
t.Errorf("asked a module's tools %d time(s); describing one tool asks once, and discovery never", n)
|
||||
}
|
||||
|
||||
// The old names still answer, unannounced.
|
||||
if got, isErr := call(t, endpoint, "alpha.one", nil); isErr || !strings.Contains(got, `"alpha": 1`) {
|
||||
t.Errorf("an old name: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControllersRecordsAreReadAsJSON(t *testing.T) {
|
||||
if got := jsonIn("a warning printed first\n[{\"name\":\"ace\"}]"); got != `[{"name":"ace"}]` {
|
||||
t.Errorf("an array after a warning: %q", got)
|
||||
}
|
||||
if got := jsonIn(`{"seats":[]}`); got != `{"seats":[]}` {
|
||||
t.Errorf("an object: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// Tool is a tool as its module — or, for a role's tool, the mesh's records — describes it.
|
||||
type Tool struct {
|
||||
Module string
|
||||
Name string
|
||||
Description string
|
||||
Input json.RawMessage
|
||||
Seat bool
|
||||
Scope string
|
||||
Subjects []string
|
||||
}
|
||||
|
||||
// Listing is what the mesh could say about its tools; silence is named, never dropped (design 34 §3).
|
||||
type Listing struct {
|
||||
Tools []Tool
|
||||
NotAnswering []string
|
||||
}
|
||||
|
||||
// Seats is the seats and their verbs, so `<seat>.<verb>` resolves to the role.
|
||||
type Seats map[string]map[string]bool
|
||||
|
||||
func seatsIn(l *Listing) Seats {
|
||||
out := Seats{}
|
||||
for _, t := range l.Tools {
|
||||
if !t.Seat {
|
||||
continue
|
||||
}
|
||||
if out[t.Module] == nil {
|
||||
out[t.Module] = map[string]bool{}
|
||||
}
|
||||
out[t.Module][t.Name] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// toolKey is the key a call uses: a role's when the prefix is a seat declaring that verb.
|
||||
func toolKey(name string, seats Seats) string {
|
||||
if strings.HasPrefix(name, "seat:") {
|
||||
return name
|
||||
}
|
||||
dot := strings.Index(name, ".")
|
||||
if dot < 0 {
|
||||
return name
|
||||
}
|
||||
if seats[name[:dot]][name[dot+1:]] {
|
||||
return "seat:" + name
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// toolsOn is the flat catalogue (MESH_CONSOLE_FLAT=1): what announced itself on the bus, every
|
||||
// tool and seat verb, and the assignments with tools that did not (novox/hq ADR 0197).
|
||||
func toolsOn(conn *bus.Conn) (*Listing, error) {
|
||||
x, err := indexOn(conn, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return x.Listing, nil
|
||||
}
|
||||
|
||||
// callTool calls `<module>.<tool>[@<node>]`, on the subject the listing names for it when it names one.
|
||||
func callTool(conn *bus.Conn, key string, args any, seats Seats, l *Listing) (bus.Answered, error) {
|
||||
name, node, _ := strings.Cut(key, "@")
|
||||
if !strings.Contains(name, ".") {
|
||||
return bus.Answered{}, fmt.Errorf("%q does not name a tool: write <module>.<tool>, as `mesh tools` lists "+
|
||||
"them, or <module>.<tool>@<node> for the instance on one machine", key)
|
||||
}
|
||||
resolved := toolKey(name, seats)
|
||||
if node != "" {
|
||||
resolved += "@" + node
|
||||
}
|
||||
return conn.Ask(resolved, args, subjectListed(name, node, l))
|
||||
}
|
||||
|
||||
func subjectListed(name, node string, l *Listing) string {
|
||||
if l == nil {
|
||||
return ""
|
||||
}
|
||||
dot := strings.Index(name, ".")
|
||||
module, tool := name[:dot], name[dot+1:]
|
||||
for _, t := range l.Tools {
|
||||
if t.Module == module && t.Name == tool && !t.Seat {
|
||||
if len(t.Subjects) == 0 {
|
||||
return ""
|
||||
}
|
||||
if node == "" {
|
||||
return t.Subjects[0]
|
||||
}
|
||||
for _, s := range t.Subjects {
|
||||
if strings.HasSuffix(s, "."+node) {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var (
|
||||
// The bus's own two refusals, by their whole phrase. "authorization" alone also matched a tool's own
|
||||
// answer that merely mentions the word — the runtime refusing a state value with an Authorization
|
||||
// header (novox/hq ADR 0201) read as "this account may not call", which sent the reader the wrong way.
|
||||
refused = regexp.MustCompile(`(?i)permissions violation|authorization violation`)
|
||||
timedOut = regexp.MustCompile(`(?i)timeout`)
|
||||
)
|
||||
|
||||
// controllerSeat is the seat whose holder keeps what came of every call (novox/hq issue 265).
|
||||
const controllerSeat = "mesh-controller"
|
||||
|
||||
// whyItFailed says why a call failed, so the remedy is in the words.
|
||||
func whyItFailed(key string, err error) string {
|
||||
if err == nil {
|
||||
err = errors.New("failed")
|
||||
}
|
||||
msg := err.Error()
|
||||
switch {
|
||||
// By type, not text: a tool's own error that mentions 503 is the tool's answer (issue 1529).
|
||||
case errors.Is(err, bus.ErrNoResponders) || errors.Is(err, nats.ErrNoResponders):
|
||||
extra := ""
|
||||
if strings.HasPrefix(key, "seat:") {
|
||||
extra = ", or nothing holds that seat"
|
||||
}
|
||||
return "nothing serves " + key + ". The module may not be assigned to any machine, or it is down" +
|
||||
extra + " — `mesh tools` lists what answered."
|
||||
case refused.MatchString(msg):
|
||||
return "this account may not call " + key + ". What it may call was fixed when it was issued — a " +
|
||||
"person's by `operator issue`, the console's by its manifest."
|
||||
case timedOut.MatchString(msg):
|
||||
// **A timeout is not a failure** (novox/hq issue 265). Something took the call, so it may
|
||||
// still be running and may already have done what was asked: a push said this while it pushed.
|
||||
said := key + " did not answer within " + bus.RequestTimeout.String() + ". That is not a failure: " +
|
||||
"something is serving it, so it may still be running and may already have done what was " +
|
||||
"asked — check before repeating a call that changes the mesh."
|
||||
if strings.HasPrefix(strings.TrimPrefix(key, "seat:"), controllerSeat+".") {
|
||||
// The controller answers every call within seconds, with an id while it runs, so silence
|
||||
// from it is an answer lost on the way — which it keeps.
|
||||
said += " The controller answers every call within seconds, so its answer was lost on the " +
|
||||
"way; " + controllerSeat + ".calls lists its recent calls and what came of each."
|
||||
}
|
||||
return said
|
||||
}
|
||||
return key + " failed: " + msg
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
func connect(t *testing.T, module, node string) *bus.Conn {
|
||||
t.Helper()
|
||||
c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: module, Node: node})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Logf = func(string, ...any) {}
|
||||
t.Cleanup(c.Close)
|
||||
return c
|
||||
}
|
||||
|
||||
func post(t *testing.T, endpoint string, body string) map[string]any {
|
||||
t.Helper()
|
||||
res, err := http.Post(endpoint, "application/json", bytes.NewBufferString(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
raw, _ := io.ReadAll(res.Body)
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatalf("%d %s", res.StatusCode, raw)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// As node-tools, the runtime serves the bundles and is the console on loopback: the listing is what
|
||||
// the modules and the mesh's records answered, and a call reaches the module on the machine named.
|
||||
func TestTheConsoleListsAndCallsOverHTTP(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "desk", true, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "desk", false, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
nodeTools := connect(t, "node-tools", "desk")
|
||||
stop, err := runtime.Run(nodeTools, []runtime.Served{
|
||||
{Module: "alpha", Entrypoints: []string{mt.Fixture("many-alpha.serve.mjs")}},
|
||||
{Module: "beta", Entrypoints: []string{mt.Fixture("many-beta.serve.mjs")}},
|
||||
}, nil, (&mt.Logs{}).Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
|
||||
// The controller's records (ADR 0197): ghost declares tools on desk and nothing announces it.
|
||||
controller := connect(t, "mesh-controller", "")
|
||||
stopSeat, _ := controller.HandleSubject("mesh.seat.mesh-controller.tool.modules", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"ok": true, "output": `[{"module":"alpha","on":["desk"],"tools":true},` +
|
||||
`{"module":"beta","on":["desk"],"tools":true},{"module":"ghost","on":["desk"],"tools":true}]`}, nil
|
||||
})
|
||||
defer stopSeat()
|
||||
controller.Flush()
|
||||
|
||||
flat := NewSurface(nodeTools, "desk.node-tools")
|
||||
flat.Flat = true // the whole catalogue, as before ADR 0195: still reachable, no longer announced
|
||||
up, err := Serve(flat, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer up.Close()
|
||||
endpoint := "http://" + up.Address + "/mcp"
|
||||
|
||||
init := post(t, endpoint, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{}}}`)
|
||||
if !strings.Contains(init["result"].(map[string]any)["instructions"].(string), "reached as desk.node-tools") {
|
||||
t.Errorf("initialize: %v", init)
|
||||
}
|
||||
listed := post(t, endpoint, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`)["result"].(map[string]any)
|
||||
var names []string
|
||||
for _, x := range listed["tools"].([]any) {
|
||||
names = append(names, x.(map[string]any)["name"].(string))
|
||||
}
|
||||
if got := strings.Join(names, ","); got != "alpha.one,alpha.two,beta.five,beta.four,beta.three,node-shelf.clear,node-shelf.list" {
|
||||
t.Errorf("listed %s", got)
|
||||
}
|
||||
if got := listed["_meta"].(map[string]any)["notAnswering"]; len(got.([]any)) != 1 || got.([]any)[0] != "ghost on desk" {
|
||||
t.Errorf("not answering: %v", got)
|
||||
}
|
||||
for _, x := range listed["tools"].([]any) {
|
||||
tool := x.(map[string]any)
|
||||
if tool["name"] == "node-shelf.list" {
|
||||
schema := tool["inputSchema"].(map[string]any)
|
||||
if req, _ := schema["required"].([]any); len(req) != 1 || req[0] != "node" {
|
||||
t.Errorf("a node seat's verb does not require node: %v", schema)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
called := post(t, endpoint, `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"alpha.one","arguments":{"node":"desk"}}}`)
|
||||
content := called["result"].(map[string]any)["content"].([]any)
|
||||
var got map[string]any
|
||||
_ = json.Unmarshal([]byte(content[0].(map[string]any)["text"].(string)), &got)
|
||||
if got["alpha"] != float64(1) || content[1].(map[string]any)["text"] != "answered by desk" {
|
||||
t.Errorf("called: %v", called)
|
||||
}
|
||||
seat := post(t, endpoint, `{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"node-shelf.list","arguments":{"node":"desk"}}}`)
|
||||
if text := seat["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string); !strings.Contains(text, `"a"`) {
|
||||
t.Errorf("seat verb: %v", seat)
|
||||
}
|
||||
refused := post(t, endpoint, `{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"node-shelf.list","arguments":{}}}`)
|
||||
if refused["error"] == nil {
|
||||
t.Errorf("a node seat's verb was called without its machine: %v", refused)
|
||||
}
|
||||
absent := post(t, endpoint, `{"jsonrpc":"2.0","id":6,"method":"tools/call","params":{"name":"ghost.boo","arguments":{}}}`)
|
||||
result := absent["result"].(map[string]any)
|
||||
if result["isError"] != true || !strings.Contains(result["content"].([]any)[0].(map[string]any)["text"].(string), "nothing serves ghost.boo") {
|
||||
t.Errorf("an absent tool: %v", absent)
|
||||
}
|
||||
res, err := http.Post(endpoint, "application/json", strings.NewReader(`{"jsonrpc":"2.0","method":"notifications/initialized"}`))
|
||||
if err != nil || res.StatusCode != 202 {
|
||||
t.Errorf("a notification: %v %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheConsoleListensOnLoopbackAndNowhereElse(t *testing.T) {
|
||||
if _, err := Serve(NewSurface(nil, "x"), "0.0.0.0:0"); err == nil || !strings.Contains(err.Error(), "loopback and nowhere else") {
|
||||
t.Errorf("a non-loopback console was not refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq issue 554, reproduced: the laptop's console heard every runtime, then a round in which
|
||||
// nothing answered discovery in time — and every seat and module call was refused ("it answered
|
||||
// discovery 2s ago, not now") for the five seconds that round was kept, again and again. Now a call
|
||||
// is resolved against what the runtime announced last, says how old that is, and reaches the tool; the
|
||||
// empty round is logged once, is not kept, and the next question asks again.
|
||||
func TestACallAfterARoundThatHeardNothingUsesWhatWasHeardBefore(t *testing.T) {
|
||||
keptWas, patienceWas := IndexKept, announce.Patience
|
||||
IndexKept, announce.Patience = 0, 1200*time.Millisecond
|
||||
t.Cleanup(func() { IndexKept, announce.Patience = keptWas, patienceWas })
|
||||
mt.New(t)
|
||||
|
||||
// The tools answer on their own connection; discovery is answered on another, which goes quiet.
|
||||
tools := connect(t, "node-tools", "laptop")
|
||||
stopTool, err := tools.HandleSubject("mesh.mod.slack.tool.slack_check.laptop", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"ok": true}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopTool)
|
||||
stopSeat, err := tools.HandleSubject("mesh.seat.issue-tracker.tool.list", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"issues": []int{554}}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopSeat)
|
||||
tools.Flush()
|
||||
endpoints := []announce.Endpoint{
|
||||
{Kind: announce.KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Subject: "mesh.mod.slack.tool.slack_check.laptop"},
|
||||
{Kind: announce.KindSeat, Module: "issue-tracker", Seat: "issue-tracker", Tool: "list", Scope: "mesh", Node: "laptop",
|
||||
Schema: json.RawMessage(`{}`), Subject: "mesh.seat.issue-tracker.tool.list"},
|
||||
}
|
||||
announcing := connect(t, "node-tools", "laptop")
|
||||
stopAnnounce, err := announce.Serve(announcing, announce.Service{Name: "node-tools", ID: "laptop",
|
||||
Metadata: map[string]string{"node": "laptop"}}, func() []announce.Endpoint { return endpoints })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
announcing.Flush()
|
||||
|
||||
console := connect(t, "node-tools", "laptop-console")
|
||||
var mu sync.Mutex
|
||||
var logged []string
|
||||
console.Logf = func(f string, a ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
logged = append(logged, fmt.Sprintf(f, a...))
|
||||
}
|
||||
said := func(words string) int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range logged {
|
||||
if strings.Contains(l, words) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
surface := NewSurface(console, "laptop.node-tools")
|
||||
up, err := Serve(surface, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer up.Close()
|
||||
endpoint := "http://" + up.Address + "/mcp"
|
||||
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/slack.slack_check"}); isErr || strings.Contains(got, "announced") {
|
||||
t.Fatalf("while discovery hears the runtime: %s", got)
|
||||
}
|
||||
|
||||
// Discovery goes quiet: no PING arrives at all.
|
||||
stopAnnounce()
|
||||
announcing.Flush()
|
||||
for _, address := range []string{"laptop/slack.slack_check", "issue-tracker.list"} {
|
||||
got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": address})
|
||||
if isErr || !strings.Contains(got, "found from what node-tools on laptop announced") ||
|
||||
!strings.Contains(got, "ago: it did not answer the latest round of discovery") {
|
||||
t.Errorf("%s after a round that heard nothing: %s", address, got)
|
||||
}
|
||||
}
|
||||
if got, isErr := call(t, endpoint, "mesh_describe", map[string]any{"address": "issue-tracker.list"}); isErr || !strings.Contains(got, `"found"`) {
|
||||
t.Errorf("describe after a round that heard nothing: %s", got)
|
||||
}
|
||||
if n := said("discovery heard nothing at all"); n != 1 {
|
||||
t.Errorf("the empty round was logged %d time(s), want once: %v", n, logged)
|
||||
}
|
||||
// A runtime really gone: the call is sent on what it announced, and fails with the bus's own refusal,
|
||||
// saying where the address was found.
|
||||
stopTool()
|
||||
tools.Flush()
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/slack.slack_check"}); !isErr ||
|
||||
!strings.Contains(got, "nothing serves") || !strings.Contains(got, "found from what node-tools on laptop announced") {
|
||||
t.Errorf("a gone runtime's tool: %s", got)
|
||||
}
|
||||
surface.mu.Lock()
|
||||
kept := surface.idx != nil && !surface.idx.Discovery.Empty() && time.Since(surface.idxAt) < time.Second
|
||||
surface.mu.Unlock()
|
||||
if kept {
|
||||
t.Error("a round that heard nothing was kept as the answer")
|
||||
}
|
||||
|
||||
// Discovery answers again: the calls are plain again, and that is logged once.
|
||||
stopAgain, err := announce.Serve(announcing, announce.Service{Name: "node-tools", ID: "laptop",
|
||||
Metadata: map[string]string{"node": "laptop"}}, func() []announce.Endpoint { return endpoints })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopAgain)
|
||||
announcing.Flush()
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "issue-tracker.list"}); isErr || strings.Contains(got, "announced") {
|
||||
t.Errorf("once discovery hears the runtime again: %s", got)
|
||||
}
|
||||
if n := said("discovery hears every runtime again"); n != 1 {
|
||||
t.Errorf("the recovery was logged %d time(s): %v", n, logged)
|
||||
}
|
||||
}
|
||||
|
||||
// An address known only from an earlier answer says how old it is; one heard fresh on its machine is
|
||||
// plain, even where the same module is known only from an earlier answer on another.
|
||||
func TestAnAddressKnownOnlyFromAnEarlierAnswerSaysHowOld(t *testing.T) {
|
||||
at := time.Now().Add(-20 * time.Second)
|
||||
x := &index{Modules: map[string]*moduleInfo{"slack": {Module: "slack", On: []string{"laptop"},
|
||||
Tools: []Tool{{Module: "slack", Name: "slack_check"}}}}, Recorded: map[string][]string{},
|
||||
Stale: map[string]staleFrom{"mod:slack@laptop": {Runtime: "node-tools", Machine: "laptop", At: at}},
|
||||
Discovery: announce.Discovery{Pinged: 0}}
|
||||
got, err := resolve(x, "laptop/slack.slack_check")
|
||||
if err != nil || got.Stale == nil || !strings.Contains(got.Stale.say(), "announced 20s ago") {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
// A module heard fresh on one machine and only earlier on another: the fresh one is plain.
|
||||
x.Modules["slack"].On = []string{"desk", "laptop"}
|
||||
if got, err := resolve(x, "desk/slack.slack_check"); err != nil || got.Stale != nil {
|
||||
t.Errorf("a fresh machine's module marked stale: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/wire"
|
||||
)
|
||||
|
||||
// bodyLimit is the most a request body may be.
|
||||
const bodyLimit = 1 << 20
|
||||
|
||||
var loopback = map[string]bool{"127.0.0.1": true, "::1": true, "localhost": true, "[::1]": true}
|
||||
|
||||
// Listening is a console that listens: where, with the port the machine gave, and how to stop it.
|
||||
type Listening struct {
|
||||
Address string
|
||||
Close func() error
|
||||
}
|
||||
|
||||
// Serve listens on host:port, refused unless the host is loopback — said before binding, so a
|
||||
// console that would open to a network is a startup failure (ADR 0152).
|
||||
func Serve(s *Surface, listen string) (*Listening, error) {
|
||||
at := strings.LastIndex(listen, ":")
|
||||
if at < 0 {
|
||||
return nil, fmt.Errorf("%q is not host:port", listen)
|
||||
}
|
||||
host, portText := listen[:at], listen[at+1:]
|
||||
if !loopback[host] {
|
||||
return nil, fmt.Errorf(`the console listens on loopback and nowhere else (novox/hq ADR 0152): %q is not this `+
|
||||
"machine's own address — whoever is on the machine owns the mesh there, and nobody else may reach this", host)
|
||||
}
|
||||
port, err := strconv.Atoi(portText)
|
||||
if err != nil || port < 0 || port > 65535 {
|
||||
return nil, fmt.Errorf("%q is not a port", portText)
|
||||
}
|
||||
ln, err := net.Listen("tcp", net.JoinHostPort(strings.Trim(host, "[]"), portText))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Every request's context ends when the console is closed, so a mesh_wait in flight ends at once rather
|
||||
// than holding a stop or a restart for up to its ten minutes (the review of issue 745).
|
||||
base, cancel := context.WithCancel(context.Background())
|
||||
server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route(w, r, s) }),
|
||||
BaseContext: func(net.Listener) context.Context { return base }}
|
||||
go func() { _ = server.Serve(ln) }()
|
||||
bound := ln.Addr().(*net.TCPAddr).Port
|
||||
return &Listening{Address: host + ":" + strconv.Itoa(bound), Close: func() error {
|
||||
cancel()
|
||||
// A backstop: what is still answering after the cancel is cut off, never waited on for long.
|
||||
ctx, done := context.WithTimeout(context.Background(), ShutdownWithin)
|
||||
defer done()
|
||||
err := server.Shutdown(ctx)
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return server.Close()
|
||||
}
|
||||
return err
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// ShutdownWithin is how long closing the console waits for the requests still being answered after their
|
||||
// context was cancelled.
|
||||
var ShutdownWithin = 5 * time.Second
|
||||
|
||||
func route(w http.ResponseWriter, r *http.Request, s *Surface) {
|
||||
switch r.URL.Path {
|
||||
case "/":
|
||||
w.Header().Set("content-type", "text/plain; charset=utf-8")
|
||||
_, _ = io.WriteString(w, "the mesh's console: MCP over HTTP at POST /mcp (novox/hq design 34)\n")
|
||||
return
|
||||
case "/mcp":
|
||||
default:
|
||||
writeJSON(w, 404, map[string]any{"error": "the console serves /mcp and nothing else"})
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
case http.MethodDelete:
|
||||
w.WriteHeader(204) // no session to end
|
||||
return
|
||||
default:
|
||||
w.Header().Set("allow", "POST, DELETE")
|
||||
w.WriteHeader(405)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, bodyLimit+1))
|
||||
if err != nil || len(body) > bodyLimit {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("the request is larger than %d bytes", bodyLimit)
|
||||
}
|
||||
writeJSON(w, 413, map[string]any{"jsonrpc": "2.0", "id": nil, "error": map[string]any{"code": -32600, "message": err.Error()}})
|
||||
return
|
||||
}
|
||||
trimmed := strings.TrimSpace(string(body))
|
||||
if strings.HasPrefix(trimmed, "[") {
|
||||
var batch []Request
|
||||
if err := json.Unmarshal(body, &batch); err != nil {
|
||||
writeJSON(w, 400, map[string]any{"jsonrpc": "2.0", "id": nil, "error": map[string]any{"code": -32700, "message": "the body is not JSON"}})
|
||||
return
|
||||
}
|
||||
replies := []*Reply{}
|
||||
for _, req := range batch {
|
||||
if reply := s.HandleContext(r.Context(), req); reply != nil {
|
||||
replies = append(replies, reply)
|
||||
}
|
||||
}
|
||||
if len(replies) == 0 {
|
||||
w.WriteHeader(202)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, replies)
|
||||
return
|
||||
}
|
||||
var req Request
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
writeJSON(w, 400, map[string]any{"jsonrpc": "2.0", "id": nil, "error": map[string]any{"code": -32700, "message": "the body is not JSON"}})
|
||||
return
|
||||
}
|
||||
reply := s.HandleContext(r.Context(), req)
|
||||
if reply == nil {
|
||||
w.WriteHeader(202)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, reply)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, body any) {
|
||||
text, err := wire.Marshal(body)
|
||||
if err != nil {
|
||||
text, _ = json.Marshal(map[string]any{"error": errors.New("unencodable answer").Error()})
|
||||
}
|
||||
w.Header().Set("content-type", "application/json; charset=utf-8")
|
||||
w.Header().Set("content-length", strconv.Itoa(len(text)))
|
||||
w.WriteHeader(status)
|
||||
_, _ = w.Write(text)
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0367 §5: the tool runner's manifest names the families a session waits on today, one by one,
|
||||
// and the asker's `decided` for the claude-code module alone. Adding one is a reviewed change to this file
|
||||
// (§6), so this list changes with it. The delivery's transitions are waited on under the gate's name alone (ADR 0450 §8, issues 1350
|
||||
// and 1406): today's module is forgotten.
|
||||
func TestTheManifestNamesTheFirstFamiliesOneByOne(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Waits []string `json:"waits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"mesh-knowledge.reviewed", "mesh-delivery-gate.transition", "gitea.forge.status",
|
||||
"mesh-controller.checked", "operator-channel.decided.claude-code"}
|
||||
if m.Module != "node-tools" || !reflect.DeepEqual(m.Waits, want) {
|
||||
t.Errorf("%s waits on %v, want %v", m.Module, m.Waits, want)
|
||||
}
|
||||
for _, w := range m.Waits {
|
||||
if strings.ContainsAny(w, "*>") {
|
||||
t.Errorf("a family named by a wildcard: %s", w)
|
||||
}
|
||||
}
|
||||
if len(m.Consumes) > 0 {
|
||||
t.Errorf("the tool runner consumes %v: a wait is no consumer (ADR 0367 §2)", m.Consumes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
// Package console is the mesh's tools as an MCP server on a machine's loopback (novox/hq design 34,
|
||||
// ADR 0152, ADR 0175 §6): the Go port of node-tools' http.ts, mcp.ts and client.ts. A thin adapter:
|
||||
// every tool listed is one a module answered for, the schema is the module's, the answer the module's.
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// Protocol is the MCP version spoken to an agent host.
|
||||
const Protocol = "2025-03-26"
|
||||
|
||||
// ListingKept is how long a fetched tool list is kept before the modules are asked again.
|
||||
var ListingKept = 30 * time.Second
|
||||
|
||||
// Request is one JSON-RPC message from a host.
|
||||
type Request struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Method string `json:"method"`
|
||||
Params json.RawMessage `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
// Reply is one JSON-RPC answer.
|
||||
type Reply struct {
|
||||
JSONRPC string `json:"jsonrpc"`
|
||||
ID json.RawMessage `json:"id"`
|
||||
Result any `json:"result,omitempty"`
|
||||
Error *RPCError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// RPCError is a protocol-level refusal.
|
||||
type RPCError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// Surface answers MCP requests over one bus connection, as one account.
|
||||
type Surface struct {
|
||||
conn *bus.Conn
|
||||
who string
|
||||
mu sync.Mutex
|
||||
known *Listing
|
||||
at time.Time
|
||||
idx *index
|
||||
idxAt time.Time
|
||||
// runtimes is every runtime discovery has heard, as it last heard it (mesh_runtimes).
|
||||
runtimes map[string]*runtimeSeen
|
||||
// roundSaid is what the last round logged was missing, so a round is logged once (novox/hq issue 554).
|
||||
roundSaid string
|
||||
// Flat announces the whole catalogue, as the console did before ADR 0195: for a person reading
|
||||
// it or a client that wants it. Off by default; MESH_CONSOLE_FLAT=1 turns it on.
|
||||
Flat bool
|
||||
}
|
||||
|
||||
// NewSurface is the surface over a connection, as `who`.
|
||||
func NewSurface(conn *bus.Conn, who string) *Surface {
|
||||
return &Surface{conn: conn, who: who, Flat: os.Getenv("MESH_CONSOLE_FLAT") == "1"}
|
||||
}
|
||||
|
||||
func (s *Surface) listing() (*Listing, error) {
|
||||
s.mu.Lock()
|
||||
if s.known != nil && time.Since(s.at) <= ListingKept {
|
||||
l := s.known
|
||||
s.mu.Unlock()
|
||||
return l, nil
|
||||
}
|
||||
s.mu.Unlock()
|
||||
l, err := toolsOn(s.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.known, s.at = l, time.Now()
|
||||
s.mu.Unlock()
|
||||
return l, nil
|
||||
}
|
||||
|
||||
func isNotification(id json.RawMessage) bool {
|
||||
t := strings.TrimSpace(string(id))
|
||||
return t == "" || t == "null"
|
||||
}
|
||||
|
||||
func answer(id json.RawMessage, result any) *Reply {
|
||||
return &Reply{JSONRPC: "2.0", ID: idOrNull(id), Result: result}
|
||||
}
|
||||
|
||||
func refuse(id json.RawMessage, code int, message string) *Reply {
|
||||
return &Reply{JSONRPC: "2.0", ID: idOrNull(id), Error: &RPCError{Code: code, Message: message}}
|
||||
}
|
||||
|
||||
func idOrNull(id json.RawMessage) json.RawMessage {
|
||||
if isNotification(id) {
|
||||
return json.RawMessage("null")
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// Handle answers one request; nil for a notification, which expects none.
|
||||
func (s *Surface) Handle(r Request) *Reply { return s.HandleContext(context.Background(), r) }
|
||||
|
||||
// HandleContext answers one request for as long as ctx lasts: a wait whose client hung up or cancelled
|
||||
// ends at once, its subscription closed (the review of issue 745).
|
||||
func (s *Surface) HandleContext(ctx context.Context, r Request) *Reply {
|
||||
notification := isNotification(r.ID)
|
||||
switch r.Method {
|
||||
case "initialize":
|
||||
return answer(r.ID, map[string]any{
|
||||
"protocolVersion": Protocol,
|
||||
"capabilities": map[string]any{"tools": map[string]any{}},
|
||||
"serverInfo": map[string]any{"name": "mesh", "version": "1"},
|
||||
"instructions": s.instructions(),
|
||||
})
|
||||
case "notifications/initialized":
|
||||
return nil
|
||||
case "ping":
|
||||
if notification {
|
||||
return nil
|
||||
}
|
||||
return answer(r.ID, map[string]any{})
|
||||
case "tools/list":
|
||||
if !s.Flat {
|
||||
return answer(r.ID, map[string]any{"tools": discovery()})
|
||||
}
|
||||
l, err := s.listing()
|
||||
if err != nil {
|
||||
return refuse(r.ID, -32603, "the mesh's discovery failed: "+err.Error())
|
||||
}
|
||||
tools := make([]map[string]any, 0, len(l.Tools))
|
||||
inputs := map[string]map[string]json.RawMessage{} // module → tool → input, asked once per module
|
||||
for i, t := range l.Tools {
|
||||
if !t.Seat && len(t.Input) == 0 {
|
||||
if inputs[t.Module] == nil {
|
||||
inputs[t.Module] = toolInputs(s.conn, t.Module)
|
||||
}
|
||||
l.Tools[i].Input = inputs[t.Module][t.Name]
|
||||
t = l.Tools[i]
|
||||
}
|
||||
var schema map[string]any
|
||||
switch {
|
||||
case t.Seat && t.Scope != "node":
|
||||
schema = asSchema(t.Input)
|
||||
case t.Seat:
|
||||
schema = withNode(asSchema(t.Input), "the machine whose seat answers; required, the seat is held once per machine", true)
|
||||
default:
|
||||
schema = withNode(asSchema(t.Input), "", false)
|
||||
}
|
||||
description := t.Description
|
||||
if description == "" {
|
||||
description = t.Name + ", served by " + t.Module
|
||||
}
|
||||
tools = append(tools, map[string]any{"name": t.Module + "." + t.Name, "description": description, "inputSchema": schema})
|
||||
}
|
||||
return answer(r.ID, map[string]any{"tools": tools, "_meta": map[string]any{"notAnswering": l.NotAnswering}})
|
||||
case "tools/call":
|
||||
var p struct {
|
||||
Name string `json:"name"`
|
||||
Arguments map[string]any `json:"arguments"`
|
||||
}
|
||||
_ = json.Unmarshal(r.Params, &p)
|
||||
if isDiscovery(p.Name) {
|
||||
if p.Arguments == nil {
|
||||
p.Arguments = map[string]any{}
|
||||
}
|
||||
return answer(r.ID, s.discover(ctx, p.Name, p.Arguments))
|
||||
}
|
||||
args := map[string]any{}
|
||||
for k, v := range p.Arguments {
|
||||
args[k] = v
|
||||
}
|
||||
l, _ := s.listing()
|
||||
var roles Seats
|
||||
if l != nil {
|
||||
roles = seatsIn(l)
|
||||
}
|
||||
bare, _, _ := strings.Cut(p.Name, "@")
|
||||
isSeatVerb := roles != nil && strings.HasPrefix(toolKey(bare, roles), "seat:")
|
||||
nodeScoped := false
|
||||
if isSeatVerb && l != nil {
|
||||
for _, t := range l.Tools {
|
||||
if t.Seat && t.Scope == "node" && t.Module+"."+t.Name == bare {
|
||||
nodeScoped = true
|
||||
}
|
||||
}
|
||||
}
|
||||
takesNode := !isSeatVerb || nodeScoped
|
||||
node := ""
|
||||
if takesNode {
|
||||
if n, ok := args["node"].(string); ok {
|
||||
node = n
|
||||
}
|
||||
delete(args, "node")
|
||||
}
|
||||
if nodeScoped && node == "" && !strings.Contains(p.Name, "@") {
|
||||
return refuse(r.ID, -32602, p.Name+" is a machine's seat's verb: name the machine with `node`")
|
||||
}
|
||||
name := p.Name
|
||||
if node != "" && !strings.Contains(p.Name, "@") {
|
||||
name = p.Name + "@" + node
|
||||
}
|
||||
got, err := callTool(s.conn, name, args, roles, l)
|
||||
if err != nil {
|
||||
return answer(r.ID, map[string]any{
|
||||
"content": []map[string]any{{"type": "text", "text": whyItFailed(name, err)}},
|
||||
"isError": true,
|
||||
})
|
||||
}
|
||||
content := []map[string]any{{"type": "text", "text": pretty(got.Result)}}
|
||||
if got.Node != "" {
|
||||
content = append(content, map[string]any{"type": "text", "text": "answered by " + got.Node})
|
||||
}
|
||||
return answer(r.ID, map[string]any{"content": content})
|
||||
}
|
||||
if notification {
|
||||
return nil
|
||||
}
|
||||
return refuse(r.ID, -32601, "mesh's MCP surface has no "+r.Method)
|
||||
}
|
||||
|
||||
// pretty is a module's answer as JSON text, indented as JSON.stringify(result, null, 2) writes it.
|
||||
func pretty(raw json.RawMessage) string {
|
||||
if len(raw) == 0 {
|
||||
return "null"
|
||||
}
|
||||
var v any
|
||||
if json.Unmarshal(raw, &v) != nil {
|
||||
return string(raw)
|
||||
}
|
||||
b, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return string(raw)
|
||||
}
|
||||
return strings.NewReplacer(`<`, "<", `>`, ">", `&`, "&").Replace(string(b))
|
||||
}
|
||||
|
||||
// asSchema is a module's declared input as a JSON schema: wrapped when it is a bare map of
|
||||
// properties, passed through when it is a schema, empty when nothing was declared.
|
||||
func asSchema(raw json.RawMessage) map[string]any {
|
||||
var given map[string]any
|
||||
if json.Unmarshal(raw, &given) != nil || given == nil {
|
||||
return map[string]any{"type": "object", "properties": map[string]any{}}
|
||||
}
|
||||
if given["type"] == "object" {
|
||||
return given
|
||||
}
|
||||
if _, has := given["properties"]; has {
|
||||
return given
|
||||
}
|
||||
if len(given) == 0 {
|
||||
return map[string]any{"type": "object", "properties": map[string]any{}}
|
||||
}
|
||||
return map[string]any{"type": "object", "properties": given}
|
||||
}
|
||||
|
||||
// withNode adds the optional — or, for a node seat, required — `node` argument (ADR 0159).
|
||||
func withNode(schema map[string]any, description string, required bool) map[string]any {
|
||||
properties := map[string]any{}
|
||||
if p, ok := schema["properties"].(map[string]any); ok {
|
||||
for k, v := range p {
|
||||
properties[k] = v
|
||||
}
|
||||
}
|
||||
if _, has := properties["node"]; !has {
|
||||
if description == "" {
|
||||
description = "the machine to ask, when this module runs on several; else whichever answers, and the answer says which"
|
||||
}
|
||||
properties["node"] = map[string]any{"type": "string", "description": description}
|
||||
}
|
||||
out := map[string]any{}
|
||||
for k, v := range schema {
|
||||
out[k] = v
|
||||
}
|
||||
out["type"] = "object"
|
||||
out["properties"] = properties
|
||||
if required {
|
||||
have := []any{}
|
||||
if r, ok := schema["required"].([]any); ok {
|
||||
have = r
|
||||
}
|
||||
hasNode := false
|
||||
for _, x := range have {
|
||||
hasNode = hasNode || x == "node"
|
||||
}
|
||||
if !hasNode {
|
||||
have = append(have, "node")
|
||||
}
|
||||
out["required"] = have
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// instructions is what an agent host is told about this surface when it connects.
|
||||
func (s *Surface) instructions() string {
|
||||
if s.Flat {
|
||||
return "These are the tools of a Novox mesh, reached as " + s.who + ". Every call goes to the module " +
|
||||
"that serves it; what may be called was fixed when this account was issued, so a " +
|
||||
"refusal means the account, not the tool. The list is what the running modules " +
|
||||
"answered, plus every role's tools from the mesh's records — the mesh's own verbs " +
|
||||
"(mesh-controller.status, .push, .assign …) among them; a module that did not answer " +
|
||||
"is named in the list's _meta and can still be called by <module>.<tool>."
|
||||
}
|
||||
return "The tools of a Novox mesh, reached as " + s.who + ", found by address rather than listed " +
|
||||
"whole (novox/hq ADR 0195). mesh_overview shows the mesh's seats and machines; mesh_machine one " +
|
||||
"machine's seats and modules; mesh_search finds a tool by words; mesh_describe gives one tool's " +
|
||||
"arguments; mesh_call calls it; mesh_runtimes says which machines' runtimes answered discovery, how fast " +
|
||||
"and how much — ask it when an address is said to be missing or on another machine; mesh_wait waits on the " +
|
||||
"bus for an event instead of polling (novox/hq ADR 0367). " + grammar + " What may be called was fixed when this account " +
|
||||
"was issued, so a refusal means the account, not the tool."
|
||||
}
|
||||
|
||||
// toolInputs is every tool's input schema of one module, asked of a runtime that serves it.
|
||||
func toolInputs(conn *bus.Conn, module string) map[string]json.RawMessage {
|
||||
out := map[string]json.RawMessage{}
|
||||
got, err := conn.Ask(module+".tools", map[string]any{}, "")
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
var answer struct {
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
Input json.RawMessage `json:"input"`
|
||||
} `json:"tools"`
|
||||
}
|
||||
if json.Unmarshal(got.Result, &answer) == nil {
|
||||
for _, t := range answer.Tools {
|
||||
out[t.Name] = t.Input
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq issue 244: `node` is taken out only where the address names the machine. A seat held once
|
||||
// for the mesh takes the machine it acts on as `node`, and is described and called with it — on
|
||||
// 2026-10-05 the controller's push was described as taking nothing, and a push naming one machine
|
||||
// ran as a push of every machine behind.
|
||||
func TestAMeshSeatsNodeIsItsOwnArgument(t *testing.T) {
|
||||
push := Tool{Module: "mesh-controller", Name: "push", Seat: true, Scope: "mesh",
|
||||
Input: json.RawMessage(`{"type":"object","properties":{"node":{"type":"string"},"behind":{"type":"string"}}}`)}
|
||||
mesh := target{Address: "mesh-controller.push", Tool: push, Seat: true}
|
||||
|
||||
props := schemaAsPassed(mesh)["properties"].(map[string]any)
|
||||
if _, has := props["node"]; !has {
|
||||
t.Fatalf("a mesh seat's verb was described without its node: %v", props)
|
||||
}
|
||||
sent, err := argumentsAsSent(mesh, map[string]any{"node": "g1"})
|
||||
if err != nil || sent["node"] != "g1" {
|
||||
t.Fatalf("a mesh seat's verb was called without the machine it was given: %v %v", sent, err)
|
||||
}
|
||||
|
||||
// An argument the seat's verb does not declare is refused, naming it, never sent to be dropped.
|
||||
if _, err := argumentsAsSent(mesh, map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), "machine") {
|
||||
t.Fatalf("an undeclared argument was sent: %v", err)
|
||||
}
|
||||
|
||||
// A machine's seat: the address names the machine, so `node` is out of the schema; given the same
|
||||
// machine again it is taken out, given another it is refused.
|
||||
shelf := Tool{Module: "node-shelf", Name: "list", Seat: true, Scope: "node",
|
||||
Input: json.RawMessage(`{"type":"object","properties":{"limit":{"type":"string"}}}`)}
|
||||
onDesk := target{Address: "desk/node-shelf.list", Node: "desk", Tool: shelf, Seat: true}
|
||||
if _, has := schemaAsPassed(onDesk)["properties"].(map[string]any)["node"]; has {
|
||||
t.Fatal("a machine's seat was described with a node the address already names")
|
||||
}
|
||||
if sent, err := argumentsAsSent(onDesk, map[string]any{"node": "desk", "limit": "3"}); err != nil || sent["node"] != nil || sent["limit"] != "3" {
|
||||
t.Fatalf("the same machine named twice: %v %v", sent, err)
|
||||
}
|
||||
if _, err := argumentsAsSent(onDesk, map[string]any{"node": "bench"}); err == nil || !strings.Contains(err.Error(), "bench") {
|
||||
t.Fatalf("another machine in the arguments than in the address was dropped: %v", err)
|
||||
}
|
||||
|
||||
// A module's tool is its own: what it takes beyond node is the module's to judge.
|
||||
beta := target{Address: "desk/beta.three", Node: "desk", Tool: Tool{Module: "beta", Name: "three"}}
|
||||
if sent, err := argumentsAsSent(beta, map[string]any{"node": "desk", "verbose": true}); err != nil || sent["verbose"] != true || sent["node"] != nil {
|
||||
t.Fatalf("a module's tool: %v %v", sent, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// A tool's own error that mentions 503 is the tool's answer, not a routing fault (issue 1529).
|
||||
func TestAToolsOwn503IsNotNothingServes(t *testing.T) {
|
||||
got := whyItFailed("scanner.scan", errors.New("the scanner refused the scan job: 503 Service Unavailable"))
|
||||
if strings.Contains(got, "nothing serves") || !strings.HasPrefix(got, "scanner.scan failed: ") || !strings.Contains(got, "503") {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARealNoRespondersStillSaysNothingServes(t *testing.T) {
|
||||
for _, err := range []error{bus.ErrNoResponders, fmt.Errorf("ask: %w", bus.ErrNoResponders), nats.ErrNoResponders} {
|
||||
if got := whyItFailed("a.b", err); !strings.HasPrefix(got, "nothing serves a.b") {
|
||||
t.Fatalf("%v: got %q", err, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
)
|
||||
|
||||
// **Issue 287**: mesh-delivery is the delivery's seat, held once for the mesh, and the module holding it,
|
||||
// which answers the seat's verbs with tools of the same names and has tools of its own. Discovery put the
|
||||
// module's tool and the seat's verb under one key, so the seat was listed with no verb; and every address
|
||||
// with the name resolved to the seat — `mesh-delivery.deliveries` and `novox/mesh-delivery.delivery_status`
|
||||
// alike were refused, the seat's verbs and the module's tools both out of reach.
|
||||
func TestASeatAndAModuleOfOneNameAreEachReached(t *testing.T) {
|
||||
var endpoints []announce.Endpoint
|
||||
for _, tool := range []string{"deliveries", "delivery_status"} {
|
||||
endpoints = append(endpoints, announce.Endpoint{Kind: announce.KindTool, Module: "mesh-delivery", Tool: tool,
|
||||
Node: "anchor", Subject: "mesh.mod.mesh-delivery.tool." + tool + ".anchor"})
|
||||
}
|
||||
for _, verb := range []string{"deliveries", "stalled"} {
|
||||
endpoints = append(endpoints, announce.Endpoint{Kind: announce.KindSeat, Module: "mesh-delivery", Seat: "mesh-delivery",
|
||||
Tool: verb, Scope: "mesh", Subject: "mesh.seat.mesh-delivery.tool." + verb})
|
||||
}
|
||||
x, _, _ := indexOfEndpoints(endpoints)
|
||||
|
||||
s := x.seat("mesh-delivery")
|
||||
if s == nil || len(s.Verbs) != 2 {
|
||||
t.Fatalf("the seat is listed with verbs %v, not its two", s)
|
||||
}
|
||||
if m := x.Modules["mesh-delivery"]; m == nil || len(m.Tools) != 2 {
|
||||
t.Fatalf("the module is listed with tools %v, not its two", m)
|
||||
}
|
||||
|
||||
for address, want := range map[string]struct {
|
||||
seat bool
|
||||
key string
|
||||
}{
|
||||
"mesh-delivery.deliveries": {true, "seat:mesh-delivery.deliveries"},
|
||||
"mesh-delivery.stalled": {true, "seat:mesh-delivery.stalled"},
|
||||
"anchor/mesh-delivery.delivery_status": {false, "mesh-delivery.delivery_status@anchor"},
|
||||
"anchor/mesh-delivery.deliveries": {false, "mesh-delivery.deliveries@anchor"},
|
||||
} {
|
||||
got, err := resolve(x, address)
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", address, err)
|
||||
continue
|
||||
}
|
||||
if got.Seat != want.seat || got.Key != want.key {
|
||||
t.Errorf("%s resolves to %+v, not the %s", address, got, want.key)
|
||||
}
|
||||
}
|
||||
// A module tool of the name without a machine: the module keeps state on its machine, so it says so.
|
||||
if _, err := resolve(x, "mesh-delivery.delivery_status"); err == nil || !strings.Contains(err.Error(), "<node>/") {
|
||||
t.Errorf("the module's tool without a machine: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A refusal is said only for the bus's own refusals; a tool's answer that mentions authorization is the
|
||||
// tool's answer, not the account's.
|
||||
func TestARefusalIsSaidOnlyForTheBussOwn(t *testing.T) {
|
||||
for msg, refusal := range map[string]bool{
|
||||
`nats: Permissions Violation for Publish to "mesh.mod.x.tool.y"`: true,
|
||||
"nats: Authorization Violation": true,
|
||||
`claude-code's servers.all.x carries a field "Authorization", which names a credential`: false,
|
||||
} {
|
||||
got := strings.HasPrefix(whyItFailed("x.y", errors.New(msg)), "this account may not call")
|
||||
if got != refusal {
|
||||
t.Errorf("%q read as a refusal: %v, want %v", msg, got, refusal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A timeout says it is not a failure** (novox/hq issue 265): a push that outlasted the wait had
|
||||
// pushed, and the caller read "did not answer in time". And from the controller, where its answer is.
|
||||
func TestATimeoutIsNotAFailure(t *testing.T) {
|
||||
got := whyItFailed("seat:mesh-controller.push", errors.New("timeout"))
|
||||
for _, want := range []string{"not a failure", "may already have done", "mesh-controller.calls"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("a controller timeout does not say %q: %s", want, got)
|
||||
}
|
||||
}
|
||||
if other := whyItFailed("novox/postgres.postgres_list_databases", errors.New("timeout")); !strings.Contains(other, "not a failure") ||
|
||||
strings.Contains(other, "calls") {
|
||||
t.Errorf("a module's timeout: %s", other)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
package console
|
||||
|
||||
// What discovery heard from each runtime (2026-10-05): the console said a module ran nowhere when the
|
||||
// largest runtime's answer arrived after the window closed, and nothing showed that it had. Every
|
||||
// round of discovery is remembered per runtime, so a runtime heard before and not now is named, and
|
||||
// `mesh_runtimes` says who answered, how fast and how much.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
)
|
||||
|
||||
// RuntimeForgotten is how long a runtime heard once is expected again: within it, its silence is said
|
||||
// wherever an address it might answer is asked for; after it, it is listed only by mesh_runtimes.
|
||||
var RuntimeForgotten = 15 * time.Minute
|
||||
|
||||
// runtimeSeen is one runtime instance as discovery last heard it.
|
||||
type runtimeSeen struct {
|
||||
Name string
|
||||
ID string
|
||||
Machine string
|
||||
LastHeard time.Time
|
||||
Took time.Duration
|
||||
Bytes int
|
||||
Modules int
|
||||
Tools int
|
||||
SeatVerbs int
|
||||
Shortened string
|
||||
Answered bool // in the latest round
|
||||
Why string // why not, when it did not
|
||||
// Reused is true when its PING said its announcement was unchanged, so it was not fetched again.
|
||||
Reused bool
|
||||
// Last is its last whole answer: what a round that does not hear it uses instead, and what an
|
||||
// unchanged digest stands for (novox/hq issue 554).
|
||||
Last announce.Heard
|
||||
}
|
||||
|
||||
func runtimeKey(name, id string) string { return name + "/" + id }
|
||||
|
||||
// seenOf is what one answer announced, counted.
|
||||
func seenOf(h announce.Heard, at time.Time) *runtimeSeen {
|
||||
modules := map[string]bool{}
|
||||
tools := map[string]bool{}
|
||||
verbs := map[string]bool{}
|
||||
for _, e := range announce.Endpoints(h.Info) {
|
||||
switch e.Kind {
|
||||
case announce.KindTool:
|
||||
modules[e.Module] = true
|
||||
tools[e.Module+"."+e.Tool] = true
|
||||
case announce.KindSeat:
|
||||
verbs[e.Seat+"."+e.Tool] = true
|
||||
}
|
||||
}
|
||||
return &runtimeSeen{Name: h.Info.Name, ID: h.Info.ID, Machine: h.Machine(), LastHeard: at, Took: h.Took,
|
||||
Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true,
|
||||
Reused: h.Reused, Last: h}
|
||||
}
|
||||
|
||||
// held is the answer this console holds for an instance at this digest (announce.Known): an instance
|
||||
// whose announcement is unchanged is not asked for it again (novox/hq issue 554).
|
||||
func (s *Surface) held(name, id, digest string) (announce.Heard, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
r := s.runtimes[runtimeKey(name, id)]
|
||||
if r == nil || r.Last.Digest == "" || r.Last.Digest != digest || r.Last.Info.Name == "" {
|
||||
return announce.Heard{}, false
|
||||
}
|
||||
h := r.Last
|
||||
h.Took, h.Reused = 0, false
|
||||
return h, true
|
||||
}
|
||||
|
||||
// before is the last answer of every runtime this round did not hear, heard within RuntimeForgotten
|
||||
// and not since restarted under another instance (novox/hq issue 554).
|
||||
func (s *Surface) before(d announce.Discovery) []earlier {
|
||||
heard := map[string]bool{}
|
||||
running := map[string]bool{}
|
||||
for _, h := range d.Heard {
|
||||
heard[runtimeKey(h.Info.Name, h.Info.ID)] = true
|
||||
if m := h.Machine(); m != "" {
|
||||
running[h.Info.Name+"@"+m] = true
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []earlier
|
||||
for k, r := range s.runtimes {
|
||||
if heard[k] || r.Last.Info.Name == "" || r.LastHeard.IsZero() || time.Since(r.LastHeard) > RuntimeForgotten {
|
||||
continue
|
||||
}
|
||||
if r.Machine != "" && running[r.Name+"@"+r.Machine] {
|
||||
continue // restarted: the new instance answered
|
||||
}
|
||||
out = append(out, earlier{Heard: r.Last, At: r.LastHeard})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return runtimeKey(out[i].Heard.Info.Name, out[i].Heard.Info.ID) < runtimeKey(out[j].Heard.Info.Name, out[j].Heard.Info.ID)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// sayRound logs a round of discovery that did not hear every runtime — once, when what it missed
|
||||
// changes, and once when every runtime is heard again — so the next case leaves evidence in the
|
||||
// journal (novox/hq issue 554). Called with s.mu held, after remember.
|
||||
func (s *Surface) sayRound(x *index) {
|
||||
d := x.Discovery
|
||||
var missed []string
|
||||
for _, u := range x.Unheard {
|
||||
who := u.Runtime
|
||||
if u.Machine != "" {
|
||||
who += " on " + u.Machine
|
||||
}
|
||||
missed = append(missed, who)
|
||||
}
|
||||
sort.Strings(missed)
|
||||
shape := strings.Join(missed, ", ")
|
||||
if d.Empty() {
|
||||
shape = "empty|" + shape
|
||||
}
|
||||
if shape == s.roundSaid {
|
||||
return
|
||||
}
|
||||
s.roundSaid = shape
|
||||
reused := 0
|
||||
for _, h := range d.Heard {
|
||||
if h.Reused {
|
||||
reused++
|
||||
}
|
||||
}
|
||||
logf := log.Printf
|
||||
if s.conn != nil && s.conn.Logf != nil {
|
||||
logf = s.conn.Logf
|
||||
}
|
||||
window := fmt.Sprintf("waited %s (window %s, patience %s)", d.Waited.Round(time.Millisecond), announce.Window, announce.Patience)
|
||||
held := ""
|
||||
if len(x.Earlier) > 0 {
|
||||
held = fmt.Sprintf("; what %d of them announced earlier is used, with its age", len(x.Earlier))
|
||||
}
|
||||
switch {
|
||||
case d.Empty():
|
||||
logf("[mesh-tools] discovery heard nothing at all: no PING arrived; %s; not heard: %s%s; asking again at the next question",
|
||||
window, orNone(missed), held)
|
||||
case len(missed) > 0:
|
||||
logf("[mesh-tools] discovery did not hear every runtime: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s; not heard: %s%s",
|
||||
d.Pinged, len(d.Heard), reused, window, shape, held)
|
||||
default:
|
||||
logf("[mesh-tools] discovery hears every runtime again: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s",
|
||||
d.Pinged, len(d.Heard), reused, window)
|
||||
}
|
||||
}
|
||||
|
||||
func orNone(xs []string) string {
|
||||
if len(xs) == 0 {
|
||||
return "none heard before either"
|
||||
}
|
||||
return strings.Join(xs, ", ")
|
||||
}
|
||||
|
||||
// remember records a round of discovery and adds to the index every runtime heard lately that did not
|
||||
// answer this time. Called with s.mu held.
|
||||
func (s *Surface) remember(x *index) {
|
||||
if s.runtimes == nil {
|
||||
s.runtimes = map[string]*runtimeSeen{}
|
||||
}
|
||||
at := x.Discovery.At
|
||||
if at.IsZero() {
|
||||
at = time.Now().UTC()
|
||||
}
|
||||
heard := map[string]bool{}
|
||||
for _, h := range x.Discovery.Heard {
|
||||
k := runtimeKey(h.Info.Name, h.Info.ID)
|
||||
seen := seenOf(h, at)
|
||||
if heard[k] {
|
||||
// Two instances under one name and id: both counted, the slower answer's time kept.
|
||||
prev := s.runtimes[k]
|
||||
seen.Modules += prev.Modules
|
||||
seen.Tools += prev.Tools
|
||||
seen.SeatVerbs += prev.SeatVerbs
|
||||
seen.Bytes += prev.Bytes
|
||||
if prev.Took > seen.Took {
|
||||
seen.Took = prev.Took
|
||||
}
|
||||
}
|
||||
heard[k] = true
|
||||
s.runtimes[k] = seen
|
||||
}
|
||||
// A runtime restarted answers under a new instance id: the old one is gone, not silent, and saying
|
||||
// it was missed would hide nothing but cry wolf after every restart.
|
||||
now := map[string]bool{}
|
||||
for k := range heard {
|
||||
now[s.runtimes[k].Name+"@"+s.runtimes[k].Machine] = true
|
||||
}
|
||||
for k, r := range s.runtimes {
|
||||
if !heard[k] && r.Machine != "" && now[r.Name+"@"+r.Machine] {
|
||||
delete(s.runtimes, k)
|
||||
}
|
||||
}
|
||||
silent := map[string]string{}
|
||||
for _, u := range x.Discovery.Silent {
|
||||
silent[runtimeKey(u.Name, u.ID)] = fmt.Sprintf("it answered PING; what it serves did not arrive within %s", announce.Patience)
|
||||
}
|
||||
for k, r := range s.runtimes {
|
||||
if heard[k] {
|
||||
continue
|
||||
}
|
||||
r.Answered = false
|
||||
used := ""
|
||||
if at, ok := x.Earlier[k]; ok {
|
||||
used = fmt.Sprintf("; what it announced %s ago is used", time.Since(at).Round(time.Second))
|
||||
}
|
||||
if why, ok := silent[k]; ok {
|
||||
r.Why = why
|
||||
if used != "" {
|
||||
for i := range x.Unheard {
|
||||
if x.Unheard[i].Runtime == r.Name && x.Unheard[i].Machine == r.Machine && x.Unheard[i].Why == why {
|
||||
x.Unheard[i].Why += used
|
||||
}
|
||||
}
|
||||
}
|
||||
continue // already in the index's unheard
|
||||
}
|
||||
r.Why = "it did not answer this round of discovery"
|
||||
if time.Since(r.LastHeard) <= RuntimeForgotten {
|
||||
x.Unheard = append(x.Unheard, unheard{Runtime: r.Name, Machine: r.Machine,
|
||||
Why: fmt.Sprintf("it answered discovery %s ago, not now%s", time.Since(r.LastHeard).Round(time.Second), used)})
|
||||
}
|
||||
}
|
||||
for _, u := range x.Discovery.Silent {
|
||||
k := runtimeKey(u.Name, u.ID)
|
||||
if _, known := s.runtimes[k]; !known {
|
||||
s.runtimes[k] = &runtimeSeen{Name: u.Name, ID: u.ID, Machine: u.Machine, Why: silent[k]}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runtimesAnswer is mesh_runtimes: discovery asked now, and every runtime as it was heard.
|
||||
func (s *Surface) runtimesAnswer() map[string]any {
|
||||
x, err := s.indexAsked(true)
|
||||
if err != nil {
|
||||
return failure("the mesh's discovery failed: " + err.Error())
|
||||
}
|
||||
s.mu.Lock()
|
||||
var all []runtimeSeen
|
||||
for _, r := range s.runtimes {
|
||||
all = append(all, *r)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
sort.Slice(all, func(i, j int) bool {
|
||||
if all[i].Machine != all[j].Machine {
|
||||
return all[i].Machine < all[j].Machine
|
||||
}
|
||||
return runtimeKey(all[i].Name, all[i].ID) < runtimeKey(all[j].Name, all[j].ID)
|
||||
})
|
||||
now := time.Now()
|
||||
var answered, notHeard []map[string]any
|
||||
onMachine := map[string]bool{}
|
||||
for _, r := range all {
|
||||
entry := map[string]any{"runtime": r.Name, "instance": r.ID}
|
||||
if r.Machine != "" {
|
||||
entry["machine"] = r.Machine
|
||||
}
|
||||
if !r.LastHeard.IsZero() {
|
||||
entry["last heard"] = r.LastHeard.Format(time.RFC3339)
|
||||
entry["last heard, ago"] = now.Sub(r.LastHeard).Round(time.Second).String()
|
||||
entry["took"] = r.Took.Round(time.Millisecond).String()
|
||||
entry["bytes"] = r.Bytes
|
||||
entry["modules"] = r.Modules
|
||||
entry["tools"] = r.Tools
|
||||
entry["seat verbs"] = r.SeatVerbs
|
||||
entry["shortened"] = r.Shortened != ""
|
||||
if r.Shortened != "" {
|
||||
entry["shortened, how"] = r.Shortened
|
||||
}
|
||||
if r.Reused && r.Answered {
|
||||
entry["announcement"] = "unchanged (its PING's digest), so not fetched again; took is its PING's"
|
||||
}
|
||||
}
|
||||
if r.Answered {
|
||||
onMachine[r.Machine] = true
|
||||
answered = append(answered, entry)
|
||||
} else {
|
||||
entry["why"] = r.Why
|
||||
if r.LastHeard.IsZero() {
|
||||
entry["last heard"] = "never in full, since this console started"
|
||||
}
|
||||
notHeard = append(notHeard, entry)
|
||||
}
|
||||
}
|
||||
for _, m := range x.Machines {
|
||||
if !onMachine[m] && !machineListed(notHeard, m) {
|
||||
notHeard = append(notHeard, map[string]any{"machine": m,
|
||||
"why": "the mesh knows this machine and no runtime on it has answered discovery since this console started"})
|
||||
}
|
||||
}
|
||||
if answered == nil {
|
||||
answered = []map[string]any{}
|
||||
}
|
||||
if notHeard == nil {
|
||||
notHeard = []map[string]any{}
|
||||
}
|
||||
return answerText(map[string]any{
|
||||
"asked": x.Discovery.At.Format(time.RFC3339),
|
||||
"waited": x.Discovery.Waited.Round(time.Millisecond).String(),
|
||||
"window": fmt.Sprintf("at least %s; up to %s for a runtime that answered PING, or until one does", announce.Window, announce.Patience),
|
||||
"answered": answered,
|
||||
"not heard": notHeard,
|
||||
})
|
||||
}
|
||||
|
||||
func machineListed(entries []map[string]any, machine string) bool {
|
||||
for _, e := range entries {
|
||||
if e["machine"] == machine {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// slowRuntime answers PING at once and INFO after `lag`, or never with a negative lag.
|
||||
func slowRuntime(t *testing.T, conn *bus.Conn, node string, lag time.Duration, endpoints []announce.Endpoint) {
|
||||
t.Helper()
|
||||
id := micro.ServiceIdentity{Name: "node-tools", ID: node, Version: announce.Version, Metadata: map[string]string{"node": node}}
|
||||
ping, _ := json.Marshal(micro.Ping{ServiceIdentity: id, Type: micro.PingResponseType})
|
||||
stop, err := conn.Raw("$SRV.PING", func(string, []byte) []byte { return ping })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
// Asked of every service, and of this instance by name and id — as discovery asks it (issue 554).
|
||||
info := func(string, []byte) []byte {
|
||||
if lag < 0 {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(lag)
|
||||
body, _ := json.Marshal(announce.Info(announce.Service{Name: id.Name, ID: id.ID, Metadata: id.Metadata}, endpoints))
|
||||
return body
|
||||
}
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO." + id.Name + "." + id.ID} {
|
||||
stop, err := conn.Raw(subject, info)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
conn.Flush()
|
||||
}
|
||||
|
||||
// The laptop's symptom (2026-10-05), reproduced: its runtime's one large answer reached the console
|
||||
// after the 750 ms window, and the console said its modules ran nowhere — "nothing in the mesh is
|
||||
// called slack" — or ran only on another machine. Now a runtime that said it is there is waited for,
|
||||
// one that never says what it serves is named wherever an address it might answer is asked for, and
|
||||
// mesh_runtimes shows who answered, how fast and how much.
|
||||
func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T) {
|
||||
keptWas, patienceWas := IndexKept, announce.Patience
|
||||
// Kept for the test: one round of discovery answers every question but mesh_runtimes, which asks anew.
|
||||
IndexKept, announce.Patience = time.Minute, 2500*time.Millisecond
|
||||
t.Cleanup(func() { IndexKept, announce.Patience = keptWas, patienceWas })
|
||||
mt.New(t)
|
||||
|
||||
laptop := connect(t, "node-tools", "laptop")
|
||||
stopTool, err := laptop.HandleSubject("mesh.mod.slack.tool.slack_check.laptop", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"ok": true}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopTool)
|
||||
slowRuntime(t, laptop, "laptop", announce.Window+400*time.Millisecond, []announce.Endpoint{
|
||||
{Kind: announce.KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Description: "Is Slack started once?",
|
||||
Subject: "mesh.mod.slack.tool.slack_check.laptop"},
|
||||
{Kind: announce.KindTool, Module: "nextcloud-client", Tool: "nextcloud_check", Node: "laptop",
|
||||
Subject: "mesh.mod.nextcloud-client.tool.nextcloud_check.laptop"},
|
||||
})
|
||||
sleeper := connect(t, "node-tools", "sleeper")
|
||||
slowRuntime(t, sleeper, "sleeper", -1, nil)
|
||||
desktop := connect(t, "node-tools", "desktop")
|
||||
slowRuntime(t, desktop, "desktop", 0, []announce.Endpoint{
|
||||
{Kind: announce.KindTool, Module: "nextcloud-client", Tool: "nextcloud_check", Node: "desktop",
|
||||
Subject: "mesh.mod.nextcloud-client.tool.nextcloud_check.desktop"},
|
||||
})
|
||||
|
||||
controller := connect(t, "mesh-controller", "bench")
|
||||
for verb, output := range map[string]string{
|
||||
"nodes": `[{"name":"laptop"},{"name":"sleeper"},{"name":"desktop"},{"name":"bench"}]`,
|
||||
"modules": `[{"module":"slack","on":["laptop"],"tools":true},{"module":"nap","on":["sleeper"],"tools":true},` +
|
||||
`{"module":"nextcloud-client","on":["laptop","desktop","sleeper"],"tools":true}]`,
|
||||
} {
|
||||
output := output
|
||||
stop, err := controller.HandleSubject("mesh.seat.mesh-controller.tool."+verb, func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"output": output, "ok": true}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
controller.Flush()
|
||||
|
||||
console := connect(t, "node-tools", "laptop-console")
|
||||
up, err := Serve(NewSurface(console, "laptop.node-tools"), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer up.Close()
|
||||
endpoint := "http://" + up.Address + "/mcp"
|
||||
|
||||
// The late answer is waited for: the laptop's module is found and called.
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/slack.slack_check"}); isErr || !strings.Contains(got, `"ok": true`) {
|
||||
t.Errorf("a late runtime's module: %s", got)
|
||||
}
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/nextcloud-client.nextcloud_check"}); isErr && strings.Contains(got, "does not run on laptop") {
|
||||
t.Errorf("a late runtime's module was said to run elsewhere: %s", got)
|
||||
}
|
||||
machine, _ := call(t, endpoint, "mesh_machine", map[string]any{"node": "laptop"})
|
||||
if !strings.Contains(machine, "laptop/slack.slack_check") {
|
||||
t.Errorf("the late runtime's machine lists nothing: %s", machine)
|
||||
}
|
||||
|
||||
// The silent one is named, never called missing.
|
||||
for address, want := range map[string]string{
|
||||
"sleeper/nap.doze": "nap is assigned to sleeper in the mesh's records, but nothing that answered discovery serves it: discovery did not hear in full from node-tools on sleeper (it answered PING",
|
||||
"sleeper/nextcloud-client.nextcloud_check": "nextcloud-client on sleeper did not answer discovery: discovery did not hear in full from node-tools on sleeper",
|
||||
} {
|
||||
got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": address})
|
||||
if !isErr || !strings.Contains(got, want) || strings.Contains(got, "nothing in the mesh is called") || strings.Contains(got, "does not run on") {
|
||||
t.Errorf("%s: %s\nwant %q", address, got, want)
|
||||
}
|
||||
}
|
||||
machine, _ = call(t, endpoint, "mesh_machine", map[string]any{"node": "sleeper"})
|
||||
if !strings.Contains(machine, `"incomplete"`) || !strings.Contains(machine, `"modules": []`) {
|
||||
t.Errorf("a machine whose runtime did not answer: %s", machine)
|
||||
}
|
||||
overview, _ := call(t, endpoint, "mesh_overview", nil)
|
||||
if !strings.Contains(overview, "the runtime node-tools on sleeper") {
|
||||
t.Errorf("overview does not name the silent runtime: %s", overview)
|
||||
}
|
||||
|
||||
// mesh_runtimes: who answered, how fast, how much; who did not.
|
||||
got, isErr := call(t, endpoint, "mesh_runtimes", nil)
|
||||
if isErr {
|
||||
t.Fatal(got)
|
||||
}
|
||||
var report struct {
|
||||
Answered []map[string]any `json:"answered"`
|
||||
NotHeard []map[string]any `json:"not heard"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(got), &report); err != nil {
|
||||
t.Fatalf("%v: %s", err, got)
|
||||
}
|
||||
var lap map[string]any
|
||||
for _, r := range report.Answered {
|
||||
if r["machine"] == "laptop" {
|
||||
lap = r
|
||||
}
|
||||
}
|
||||
if lap == nil {
|
||||
t.Fatalf("the laptop's runtime is not listed as answering: %s", got)
|
||||
}
|
||||
took, _ := time.ParseDuration(lap["took"].(string))
|
||||
if took < announce.Window || lap["bytes"].(float64) <= 0 || lap["modules"].(float64) != 2 || lap["tools"].(float64) != 2 ||
|
||||
lap["shortened"] != false || lap["last heard"] == nil {
|
||||
t.Errorf("the laptop's runtime: %v", lap)
|
||||
}
|
||||
if !strings.Contains(got, `"machine": "sleeper"`) || !strings.Contains(got, "answered PING") {
|
||||
t.Errorf("the silent runtime is not listed: %s", got)
|
||||
}
|
||||
silent := false
|
||||
for _, r := range report.NotHeard {
|
||||
silent = silent || r["machine"] == "sleeper"
|
||||
}
|
||||
if !silent {
|
||||
t.Errorf("sleeper is not among the not heard: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A runtime heard once and silent now is named in the answers about its machine.
|
||||
func TestARuntimeHeardBeforeAndNotNowIsNamed(t *testing.T) {
|
||||
s := &Surface{}
|
||||
before := &index{Modules: map[string]*moduleInfo{}, Recorded: map[string][]string{}, Discovery: announce.Discovery{At: time.Now().UTC(),
|
||||
Heard: []announce.Heard{{Info: micro.Info{ServiceIdentity: micro.ServiceIdentity{Name: "node-tools", ID: "laptop",
|
||||
Metadata: map[string]string{"node": "laptop"}}}, Took: 300 * time.Millisecond, Bytes: 1000}}}}
|
||||
s.remember(before)
|
||||
now := &index{Modules: map[string]*moduleInfo{"nextcloud-client": {Module: "nextcloud-client", On: []string{"desktop"},
|
||||
Tools: []Tool{{Module: "nextcloud-client", Name: "nextcloud_check"}}}}, Recorded: map[string][]string{},
|
||||
Discovery: announce.Discovery{At: time.Now().UTC(), Pinged: 1}} // desktop's runtime answered
|
||||
s.remember(now)
|
||||
if len(now.Unheard) != 1 || now.Unheard[0].Machine != "laptop" {
|
||||
t.Fatalf("unheard: %+v", now.Unheard)
|
||||
}
|
||||
_, err := resolve(now, "laptop/nextcloud-client.nextcloud_check")
|
||||
if err == nil || !strings.Contains(err.Error(), "nextcloud-client on laptop did not answer discovery") ||
|
||||
!strings.Contains(err.Error(), "answered discovery") {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
_, err = resolve(now, "laptop/slack.slack_check")
|
||||
if err == nil || strings.Contains(err.Error(), "nothing in the mesh is called") || !strings.Contains(err.Error(), "slack may be theirs") {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
// Nobody unheard: the plain answers stand.
|
||||
quiet := &index{Modules: now.Modules, Recorded: map[string][]string{}, Discovery: now.Discovery}
|
||||
if _, err := resolve(quiet, "laptop/nextcloud-client.nextcloud_check"); err == nil || !strings.Contains(err.Error(), "does not run on laptop; it runs on desktop") {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
if _, err := resolve(quiet, "laptop/slack.slack_check"); err == nil || !strings.Contains(err.Error(), "nothing in the mesh is called slack") {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A runtime restarted answers under a new instance id; its old instance is gone, not unheard.
|
||||
func TestARestartedRuntimeIsNotNamedUnheard(t *testing.T) {
|
||||
heard := func(id string) *index {
|
||||
return &index{Modules: map[string]*moduleInfo{}, Recorded: map[string][]string{}, Discovery: announce.Discovery{At: time.Now().UTC(),
|
||||
Heard: []announce.Heard{{Info: micro.Info{ServiceIdentity: micro.ServiceIdentity{Name: "node-tools", ID: id,
|
||||
Metadata: map[string]string{"node": "laptop"}}}}}}}
|
||||
}
|
||||
s := &Surface{}
|
||||
s.remember(heard("before-restart"))
|
||||
after := heard("after-restart")
|
||||
s.remember(after)
|
||||
if len(after.Unheard) != 0 {
|
||||
t.Errorf("a restarted runtime was named unheard: %+v", after.Unheard)
|
||||
}
|
||||
if len(s.runtimes) != 1 {
|
||||
t.Errorf("the old instance is still kept: %d runtimes", len(s.runtimes))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package console
|
||||
|
||||
// Finding a tool by the words an agent already has (novox/hq ADR 0245).
|
||||
//
|
||||
// An agent that wants a unit's log reaches for `journalctl`, and one that wants a container's for `docker
|
||||
// logs`; it searches with those words, or with `logs`. Matched against names and descriptions alone, none
|
||||
// of them found the service manager's `journal` — whose description says "journal", never "logs" or
|
||||
// "journalctl" — and the agent went round the mesh over ssh. So each verb and tool says what it replaces,
|
||||
// and search matches three ways, best first:
|
||||
//
|
||||
// 1. **The query is a command a tool replaces** — `journalctl -u x`, `systemctl status x`, `docker ps`:
|
||||
// every word of the replaced command, in order, among the query's words. The most specific command wins.
|
||||
// 2. **Every word is in the tool's name, its replaced commands or its description**, a word also matching
|
||||
// through a short table of the words agents use for the mesh's (logs ⇄ journal, ps ⇄ list). A word in
|
||||
// the name or a replaced command weighs more than one in the description, and a name the query says
|
||||
// whole more than one it says in part.
|
||||
// 3. Nothing else: a search that matches nothing says so.
|
||||
//
|
||||
// Ties go to a seat — the place tools belong — then to the shorter name, then by address.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// synonyms are the words an agent uses for the mesh's own, each matching the other's (a word, not part
|
||||
// of one: `log` must not match `login`). Kept short: a command is said by the tool that replaces it.
|
||||
var synonyms = map[string][]string{
|
||||
"logs": {"log", "journal"},
|
||||
"log": {"logs", "journal"},
|
||||
"journal": {"logs", "log"},
|
||||
"ps": {"list", "running"},
|
||||
"ls": {"list"},
|
||||
"list": {"ps", "ls"},
|
||||
"restart": {"restarts"},
|
||||
"service": {"unit", "units"},
|
||||
"services": {"unit", "units"},
|
||||
"unit": {"service"},
|
||||
"container": {"containers", "docker"},
|
||||
"firewall": {"filter", "nftables"},
|
||||
"hosts": {"hostname"},
|
||||
}
|
||||
|
||||
// filler are the function words a question carries and no tool is named for: a query's words must all
|
||||
// match, so "did our weekly upgrade run on all our nodes" would otherwise find nothing for "did", "our",
|
||||
// "on" and "all". Kept short and here only; a word that names something (`every`, `list`, `not`) is not
|
||||
// in it. A query of nothing but these matches nothing.
|
||||
var filler = map[string]bool{}
|
||||
|
||||
func init() {
|
||||
for _, w := range strings.Fields("a an the did do does our my your we i is are was on in at of for to all any and or with it its there what which how has have been") {
|
||||
filler[w] = true
|
||||
}
|
||||
}
|
||||
|
||||
// queryWords are a query's words to match: split at spaces, the punctuation of a sentence trimmed from
|
||||
// each end (`nodes?` is `nodes`), filler left out.
|
||||
func queryWords(query string) []string {
|
||||
var out []string
|
||||
for _, w := range strings.Fields(query) {
|
||||
w = strings.Trim(w, "?!.,;:\"'()")
|
||||
if w != "" && !filler[w] {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hit is one match as mesh_search answers it.
|
||||
type hit struct {
|
||||
Address string `json:"address"`
|
||||
Does string `json:"does"`
|
||||
// InsteadOf is the command the query matched, when it matched one the tool replaces.
|
||||
InsteadOf string `json:"instead_of,omitempty"`
|
||||
Also string `json:"also,omitempty"`
|
||||
score int
|
||||
}
|
||||
|
||||
// words are a text's words, lower-cased, split at anything that is not a letter, a digit, `-` or `_`
|
||||
// within a word — so `node-service-manager.journal` is its three names' parts and the whole.
|
||||
func words(s string) []string {
|
||||
var out []string
|
||||
for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||||
return !(unicode.IsLetter(r) || unicode.IsDigit(r) || r == '-' || r == '_' || r == '/')
|
||||
}) {
|
||||
out = append(out, w)
|
||||
for _, part := range strings.FieldsFunc(w, func(r rune) bool { return r == '-' || r == '_' || r == '/' }) {
|
||||
if part != w {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// commandWords are a command line's words as matched against a replaced command: lower-cased, a
|
||||
// program named by path reduced to its name, and what only runs it (`sudo`) left out.
|
||||
func commandWords(line string) []string {
|
||||
var out []string
|
||||
for i, w := range strings.Fields(strings.ToLower(line)) {
|
||||
if i == 0 || strings.HasPrefix(w, "/") {
|
||||
w = path.Base(w)
|
||||
}
|
||||
if w == "sudo" {
|
||||
continue
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inOrder says whether every one of want is among have, in the same order.
|
||||
func inOrder(want, have []string) bool {
|
||||
i := 0
|
||||
for _, h := range have {
|
||||
if i < len(want) && h == want[i] {
|
||||
i++
|
||||
}
|
||||
}
|
||||
return len(want) > 0 && i == len(want)
|
||||
}
|
||||
|
||||
// replacedBy is the replaced command a query names, the most specific of those it matches, or "".
|
||||
func replacedBy(query string, replaces []string) string {
|
||||
q := commandWords(query)
|
||||
best, bestLen := "", 0
|
||||
for _, r := range replaces {
|
||||
w := commandWords(r)
|
||||
if inOrder(w, q) && len(w) > bestLen {
|
||||
best, bestLen = r, len(w)
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// scoreOf is how well a tool matches the query: 0 for not at all.
|
||||
func scoreOf(query string, qwords []string, name, description string, replaces []string) (int, string) {
|
||||
if cmd := replacedBy(query, replaces); cmd != "" {
|
||||
return 100 + 10*len(commandWords(cmd)), cmd
|
||||
}
|
||||
strong := strings.ToLower(name + " " + strings.Join(replaces, " "))
|
||||
weak := strings.ToLower(description)
|
||||
strongWords := map[string]bool{}
|
||||
for _, w := range words(strong) {
|
||||
strongWords[w] = true
|
||||
}
|
||||
weakWords := map[string]bool{}
|
||||
for _, w := range words(weak) {
|
||||
weakWords[w] = true
|
||||
}
|
||||
score := 10
|
||||
for _, w := range qwords {
|
||||
switch {
|
||||
case strings.Contains(strong, w):
|
||||
score += 5
|
||||
case strings.Contains(weak, w):
|
||||
score++
|
||||
default:
|
||||
found := 0
|
||||
for _, s := range synonyms[w] {
|
||||
if strongWords[s] {
|
||||
found = 5
|
||||
break
|
||||
}
|
||||
if weakWords[s] {
|
||||
found = 1
|
||||
}
|
||||
}
|
||||
if found == 0 {
|
||||
return 0, ""
|
||||
}
|
||||
score += found
|
||||
}
|
||||
}
|
||||
// A name the query says whole — `docker logs` for docker_logs, not docker_secrets_in_logs — first.
|
||||
own := name[strings.LastIndex(name, ".")+1:]
|
||||
whole := true
|
||||
for _, part := range strings.FieldsFunc(strings.ToLower(own), func(r rune) bool { return r == '_' || r == '-' }) {
|
||||
in := false
|
||||
for _, w := range qwords {
|
||||
in = in || w == part
|
||||
}
|
||||
whole = whole && in
|
||||
}
|
||||
if whole {
|
||||
score += 2
|
||||
}
|
||||
return score, ""
|
||||
}
|
||||
|
||||
// search is mesh_search's answer: every match, best first.
|
||||
func (x *index) search(query string) []hit {
|
||||
query = strings.ToLower(strings.TrimSpace(query))
|
||||
qwords := queryWords(query)
|
||||
var hits []hit
|
||||
if len(qwords) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, st := range x.Seats {
|
||||
for _, v := range st.Verbs {
|
||||
replaces := x.Replaces["seat:"+st.Seat+"."+v.Name]
|
||||
score, cmd := scoreOf(query, qwords, st.Seat+"."+v.Name, v.Description, replaces)
|
||||
if score == 0 {
|
||||
continue
|
||||
}
|
||||
h := hit{Does: firstLine(v.Description), InsteadOf: cmd, score: score + 1} // a seat first, on a tie
|
||||
if st.Scope == "node" {
|
||||
on := nodesOf(st.Holders)
|
||||
first := "<node>"
|
||||
if len(on) > 0 {
|
||||
first = on[0]
|
||||
if len(on) > 1 {
|
||||
h.Also = "also on " + strings.Join(on[1:], ", ")
|
||||
}
|
||||
}
|
||||
h.Address = first + "/" + st.Seat + "." + v.Name
|
||||
} else {
|
||||
h.Address = st.Seat + "." + v.Name
|
||||
}
|
||||
hits = append(hits, h)
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(x.Modules))
|
||||
for n := range x.Modules {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
m := x.Modules[n]
|
||||
for _, t := range m.Tools {
|
||||
score, cmd := scoreOf(query, qwords, m.Module+"."+t.Name, t.Description, x.Replaces[m.Module+"."+t.Name])
|
||||
if score == 0 {
|
||||
continue
|
||||
}
|
||||
h := hit{Does: firstLine(t.Description), InsteadOf: cmd, score: score}
|
||||
switch {
|
||||
case m.Interchangeable:
|
||||
h.Address, h.Also = m.Module+"."+t.Name, "any instance; on "+orNobody(m.On)
|
||||
case len(m.On) == 0:
|
||||
h.Address, h.Also = "<node>/"+m.Module+"."+t.Name, "the mesh places it on no machine"
|
||||
default:
|
||||
h.Address = m.On[0] + "/" + m.Module + "." + t.Name
|
||||
if len(m.On) > 1 {
|
||||
h.Also = "also on " + strings.Join(m.On[1:], ", ")
|
||||
}
|
||||
}
|
||||
hits = append(hits, h)
|
||||
}
|
||||
}
|
||||
// Best first; on a tie the shorter name — the plainer tool — then the address, so an answer never
|
||||
// depends on the order discovery heard things in.
|
||||
sort.SliceStable(hits, func(i, j int) bool {
|
||||
if hits[i].score != hits[j].score {
|
||||
return hits[i].score > hits[j].score
|
||||
}
|
||||
a, b := hits[i].Address[strings.LastIndex(hits[i].Address, ".")+1:], hits[j].Address[strings.LastIndex(hits[j].Address, ".")+1:]
|
||||
if len(a) != len(b) {
|
||||
return len(a) < len(b)
|
||||
}
|
||||
return hits[i].Address < hits[j].Address
|
||||
})
|
||||
return hits
|
||||
}
|
||||
|
||||
// replacesOf is what every verb and tool replaces, from the controller's records (novox/hq ADR 0245):
|
||||
// a seat's verbs from its `tools` answer, keyed `seat:<seat>.<verb>`, and a module's own tools from
|
||||
// its `modules` answer, keyed `<module>.<tool>`. A controller older than the field answers neither,
|
||||
// and search matches names and descriptions as before.
|
||||
func replacesOf(toolsAnswer json.RawMessage, modules []recordedModule) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
var seats struct {
|
||||
Seats []struct {
|
||||
Seat string `json:"seat"`
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
Replaces []string `json:"replaces"`
|
||||
} `json:"tools"`
|
||||
} `json:"seats"`
|
||||
}
|
||||
if json.Unmarshal(toolsAnswer, &seats) == nil {
|
||||
for _, s := range seats.Seats {
|
||||
for _, t := range s.Tools {
|
||||
if len(t.Replaces) > 0 {
|
||||
out["seat:"+s.Seat+"."+t.Name] = t.Replaces
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range modules {
|
||||
for tool, r := range m.Replaces {
|
||||
if len(r) > 0 {
|
||||
out[m.Module+"."+tool] = r
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
)
|
||||
|
||||
// meshToSearch is a mesh as discovery and the controller's records describe it: the service manager's
|
||||
// verbs on two machines, the controller's own, a docker module with its tools, and what each replaces
|
||||
// as the records say it — the seats' from `tools`, the module's from `modules`.
|
||||
func meshToSearch(t *testing.T) *index {
|
||||
t.Helper()
|
||||
var endpoints []announce.Endpoint
|
||||
verbs := map[string]string{
|
||||
"units": "The units the service manager knows in a scope, each with its load, active and sub state.",
|
||||
"status": "One unit as the service manager sees it now: its states, whether it starts at boot.",
|
||||
"restart": "Restart one unit.",
|
||||
"journal": "The last lines of one unit's journal (at most 2000).",
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
for verb, does := range verbs {
|
||||
endpoints = append(endpoints, announce.Endpoint{Kind: announce.KindSeat, Module: "systemd", Seat: "node-service-manager",
|
||||
Tool: verb, Scope: "node", Node: node, Description: does,
|
||||
Subject: "mesh.seat.node-service-manager.tool." + verb + "." + node})
|
||||
}
|
||||
}
|
||||
for verb, does := range map[string]string{
|
||||
"node": "What one machine reported it can do, what it is assigned, and why.",
|
||||
"nodes": "Every machine the mesh knows, with whether it is converged or adopted.",
|
||||
} {
|
||||
endpoints = append(endpoints, announce.Endpoint{Kind: announce.KindSeat, Module: "mesh-controller", Seat: "mesh-controller",
|
||||
Tool: verb, Scope: "mesh", Description: does, Subject: "mesh.seat.mesh-controller.tool." + verb})
|
||||
}
|
||||
for tool, does := range map[string]string{
|
||||
"docker_list": "Every container on this machine: name, image, state, ports.",
|
||||
"docker_logs": "The last lines a container wrote.",
|
||||
"docker_secrets_in_logs": "Whether a container's logs hold a secret.",
|
||||
"docker_restart": "Restart one container.",
|
||||
} {
|
||||
endpoints = append(endpoints, announce.Endpoint{Kind: announce.KindTool, Module: "docker", Tool: tool, Node: "anchor",
|
||||
Description: does, Subject: "mesh.mod.docker.tool." + tool + ".anchor"})
|
||||
}
|
||||
x, _, _ := indexOfEndpoints(endpoints)
|
||||
tools := json.RawMessage(`{"seats":[{"seat":"node-service-manager","scope":"node","tools":[
|
||||
{"name":"units","replaces":["systemctl list-units","systemctl --failed"]},
|
||||
{"name":"status","replaces":["systemctl status","systemctl is-active"]},
|
||||
{"name":"restart","replaces":["systemctl restart"]},
|
||||
{"name":"journal","replaces":["journalctl"]}]},
|
||||
{"seat":"mesh-controller","scope":"mesh","tools":[{"name":"node","replaces":["hostnamectl","uptime"]},{"name":"nodes"}]}]}`)
|
||||
x.Replaces = replacesOf(tools, []recordedModule{{Module: "docker", On: []string{"anchor"}, Tools: true,
|
||||
Replaces: map[string][]string{"docker_list": {"docker ps"}, "docker_logs": {"docker logs"}, "docker_restart": {"docker restart"}}}})
|
||||
return x
|
||||
}
|
||||
|
||||
// **The words an agent has find the verb first** (novox/hq ADR 0245): the command it would have run, or
|
||||
// the word it would have used, puts the mesh's tool for it at the top.
|
||||
func TestSearchFindsTheToolThatReplacesACommandFirst(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
for query, want := range map[string]string{
|
||||
"journalctl": "anchor/node-service-manager.journal",
|
||||
"journalctl -u mesh-controller -n 50": "anchor/node-service-manager.journal",
|
||||
"sudo journalctl -fu sshd": "anchor/node-service-manager.journal",
|
||||
"/usr/bin/journalctl": "anchor/node-service-manager.journal",
|
||||
"logs": "anchor/node-service-manager.journal",
|
||||
"systemctl status": "anchor/node-service-manager.status",
|
||||
"systemctl status nats.service": "anchor/node-service-manager.status",
|
||||
"systemctl restart mesh-controller": "anchor/node-service-manager.restart",
|
||||
"systemctl --failed": "anchor/node-service-manager.units",
|
||||
"docker ps": "anchor/docker.docker_list",
|
||||
"docker ps -a": "anchor/docker.docker_list",
|
||||
"docker logs": "anchor/docker.docker_logs",
|
||||
"docker logs --tail 100 nats": "anchor/docker.docker_logs",
|
||||
"docker restart nats": "anchor/docker.docker_restart",
|
||||
"container logs": "anchor/docker.docker_logs",
|
||||
"uptime": "mesh-controller.node",
|
||||
} {
|
||||
hits := x.search(query)
|
||||
if len(hits) == 0 {
|
||||
t.Errorf("%q found nothing; want %s first", query, want)
|
||||
continue
|
||||
}
|
||||
if hits[0].Address != want {
|
||||
var got []string
|
||||
for _, h := range hits {
|
||||
got = append(got, h.Address)
|
||||
}
|
||||
t.Errorf("%q found %s first; want %s (all: %s)", query, hits[0].Address, want, strings.Join(got, ", "))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A match on a replaced command says which command it replaced, and the most specific one wins.
|
||||
func TestASearchByCommandSaysWhatItReplaces(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
hits := x.search("systemctl status sshd")
|
||||
if hits[0].InsteadOf != "systemctl status" {
|
||||
t.Errorf("the match does not say what it replaces: %+v", hits[0])
|
||||
}
|
||||
if hits[0].Also != "also on laptop" {
|
||||
t.Errorf("a node seat's verb does not say its other machines: %+v", hits[0])
|
||||
}
|
||||
// `logs` also finds the container's logs, after the journal — both are answers.
|
||||
var docker bool
|
||||
for _, h := range x.search("logs")[:3] {
|
||||
docker = docker || h.Address == "anchor/docker.docker_logs"
|
||||
}
|
||||
if !docker {
|
||||
t.Errorf("`logs` does not find docker_logs among the first: %+v", x.search("logs"))
|
||||
}
|
||||
}
|
||||
|
||||
// What search did before stays: every word in a name or description, and nothing for a word nothing has.
|
||||
// A synonym is a word, never part of one: `logs` matches through `log`, and finds no `login`.
|
||||
func TestSearchStillMatchesEveryWord(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
if hits := x.search("machine converged"); len(hits) != 1 || hits[0].Address != "mesh-controller.nodes" {
|
||||
t.Errorf("words in a description: %+v", hits)
|
||||
}
|
||||
if hits := x.search("journalctl kubernetes"); len(hits) != 1 || hits[0].Address != "anchor/node-service-manager.journal" {
|
||||
t.Errorf("a command with arguments nothing has is still that command: %+v", hits)
|
||||
}
|
||||
if hits := x.search("kubernetes"); len(hits) != 0 {
|
||||
t.Errorf("a word nothing has matched: %+v", hits)
|
||||
}
|
||||
x.Modules["shell"] = &moduleInfo{Module: "shell", On: []string{"anchor"}, Tools: []Tool{{Module: "shell", Name: "shell_login",
|
||||
Description: "The account's login shell."}}}
|
||||
for _, h := range x.search("logs") {
|
||||
if h.Address == "anchor/shell.shell_login" {
|
||||
t.Errorf("`logs` found `login` through its synonym `log`: %+v", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A controller older than `replaces` answers without it: search matches names and descriptions as before.
|
||||
func TestSearchWithoutReplacesMatchesAsBefore(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
x.Replaces = replacesOf(json.RawMessage(`{"seats":[{"seat":"node-service-manager","tools":[{"name":"journal"}]}]}`),
|
||||
[]recordedModule{{Module: "docker"}})
|
||||
if len(x.Replaces) != 0 {
|
||||
t.Fatalf("replaces from records that have none: %v", x.Replaces)
|
||||
}
|
||||
if hits := x.search("journal"); len(hits) == 0 || hits[0].Address != "anchor/node-service-manager.journal" {
|
||||
t.Errorf("a verb's own name: %+v", hits)
|
||||
}
|
||||
if hits := x.search("docker logs"); len(hits) == 0 || hits[0].Address != "anchor/docker.docker_logs" {
|
||||
t.Errorf("a tool's own name: %+v", hits)
|
||||
}
|
||||
}
|
||||
|
||||
// A question's filler words do not have to match (every other word still does): the operator's own words
|
||||
// find the tool whose description says it in other words; a query of filler alone finds nothing.
|
||||
func TestSearchIgnoresFillerWords(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
x.Modules["pacman"] = &moduleInfo{Module: "pacman", On: []string{"anchor"}, Tools: []Tool{{Module: "pacman", Name: "pacman_round",
|
||||
Description: "Whether this week's system upgrade ran on every node (the weekly upgrade round, last run of all nodes)."}}}
|
||||
for _, c := range []struct {
|
||||
query string
|
||||
want string // the first hit's address, "" for none
|
||||
}{
|
||||
{"did our weekly system upgrade run on all our nodes?", "anchor/pacman.pacman_round"},
|
||||
{"the weekly upgrade", "anchor/pacman.pacman_round"},
|
||||
{"did our", ""},
|
||||
{"what is the", ""},
|
||||
{"", ""},
|
||||
{"did our weekly kubernetes upgrade run?", ""},
|
||||
} {
|
||||
hits := x.search(c.query)
|
||||
got := ""
|
||||
if len(hits) > 0 {
|
||||
got = hits[0].Address
|
||||
}
|
||||
if got != c.want || (c.want == "" && len(hits) != 0) {
|
||||
t.Errorf("%q: first hit %q, want %q (%+v)", c.query, got, c.want, hits)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package console
|
||||
|
||||
// mesh_wait: a session waits on the bus for one event instead of polling (novox/hq ADR 0367).
|
||||
//
|
||||
// **Only the families the tool runner was issued** (§2, §6): its membership names them, composed by the
|
||||
// controller from the tool runner's manifest together with the grant that lets this account subscribe them.
|
||||
// A name not listed there is refused here, before the bus would refuse it; no argument or setting adds one.
|
||||
// **An event only wakes the session** (§4): what it says is read again from the verb that owns it.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WaitDefault is how long a wait lasts when no timeout is given, and WaitLongest the most it may last
|
||||
// (ADR 0367 §1): four minutes and ten.
|
||||
var (
|
||||
WaitDefault = 4 * time.Minute
|
||||
WaitLongest = 10 * time.Minute
|
||||
)
|
||||
|
||||
func waitTool() map[string]any {
|
||||
return map[string]any{"name": verbWait, "inputSchema": map[string]any{"type": "object", "properties": map[string]any{
|
||||
"subjects": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "one or more event families to wait on, by name, e.g. `mesh-knowledge.reviewed`; only " +
|
||||
"those the mesh granted this machine's tool runner — a refusal names them"},
|
||||
"filter": map[string]any{"type": "object",
|
||||
"description": "fields of the event's body that must equal the given values, e.g. {\"kind\": \"decision\", \"number\": 367} (optional)"},
|
||||
"timeout": map[string]any{"type": "number",
|
||||
"description": fmt.Sprintf("seconds to wait: %d when absent, %d at most", int(WaitDefault.Seconds()), int(WaitLongest.Seconds()))},
|
||||
}, "required": []string{"subjects"}},
|
||||
"description": "Wait for the first event of the named families that matches the filter, and answer it with its " +
|
||||
"family, body and time — or say it timed out. Call it in a loop from a background command, and after every " +
|
||||
"answer read the state from the verb that owns it: an event only wakes you, and one published between two " +
|
||||
"waits is not heard (novox/hq ADR 0367)."}
|
||||
}
|
||||
|
||||
// waitAnswer answers one mesh_wait call.
|
||||
func (s *Surface) waitAnswer(ctx context.Context, args map[string]any) map[string]any {
|
||||
names, err := waitNames(args["subjects"])
|
||||
if err != nil {
|
||||
return failure(err.Error())
|
||||
}
|
||||
timeout, err := waitTimeout(args["timeout"])
|
||||
if err != nil {
|
||||
return failure(err.Error())
|
||||
}
|
||||
filter, _ := args["filter"].(map[string]any)
|
||||
if args["filter"] != nil && filter == nil {
|
||||
return failure("mesh_wait's filter is an object of body fields to the values they must equal")
|
||||
}
|
||||
|
||||
issued := map[string]string{} // name → subject
|
||||
if m := s.conn.Membership(s.conn.Module()); m != nil {
|
||||
for _, w := range m.Waits {
|
||||
issued[w.Event] = w.Subject
|
||||
}
|
||||
}
|
||||
if len(issued) == 0 {
|
||||
return failure("this machine's tool runner was issued no event families to wait on: the mesh grants them from " +
|
||||
"the tool runner's manifest (novox/hq ADR 0367 §2), and its membership names none yet")
|
||||
}
|
||||
byName := make([]string, 0, len(issued))
|
||||
for name := range issued {
|
||||
byName = append(byName, name)
|
||||
}
|
||||
sort.Strings(byName)
|
||||
family := map[string]string{} // subject → name
|
||||
var subjects []string
|
||||
for _, name := range names {
|
||||
subject, granted := issued[name]
|
||||
if !granted {
|
||||
return failure(fmt.Sprintf("mesh_wait may not wait on %q: this machine's tool runner is granted %s alone. "+
|
||||
"A family is added only by a reviewed change to the tool runner's manifest (node-tools' module.json, "+
|
||||
"novox/hq ADR 0367 §6); no argument, call or setting adds one", name, strings.Join(byName, ", ")))
|
||||
}
|
||||
if _, twice := family[subject]; !twice {
|
||||
subjects = append(subjects, subject)
|
||||
}
|
||||
family[subject] = name
|
||||
}
|
||||
|
||||
heard, ok, err := s.conn.HearFirst(ctx, subjects, timeout, func(body json.RawMessage) bool { return bodyMatches(body, filter) })
|
||||
if err != nil {
|
||||
return failure("mesh_wait could not wait: " + err.Error())
|
||||
}
|
||||
if !ok {
|
||||
return answerText(map[string]any{"timed-out": true, "waited": timeout.String(), "subjects": names,
|
||||
"read": "nothing matching was heard; read the state once from the verb that owns it, then wait again"})
|
||||
}
|
||||
return answerText(map[string]any{
|
||||
"heard": map[string]any{"subject": family[heard.Subject], "body": heard.Body, "at": heard.At.Format(time.RFC3339Nano)},
|
||||
"read": "an event only wakes you: read the state from the verb that owns it before acting (novox/hq ADR 0367 §4)",
|
||||
})
|
||||
}
|
||||
|
||||
// waitNames is the families a call names: a list of names, or one name.
|
||||
func waitNames(given any) ([]string, error) {
|
||||
var names []string
|
||||
switch v := given.(type) {
|
||||
case string:
|
||||
names = []string{v}
|
||||
case []any:
|
||||
for _, x := range v {
|
||||
name, ok := x.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("mesh_wait's subjects are event names, such as mesh-knowledge.reviewed; %v is not one", x)
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
for i := range names {
|
||||
names[i] = strings.TrimSpace(names[i])
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return nil, fmt.Errorf("mesh_wait names at least one event family in `subjects`, such as mesh-knowledge.reviewed")
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// waitTimeout is the call's timeout: WaitDefault when absent, refused beyond WaitLongest or below a second.
|
||||
func waitTimeout(given any) (time.Duration, error) {
|
||||
if given == nil {
|
||||
return WaitDefault, nil
|
||||
}
|
||||
longest := int(WaitLongest.Seconds())
|
||||
seconds, ok := given.(float64)
|
||||
if !ok {
|
||||
if n, isInt := given.(int); isInt {
|
||||
seconds, ok = float64(n), true
|
||||
}
|
||||
}
|
||||
if !ok || math.IsNaN(seconds) || seconds < 1 || seconds > float64(longest) {
|
||||
return 0, fmt.Errorf("mesh_wait's timeout is a number of seconds from 1 to %d (novox/hq ADR 0367 §1), not %v", longest, given)
|
||||
}
|
||||
return time.Duration(seconds * float64(time.Second)), nil
|
||||
}
|
||||
|
||||
// bodyMatches says whether every field the filter names equals, in the event's body, the value given.
|
||||
func bodyMatches(body json.RawMessage, filter map[string]any) bool {
|
||||
if len(filter) == 0 {
|
||||
return true
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(body, &fields) != nil {
|
||||
return false
|
||||
}
|
||||
for k, want := range filter {
|
||||
got, has := fields[k]
|
||||
if !has {
|
||||
return false
|
||||
}
|
||||
wanted, _ := json.Marshal(want)
|
||||
if !sameJSON(got, wanted) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// sameJSON compares two JSON values as values: 367 and 367.0 are one number, key order does not matter.
|
||||
func sameJSON(a, b []byte) bool {
|
||||
var x, y any
|
||||
da, db := json.NewDecoder(bytes.NewReader(a)), json.NewDecoder(bytes.NewReader(b))
|
||||
if da.Decode(&x) != nil || db.Decode(&y) != nil {
|
||||
return false
|
||||
}
|
||||
ja, _ := json.Marshal(x)
|
||||
jb, _ := json.Marshal(y)
|
||||
return bytes.Equal(ja, jb)
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// waitingRuntime is the tool runner on a machine, issued the families ADR 0367 §5 names, as the controller
|
||||
// composes them from its manifest: the surface over its connection, and the mesh to publish on.
|
||||
func waitingRuntime(t *testing.T) (*Surface, *mt.Mesh) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
m := mt.MembershipOf("node-tools", "desk", false, nil)
|
||||
m.Waits = []bus.WaitIssued{
|
||||
{Event: "mesh-knowledge.reviewed", Subject: "mesh.mod.mesh-knowledge.event.reviewed"},
|
||||
{Event: "operator-channel.decided.claude-code", Subject: "mesh.seat.operator-channel.event.decided.claude-code"},
|
||||
}
|
||||
mesh.Issue(t, m)
|
||||
conn := connect(t, "node-tools", "desk")
|
||||
mt.Until(t, func() error {
|
||||
if got := conn.Membership("node-tools"); got == nil || len(got.Waits) == 0 {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return NewSurface(conn, "desk.node-tools"), mesh
|
||||
}
|
||||
|
||||
// callWait is one tools/call of mesh_wait, answered as an agent host reads it.
|
||||
func callWait(s *Surface, args map[string]any) (text string, isError bool) {
|
||||
raw, _ := json.Marshal(map[string]any{"name": verbWait, "arguments": args})
|
||||
r := s.Handle(Request{JSONRPC: "2.0", ID: json.RawMessage("1"), Method: "tools/call", Params: raw})
|
||||
result, _ := r.Result.(map[string]any)
|
||||
content, _ := result["content"].([]map[string]any)
|
||||
var parts []string
|
||||
for _, c := range content {
|
||||
parts = append(parts, c["text"].(string))
|
||||
}
|
||||
isErr, _ := result["isError"].(bool)
|
||||
return strings.Join(parts, "\n"), isErr
|
||||
}
|
||||
|
||||
// novox/hq ADR 0367 §1: mesh_wait answers the first event that matches — by its family and its filter — with
|
||||
// its subject, body and time, within a second of its publication, and ignores the others.
|
||||
func TestMeshWaitAnswersTheFirstMatchingEvent(t *testing.T) {
|
||||
s, mesh := waitingRuntime(t)
|
||||
type answer struct {
|
||||
text string
|
||||
isErr bool
|
||||
at time.Time
|
||||
}
|
||||
done := make(chan answer, 1)
|
||||
go func() {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"},
|
||||
"filter": map[string]any{"kind": "decision", "number": 367}, "timeout": 20})
|
||||
done <- answer{text, isErr, time.Now()}
|
||||
}()
|
||||
// Once it listens: another record's verdict, another family's event, then the one it waits for.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
mesh.Emit(t, "mesh.mod.mesh-knowledge.event.reviewed", map[string]any{"kind": "decision", "number": 366, "what": "approved"})
|
||||
mesh.Emit(t, "mesh.mod.mesh-knowledge.event.proposed", map[string]any{"kind": "decision", "number": 367})
|
||||
published := time.Now()
|
||||
mesh.Emit(t, "mesh.mod.mesh-knowledge.event.reviewed", map[string]any{"kind": "decision", "number": 367, "what": "approved"})
|
||||
var got answer
|
||||
select {
|
||||
case got = <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("mesh_wait did not answer the matching event")
|
||||
}
|
||||
if got.isErr {
|
||||
t.Fatalf("mesh_wait failed: %s", got.text)
|
||||
}
|
||||
if got.at.Sub(published) > time.Second {
|
||||
t.Errorf("answered %s after the event was published, want within a second", got.at.Sub(published))
|
||||
}
|
||||
var heard struct {
|
||||
Heard struct {
|
||||
Subject string `json:"subject"`
|
||||
Body map[string]any `json:"body"`
|
||||
At time.Time `json:"at"`
|
||||
} `json:"heard"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(got.text), &heard); err != nil {
|
||||
t.Fatalf("not an answer: %s", got.text)
|
||||
}
|
||||
if heard.Heard.Subject != "mesh-knowledge.reviewed" || heard.Heard.Body["number"] != float64(367) || heard.Heard.At.IsZero() {
|
||||
t.Errorf("answered %+v, want record 367's verdict with its time", heard.Heard)
|
||||
}
|
||||
// And the subject it is granted on the asker's own `decided`, by the name a session asks for.
|
||||
go func() {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": "operator-channel.decided.claude-code", "timeout": 20})
|
||||
done <- answer{text, isErr, time.Now()}
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
// A seat's event, said by the router on core NATS here: no stream of the test's bus carries the seat.
|
||||
if err := connect(t, "messenger", "").Say("mesh.seat.operator-channel.event.decided.claude-code", []byte(`{"ask":"a1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case got = <-done:
|
||||
if got.isErr || !strings.Contains(got.text, `"subject": "operator-channel.decided.claude-code"`) {
|
||||
t.Errorf("the asker's decided: %s", got.text)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("mesh_wait did not answer the asker's decided")
|
||||
}
|
||||
}
|
||||
|
||||
// It says it timed out at its timeout, and leaves no subscription behind (ADR 0367 §1, §2).
|
||||
func TestMeshWaitSaysItTimedOutAndKeepsNothing(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
before := s.conn.Subscriptions()
|
||||
start := time.Now()
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"}, "timeout": 1})
|
||||
if took := time.Since(start); took < time.Second || took > 3*time.Second {
|
||||
t.Errorf("a one-second wait took %s", took)
|
||||
}
|
||||
if isErr || !strings.Contains(text, `"timed-out": true`) {
|
||||
t.Errorf("a wait that heard nothing: %s", text)
|
||||
}
|
||||
if after := s.conn.Subscriptions(); after != before {
|
||||
t.Errorf("%d subscriptions before the wait, %d after it", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// Its timeout is four minutes when none is given and ten at most: a longer one, or none at all, is refused
|
||||
// with the bound, never shortened in silence.
|
||||
func TestMeshWaitsTimeoutIsBounded(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
if WaitDefault != 4*time.Minute || WaitLongest != 10*time.Minute {
|
||||
t.Errorf("the bounds are %s and %s, want 4m and 10m (ADR 0367 §1)", WaitDefault, WaitLongest)
|
||||
}
|
||||
for _, timeout := range []any{601, 0, -5, "soon"} {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"}, "timeout": timeout})
|
||||
if !isErr || !strings.Contains(text, "600") {
|
||||
t.Errorf("a timeout of %v: %s", timeout, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0367 §2 and §6: an event outside the granted families is refused, naming the families and how
|
||||
// one is added; no argument — a wildcard, another asker's `decided`, a subject spelled out — widens it.
|
||||
func TestMeshWaitRefusesAnEventOutsideTheGrantedFamilies(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
for _, subjects := range []any{
|
||||
[]any{"mesh-knowledge.proposed"},
|
||||
[]any{"mesh-knowledge.reviewed", "gitea.forge.push"},
|
||||
[]any{"mesh-knowledge.*"},
|
||||
[]any{"operator-channel.decided.shop"},
|
||||
[]any{"operator-channel.decided.*"},
|
||||
[]any{"mesh.mod.mesh-knowledge.event.reviewed"},
|
||||
[]any{">"},
|
||||
[]any{},
|
||||
} {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": subjects, "timeout": 1})
|
||||
if !isErr {
|
||||
t.Errorf("%v was waited on: %s", subjects, text)
|
||||
continue
|
||||
}
|
||||
if len(subjects.([]any)) > 0 && (!strings.Contains(text, "mesh-knowledge.reviewed") || !strings.Contains(text, "manifest")) {
|
||||
t.Errorf("%v refused without naming the families and how one is added: %s", subjects, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A tool runner issued no families waits on nothing, and says so.
|
||||
func TestMeshWaitWithNoFamiliesIssuedWaitsOnNothing(t *testing.T) {
|
||||
mt.New(t)
|
||||
s := NewSurface(connect(t, "node-tools", "desk"), "desk.node-tools")
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"}, "timeout": 1})
|
||||
if !isErr || !strings.Contains(text, "no event families") {
|
||||
t.Errorf("a runtime issued no families: %s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// The mesh MCP server announces seven tools (ADR 0367 v13): ADR 0195's five, mesh_runtimes and mesh_wait, its timeout bounded.
|
||||
func TestMeshWaitIsAnnouncedAmongSeven(t *testing.T) {
|
||||
var names []string
|
||||
var wait map[string]any
|
||||
for _, tool := range discovery() {
|
||||
names = append(names, tool["name"].(string))
|
||||
if tool["name"] == verbWait {
|
||||
wait = tool
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
want := []string{"mesh_call", "mesh_describe", "mesh_machine", "mesh_overview", "mesh_runtimes", "mesh_search", "mesh_wait"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("announced %v, want %v", names, want)
|
||||
}
|
||||
if wait == nil || !isDiscovery(verbWait) {
|
||||
t.Fatal("mesh_wait is not announced")
|
||||
}
|
||||
schema, _ := json.Marshal(wait["inputSchema"])
|
||||
for _, word := range []string{`"subjects"`, `"filter"`, `"timeout"`} {
|
||||
if !strings.Contains(string(schema), word) {
|
||||
t.Errorf("mesh_wait's schema has no %s: %s", word, schema)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A client that hangs up or cancels ends its wait at once: the subscription is closed and nothing is left
|
||||
// waiting for the timeout (the review of issue 745). Over the console's own HTTP, as an agent host calls it.
|
||||
func TestMeshWaitEndsWhenItsClientHangsUp(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
up, err := Serve(s, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = up.Close() })
|
||||
before := s.conn.Subscriptions()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
body := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"mesh_wait",` +
|
||||
`"arguments":{"subjects":["mesh-knowledge.reviewed"],"timeout":600}}}`
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "http://"+up.Address+"/mcp", strings.NewReader(body))
|
||||
req.Header.Set("content-type", "application/json")
|
||||
go func() { _, _ = http.DefaultClient.Do(req) }()
|
||||
mt.Until(t, func() error {
|
||||
if s.conn.Subscriptions() <= before {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
cancel()
|
||||
start := time.Now()
|
||||
for s.conn.Subscriptions() > before {
|
||||
if time.Since(start) > 3*time.Second {
|
||||
t.Fatalf("%d subscriptions 3s after the client hung up, %d before the wait", s.conn.Subscriptions(), before)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// The tool runner stopping ends every wait in flight at once (the second review of issue 745): Close cancels
|
||||
// the requests' context rather than waiting up to ten minutes for a wait to time out.
|
||||
func TestMeshWaitEndsWhenTheConsoleCloses(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
up, err := Serve(s, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := s.conn.Subscriptions()
|
||||
body := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"mesh_wait",` +
|
||||
`"arguments":{"subjects":["mesh-knowledge.reviewed"],"timeout":600}}}`
|
||||
go func() { _, _ = http.Post("http://"+up.Address+"/mcp", "application/json", strings.NewReader(body)) }()
|
||||
mt.Until(t, func() error {
|
||||
if s.conn.Subscriptions() <= before {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
closed := make(chan error, 1)
|
||||
start := time.Now()
|
||||
go func() { closed <- up.Close() }()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("closing the console waited on a mesh_wait in flight")
|
||||
}
|
||||
if took := time.Since(start); took > 3*time.Second {
|
||||
t.Errorf("closing took %s", took)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
if s.conn.Subscriptions() > before {
|
||||
return os.ErrExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package launch
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type lostWrite struct{}
|
||||
|
||||
func (lostWrite) Error() string {
|
||||
return "issues.numbers.issue: the key was written by another writer first"
|
||||
}
|
||||
func (lostWrite) ErrorCode() int { return CodeChanged }
|
||||
|
||||
// A lost compare-and-set is answered to a bundle with its own code, wrapped or not, and the SDK tells it by
|
||||
// that code alone (mesh-sdk's stdio.CodeChanged); every other refusal is the general code, never that one.
|
||||
func TestALostWriteIsAnsweredWithItsOwnCodeAndNothingElseIs(t *testing.T) {
|
||||
if CodeChanged != -32010 {
|
||||
t.Fatalf("the lost-write code is %d; mesh-sdk tells it by -32010", CodeChanged)
|
||||
}
|
||||
for _, err := range []error{lostWrite{}, fmt.Errorf("asked by issues: %w", lostWrite{})} {
|
||||
if got := errorCode(err); got != CodeChanged {
|
||||
t.Errorf("%v is answered %d", err, got)
|
||||
}
|
||||
}
|
||||
if got := errorCode(errors.New("issues keeps no state numbers")); got == CodeChanged || got != -32000 {
|
||||
t.Errorf("a plain refusal is answered %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A bundle that answers the handshake and an empty tool list, and notes each start and its pid in $OUT.
|
||||
const notingBundle = `#!/bin/sh
|
||||
printf 'started %s\n' "$$" >> "$OUT"
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id" ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
// The service manager stops the runtime by signalling its whole group, every bundle with it (issue 1126, ace's
|
||||
// reboot of 2026-10-28: every bundle "exited (SIGTERM); started again in 500ms" for the 15 s the runtime took
|
||||
// to stop, the power bundle among them dialling the system bus again, which restarted the bus during the
|
||||
// shutdown and held it 29 minutes). A bundle that exits while it serves is started again; once the runtime
|
||||
// itself is stopping, none is, by a restart or by a call.
|
||||
func TestABundleIsStartedAgainUnlessTheRuntimeIsStopping(t *testing.T) {
|
||||
saved := RestartFirst
|
||||
RestartFirst = 10 * time.Millisecond
|
||||
t.Cleanup(func() { RestartFirst = saved; shuttingDown.Store(false) })
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
stopping bool
|
||||
starts int
|
||||
}{
|
||||
{"a bundle that dies while the runtime serves", false, 2},
|
||||
{"a bundle signalled with the stopping runtime", true, 1},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
shuttingDown.Store(false)
|
||||
dir := t.TempDir()
|
||||
entry, out := filepath.Join(dir, "bundle"), filepath.Join(dir, "out")
|
||||
if err := os.WriteFile(entry, []byte(notingBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l, err := Start("noting", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
said, _ := os.ReadFile(out)
|
||||
var pid int
|
||||
if _, err := fmt.Sscanf(string(said), "started %d", &pid); err != nil {
|
||||
t.Fatalf("the bundle noted %q", said)
|
||||
}
|
||||
if row.stopping {
|
||||
ShutDown()
|
||||
}
|
||||
if err := syscall.Kill(pid, syscall.SIGTERM); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The restart row ends at its second start; the stopping row watches a window many restart pauses long.
|
||||
deadline := time.Now().Add(250 * time.Millisecond)
|
||||
for {
|
||||
said, _ = os.ReadFile(out)
|
||||
n := strings.Count(string(said), "started")
|
||||
if n > row.starts {
|
||||
t.Fatalf("started %d times, want %d", n, row.starts)
|
||||
}
|
||||
if n == row.starts && !row.stopping {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
if n != row.starts {
|
||||
t.Fatalf("started %d times, want %d", n, row.starts)
|
||||
}
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if row.stopping {
|
||||
// And a bundle first loaded now (a reload) is not started either.
|
||||
if _, err := Start("late", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf); err == nil {
|
||||
t.Error("a bundle was started after the runtime began stopping")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package launch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A bundle that answers `mesh/event` as $ANSWER says: an error, a result, or an answer with neither.
|
||||
const answeringBundle = `#!/bin/sh
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*)
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
||||
printf '{"jsonrpc":"2.0","id":"s1","method":"mesh/subscribe","params":{"pattern":"#"}}\n' ;;
|
||||
*'"method":"mesh/event"'*)
|
||||
case "$ANSWER" in
|
||||
error) printf '{"jsonrpc":"2.0","id":%s,"error":{"code":-32000,"message":"Unexpected end of JSON input"}}\n' "$id" ;;
|
||||
result) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
bare) printf '{"jsonrpc":"2.0","id":%s}\n' "$id" ;;
|
||||
esac ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
type subscribing struct {
|
||||
mu sync.Mutex
|
||||
deliver func(json.RawMessage) error
|
||||
}
|
||||
|
||||
func (b *subscribing) Publish(json.RawMessage) error { return nil }
|
||||
func (b *subscribing) Ask(json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) State(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) Log(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) StreamRead(json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) Watch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
func (b *subscribing) Seat(string, json.RawMessage, func(string, any) (json.RawMessage, error)) (json.RawMessage, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (b *subscribing) Provision(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) ProvisionWatch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
func (b *subscribing) Subscribe(d func(json.RawMessage) error) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.deliver = d
|
||||
return nil
|
||||
}
|
||||
|
||||
func deliverTo(t *testing.T, answer string) error {
|
||||
t.Helper()
|
||||
entry := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.WriteFile(entry, []byte(answeringBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := &subscribing{}
|
||||
l, err := Start("plex", entry, append(os.Environ(), "ANSWER="+answer), b, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
for i := 0; ; i++ {
|
||||
b.mu.Lock()
|
||||
d := b.deliver
|
||||
b.mu.Unlock()
|
||||
if d != nil {
|
||||
return d(json.RawMessage(`{"key":"radarr.download.completed","body":{"title":"x"}}`))
|
||||
}
|
||||
if i > 100 {
|
||||
t.Fatal("the bundle never subscribed")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// An event is taken by any answer that is not an error, and not taken by an error, which reaches the
|
||||
// runtime as the bundle's own words — not as something the runtime failed to read (hq issue 276).
|
||||
func TestAnEventIsTakenByAnyAnswerThatIsNotAnError(t *testing.T) {
|
||||
for _, answer := range []string{"result", "bare"} {
|
||||
if err := deliverTo(t, answer); err != nil {
|
||||
t.Errorf("an answer %q did not take the event: %v", answer, err)
|
||||
}
|
||||
}
|
||||
err := deliverTo(t, "error")
|
||||
var refused *Refused
|
||||
if !errors.As(err, &refused) || refused.Method != "mesh/event" || refused.Message != "Unexpected end of JSON input" {
|
||||
t.Fatalf("an error answer was not the bundle's refusal of mesh/event: %#v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,725 @@
|
||||
// Package launch starts a bundle the runtime serves and speaks MCP over stdio to it (novox/hq ADR
|
||||
// 0188, ADR 0193): `initialize`, `tools/list` once, `tools/call` per call. A Go binary, a Python
|
||||
// script and a Node launcher are the same thing here: an executable that answers those. The runtime
|
||||
// knows no language; it starts the path it is given.
|
||||
package launch
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/wire"
|
||||
)
|
||||
|
||||
// Protocol is the MCP version spoken to a bundle.
|
||||
const Protocol = "2025-03-26"
|
||||
|
||||
// How long a child has for its handshake, and a call before the caller is told it is slow.
|
||||
var (
|
||||
HandshakeTimeout = 10 * time.Second
|
||||
CallTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// Tool is one tool a launched bundle listed, and how to call it.
|
||||
type Tool struct {
|
||||
Name string
|
||||
Description string
|
||||
Input json.RawMessage
|
||||
// Run calls the tool for caller: the bus user the call's subject named, or "" when it named none. A
|
||||
// caller is handed to the bundle in the call's `_meta` (MetaCaller), never among its arguments.
|
||||
Run func(caller string, args json.RawMessage) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// MetaCaller is where in a `tools/call`'s `_meta` the runtime names the caller it took from the subject the
|
||||
// call arrived on (novox/hq issue 365): mesh-sdk's stdio.MetaCaller, which hands it to the tool as
|
||||
// stdio.CallerOf, and says Unproven when it is absent.
|
||||
const MetaCaller = "mesh/caller"
|
||||
|
||||
// callParams is a `tools/call`'s params: the tool, its arguments as the caller sent them, and — only when the
|
||||
// subject named one — the caller, in `_meta`, which the runtime writes and no caller can.
|
||||
func callParams(name, caller string, args json.RawMessage) map[string]any {
|
||||
params := map[string]any{"name": name, "arguments": args}
|
||||
if caller != "" {
|
||||
params["_meta"] = map[string]any{MetaCaller: caller}
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
// Registration is the tools a bundle listed under one name: its module's, or a seat's.
|
||||
type Registration struct {
|
||||
Module string
|
||||
Tools []Tool
|
||||
}
|
||||
|
||||
// Bus is what a launched bundle reaches the mesh through (novox/hq ADR 0193, ADR 0198): the runtime
|
||||
// publishes, asks and subscribes on the module's behalf. Subscribe is asked each time the module's
|
||||
// code subscribes; the runtime binds the module's consumer once and calls deliver for every event,
|
||||
// acknowledging it on the bus when deliver returns nil.
|
||||
//
|
||||
// State and Watch reach the module's state (novox/hq ADR 0201): State answers `get`, `put`, `delete`
|
||||
// and `keys`; Watch hands deliver the current values and then every change, and returns once the
|
||||
// current values are delivered — the watch is the child's, and stops when the child does.
|
||||
type Bus interface {
|
||||
Publish(params json.RawMessage) error
|
||||
Ask(params json.RawMessage) (json.RawMessage, error)
|
||||
Subscribe(deliver func(envelope json.RawMessage) error) error
|
||||
State(verb string, params json.RawMessage) (json.RawMessage, error)
|
||||
Watch(params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error)
|
||||
// Log answers a bundle's `append`, `read`, `last` and `info` on its module's log (novox/hq ADR 0297).
|
||||
Log(verb string, params json.RawMessage) (json.RawMessage, error)
|
||||
// StreamRead answers a bundle's `read` of one of its module's own streams (novox/hq ADR 0363): {stream, subject,
|
||||
// from, limit}, as a log's read.
|
||||
StreamRead(params json.RawMessage) (json.RawMessage, error)
|
||||
// Seat answers a bundle's seat traffic (novox/hq ADR 0259 §3): `publish`, `prove` and `record` at once;
|
||||
// `take` and `answer` bind once and hand each message or proof on through handOn, to whichever child
|
||||
// of the module asked last — a restarted child asks again as its code runs again.
|
||||
Seat(verb string, params json.RawMessage, handOn func(method string, params any) (json.RawMessage, error)) (json.RawMessage, error)
|
||||
// Provision answers a bundle's `get` of one of its module's values at runtime, and ProvisionWatch hands deliver
|
||||
// that value now and each change after, as the module whose bundle asks and never a module the bundle names
|
||||
// (novox/hq ADR 0349 §14, ADR 0370 §4): its config, and its bindings' values that are not secret.
|
||||
Provision(verb string, params json.RawMessage) (json.RawMessage, error)
|
||||
ProvisionWatch(params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error)
|
||||
}
|
||||
|
||||
// EventTimeout bounds how long a bundle has to handle one event before it is offered again.
|
||||
var EventTimeout = 2 * time.Minute
|
||||
|
||||
// Refused is a bundle's error answer to what the runtime asked it: the bundle's own words, as opposed
|
||||
// to the runtime not reaching it (not running, exited, no answer in time). For `mesh/event` it is the
|
||||
// one way a bundle says it did not take an event: any answer that is not an error — `{}`, `null`, no
|
||||
// result at all — takes it (mesh-sdk, "Taking an event"; novox/hq issue 276).
|
||||
type Refused struct {
|
||||
Method string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (r *Refused) Error() string { return r.Message }
|
||||
|
||||
// Restart backoff: a bundle that exits is started again at once, then after growing pauses while it
|
||||
// keeps exiting, back to at once once it has run a while.
|
||||
var (
|
||||
RestartFirst = 500 * time.Millisecond
|
||||
RestartMost = 30 * time.Second
|
||||
RestartSettle = time.Minute
|
||||
)
|
||||
|
||||
// shuttingDown is set once the runtime itself is stopping (ShutDown): from then on no bundle is started again.
|
||||
var shuttingDown atomic.Bool
|
||||
|
||||
// ShutDown says the runtime itself is stopping, before it stops its bundles one by one. The service manager stops
|
||||
// the runtime by signalling its whole group, so every bundle receives SIGTERM with it, and a bundle's exit must not
|
||||
// be taken for a fault: started again, it would run on through the machine's shutdown with nothing left to stop it
|
||||
// (issue 1126: on ace's reboot of 2026-10-28 the restarted bundles dialled the system bus again and started it
|
||||
// during the shutdown, which then waited 29 minutes). After ShutDown no bundle is started again, by a restart or a
|
||||
// call.
|
||||
func ShutDown() { shuttingDown.Store(true) }
|
||||
|
||||
// StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that
|
||||
// ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476).
|
||||
var StopWait = 5 * time.Second
|
||||
|
||||
// Launched is a running bundle: what it registered, and how to stop it.
|
||||
type Launched struct {
|
||||
Registrations []Registration
|
||||
Stop func()
|
||||
}
|
||||
|
||||
// Executable says whether an entrypoint can be started: a bundle the runtime serves is executable,
|
||||
// and one that is not was not built to be served (ADR 0193).
|
||||
func Executable(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || info.IsDir() {
|
||||
return false
|
||||
}
|
||||
return info.Mode().Perm()&0o111 != 0
|
||||
}
|
||||
|
||||
type message struct {
|
||||
JSONRPC string `json:"jsonrpc,omitempty"`
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Params json.RawMessage `json:"params,omitempty"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type child struct {
|
||||
cmd *exec.Cmd
|
||||
stdin io.WriteCloser
|
||||
writeMu sync.Mutex
|
||||
mu sync.Mutex
|
||||
pending map[int64]chan message
|
||||
next int64
|
||||
dead chan struct{}
|
||||
// handled is closed once the child's exit has been handled whole — its state cleared and its
|
||||
// restart decided — which Stop waits for (novox/hq issue 476).
|
||||
handled chan struct{}
|
||||
why error
|
||||
// watches are the state watches this child asked for, stopped when it exits (ADR 0201): a child
|
||||
// that starts again watches again, as its code runs again.
|
||||
watches []func()
|
||||
}
|
||||
|
||||
func (c *child) write(m any) error {
|
||||
b, err := wire.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.writeMu.Lock()
|
||||
defer c.writeMu.Unlock()
|
||||
_, err = c.stdin.Write(append(b, '\n'))
|
||||
return err
|
||||
}
|
||||
|
||||
var stackLine = regexp.MustCompile(`^\s+at\s`)
|
||||
|
||||
// colour is a terminal's colour or style sequence, which node writes into a stack trace when its
|
||||
// environment asks for colour (FORCE_COLOR): taken out before a line is judged or kept as a bundle's
|
||||
// reason, so the reason reads the same in every environment.
|
||||
var colour = regexp.MustCompile("\x1b\\[[0-9;]*m")
|
||||
|
||||
// Start launches one bundle and learns its tools. It fails when the child cannot be started or does
|
||||
// not complete the handshake. A child that exits later is started again on its next call.
|
||||
func Start(module, entry string, env []string, mesh Bus, logf func(string, ...any)) (*Launched, error) {
|
||||
if shuttingDown.Load() {
|
||||
return nil, fmt.Errorf("%s's bundle is not started: the runtime is stopping", module)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var current *child
|
||||
// The child that last subscribed is the one events are handed to: a restarted child subscribes
|
||||
// again as its code is imported, and from then on the module's events are its.
|
||||
var subscriber *child
|
||||
// The child that last asked to take a seat's work or answer its proofs is the one they are handed to.
|
||||
var seatTaker *child
|
||||
handOn := func(method string, params any) (json.RawMessage, error) {
|
||||
mu.Lock()
|
||||
c := seatTaker
|
||||
mu.Unlock()
|
||||
if c == nil {
|
||||
return nil, errors.New(module + "'s bundle is not running to take it")
|
||||
}
|
||||
return c.ask(module, method, params, EventTimeout)
|
||||
}
|
||||
stopped := false
|
||||
deliver := func(envelope json.RawMessage) error {
|
||||
mu.Lock()
|
||||
c := subscriber
|
||||
mu.Unlock()
|
||||
if c == nil {
|
||||
return errors.New(module + "'s bundle is not running to take its events")
|
||||
}
|
||||
// Taken unless the answer is an error; what a result says is not read (see Refused).
|
||||
_, err := c.ask(module, "mesh/event", map[string]any{"envelope": envelope}, EventTimeout)
|
||||
return err
|
||||
}
|
||||
var restart func(after time.Duration)
|
||||
var startedAt time.Time
|
||||
// The restart pauses are read once, here: the exit handling runs on its own goroutine and must not
|
||||
// read the package's settings while a caller (a test) changes them (novox/hq issue 476).
|
||||
pauseFirst, pauseMost, settle := RestartFirst, RestartMost, RestartSettle
|
||||
pause := pauseFirst
|
||||
|
||||
start := func() (*child, error) {
|
||||
cmd := exec.Command(entry)
|
||||
cmd.Env = env
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &child{cmd: cmd, stdin: stdin, pending: map[int64]chan message{}, next: 1, dead: make(chan struct{}),
|
||||
handled: make(chan struct{})}
|
||||
var lastSaid string
|
||||
var saidMu sync.Mutex
|
||||
stderrDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(stderrDone)
|
||||
scan := bufio.NewScanner(stderr)
|
||||
scan.Buffer(make([]byte, 64*1024), 1<<20)
|
||||
for scan.Scan() {
|
||||
line := scan.Text()
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
logf("[%s] %s", module, line)
|
||||
plain := colour.ReplaceAllString(line, "")
|
||||
if !stackLine.MatchString(plain) && !strings.HasPrefix(plain, "Node.js v") {
|
||||
saidMu.Lock()
|
||||
lastSaid = strings.TrimSpace(plain)
|
||||
saidMu.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer close(c.handled)
|
||||
scan := bufio.NewScanner(stdout)
|
||||
scan.Buffer(make([]byte, 64*1024), 16<<20)
|
||||
for scan.Scan() {
|
||||
line := strings.TrimSpace(scan.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var m message
|
||||
if err := json.Unmarshal([]byte(line), &m); err != nil {
|
||||
if len(line) > 120 {
|
||||
line = line[:120]
|
||||
}
|
||||
logf("[mesh-tools] %s's bundle said something that is not a reply: %s", module, line)
|
||||
continue
|
||||
}
|
||||
// The bundle asks the runtime (ADR 0193, ADR 0198): to emit, to call a tool, to hand it the
|
||||
// module's events. Each on the module's behalf, answered when done; nothing else.
|
||||
if m.Method != "" {
|
||||
go func(m message) {
|
||||
var result any = map[string]any{}
|
||||
var err error
|
||||
switch m.Method {
|
||||
case "mesh/publish":
|
||||
err = mesh.Publish(m.Params)
|
||||
case "mesh/ask":
|
||||
var asked json.RawMessage
|
||||
if asked, err = mesh.Ask(m.Params); err == nil {
|
||||
result = asked
|
||||
}
|
||||
case "mesh/subscribe":
|
||||
mu.Lock()
|
||||
subscriber = c
|
||||
mu.Unlock()
|
||||
err = mesh.Subscribe(deliver)
|
||||
case "mesh/seat.publish", "mesh/seat.prove", "mesh/seat.record", "mesh/seat.kinds", "mesh/seat.pull", "mesh/seat.settle":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Seat(strings.TrimPrefix(m.Method, "mesh/seat."), m.Params, nil); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/seat.take", "mesh/seat.answer":
|
||||
mu.Lock()
|
||||
seatTaker = c
|
||||
mu.Unlock()
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Seat(strings.TrimPrefix(m.Method, "mesh/seat."), m.Params, handOn); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/state.get", "mesh/state.put", "mesh/state.delete", "mesh/state.keys",
|
||||
"mesh/state.create", "mesh/state.update", "mesh/state.history":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.State(strings.TrimPrefix(m.Method, "mesh/state."), m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/log.append", "mesh/log.read", "mesh/log.last", "mesh/log.info":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Log(strings.TrimPrefix(m.Method, "mesh/log."), m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/stream.read":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.StreamRead(m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/provision.get":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Provision("get", m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/provision.watch":
|
||||
// As a state's watch: each change asked of this child in order, naming the watch by the
|
||||
// id the child asked it with, and answered once the current value is handed over.
|
||||
var stop func()
|
||||
watch := m.ID
|
||||
stop, err = mesh.ProvisionWatch(m.Params, func(change json.RawMessage) error {
|
||||
_, err := c.ask(module, "mesh/provision", map[string]any{"watch": watch, "change": change}, EventTimeout)
|
||||
return err
|
||||
})
|
||||
if err == nil {
|
||||
c.mu.Lock()
|
||||
if c.why != nil {
|
||||
c.mu.Unlock()
|
||||
stop()
|
||||
} else {
|
||||
c.watches = append(c.watches, stop)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
case "mesh/state.watch":
|
||||
// Each change is asked of this child, in order, naming the watch it is for by
|
||||
// the id the child asked it with — two watches of one state are two handlers;
|
||||
// the watch is answered once the current values have been handed over, so a
|
||||
// bundle that awaits it has the whole of the state before it goes on (ADR 0201).
|
||||
var stop func()
|
||||
watch := m.ID
|
||||
stop, err = mesh.Watch(m.Params, func(change json.RawMessage) error {
|
||||
_, err := c.ask(module, "mesh/state", map[string]any{"watch": watch, "change": change}, EventTimeout)
|
||||
return err
|
||||
})
|
||||
if err == nil {
|
||||
c.mu.Lock()
|
||||
if c.why != nil {
|
||||
c.mu.Unlock()
|
||||
stop()
|
||||
} else {
|
||||
c.watches = append(c.watches, stop)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
default:
|
||||
if len(m.ID) > 0 {
|
||||
_ = c.write(map[string]any{"jsonrpc": "2.0", "id": m.ID,
|
||||
"error": map[string]any{"code": -32601, "message": "the runtime answers no " + m.Method + " from a bundle"}})
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(m.ID) == 0 {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
_ = c.write(map[string]any{"jsonrpc": "2.0", "id": m.ID,
|
||||
"error": map[string]any{"code": errorCode(err), "message": err.Error()}})
|
||||
return
|
||||
}
|
||||
_ = c.write(map[string]any{"jsonrpc": "2.0", "id": m.ID, "result": result})
|
||||
}(m)
|
||||
continue
|
||||
}
|
||||
var id int64
|
||||
if json.Unmarshal(m.ID, &id) != nil {
|
||||
continue
|
||||
}
|
||||
c.mu.Lock()
|
||||
ch := c.pending[id]
|
||||
delete(c.pending, id)
|
||||
c.mu.Unlock()
|
||||
if ch != nil {
|
||||
ch <- m
|
||||
}
|
||||
}
|
||||
<-stderrDone
|
||||
err := cmd.Wait()
|
||||
code := "0"
|
||||
if exit := (*exec.ExitError)(nil); errors.As(err, &exit) {
|
||||
if status, ok := exit.Sys().(syscall.WaitStatus); ok && status.Signaled() {
|
||||
code = unix.SignalName(status.Signal())
|
||||
} else {
|
||||
code = fmt.Sprint(exit.ExitCode())
|
||||
}
|
||||
}
|
||||
saidMu.Lock()
|
||||
why := fmt.Sprintf("%s's bundle exited (%s)", module, code)
|
||||
if lastSaid != "" {
|
||||
why += ": " + lastSaid
|
||||
}
|
||||
saidMu.Unlock()
|
||||
c.mu.Lock()
|
||||
c.why = errors.New(why)
|
||||
watches := c.watches
|
||||
c.watches = nil
|
||||
c.mu.Unlock()
|
||||
for _, stop := range watches {
|
||||
stop()
|
||||
}
|
||||
close(c.dead)
|
||||
mu.Lock()
|
||||
// Only a child that was serving is brought back; one that died in its own handshake was
|
||||
// never accepted, and whoever started it was told why.
|
||||
wasServing := current == c
|
||||
if current == c {
|
||||
current = nil
|
||||
}
|
||||
if subscriber == c {
|
||||
subscriber = nil
|
||||
}
|
||||
if seatTaker == c {
|
||||
seatTaker = nil
|
||||
}
|
||||
wasStopped := stopped
|
||||
// Started again at once if it ran a while; after a growing pause while it keeps exiting.
|
||||
wait := pauseFirst
|
||||
if time.Since(startedAt) < settle {
|
||||
wait = pause
|
||||
if pause *= 2; pause > pauseMost {
|
||||
pause = pauseMost
|
||||
}
|
||||
} else {
|
||||
pause = pauseFirst
|
||||
}
|
||||
mu.Unlock()
|
||||
if wasServing && !wasStopped && shuttingDown.Load() {
|
||||
logf("[mesh-tools] %s; not started again: the runtime is stopping", why)
|
||||
}
|
||||
if !wasStopped && wasServing && !shuttingDown.Load() {
|
||||
// Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner —
|
||||
// is not waiting for a call to bring it back, and a tool bundle back early costs nothing.
|
||||
logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond))
|
||||
restart(wait)
|
||||
}
|
||||
}()
|
||||
if _, err := c.ask(module, "initialize", map[string]any{"protocolVersion": Protocol, "capabilities": map[string]any{},
|
||||
"clientInfo": map[string]any{"name": "node-tools", "version": "1"}}, HandshakeTimeout); err != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
return nil, err
|
||||
}
|
||||
_ = c.write(map[string]any{"jsonrpc": "2.0", "method": "notifications/initialized"})
|
||||
return c, nil
|
||||
}
|
||||
|
||||
asking := func(method string, params any, timeout time.Duration) (json.RawMessage, error) {
|
||||
mu.Lock()
|
||||
c, gone := current, stopped
|
||||
mu.Unlock()
|
||||
if gone {
|
||||
// A stopped bundle is never started again by a call: a process started now would be one
|
||||
// nothing stops (issue 1945).
|
||||
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
|
||||
}
|
||||
if c == nil {
|
||||
if shuttingDown.Load() {
|
||||
return nil, fmt.Errorf("%s's bundle is not started again: the runtime is stopping", module)
|
||||
}
|
||||
fresh, err := start()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mu.Lock()
|
||||
if stopped {
|
||||
mu.Unlock()
|
||||
_ = fresh.cmd.Process.Signal(syscall.SIGTERM)
|
||||
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
|
||||
}
|
||||
if current == nil {
|
||||
current = fresh
|
||||
startedAt = time.Now()
|
||||
} else {
|
||||
_ = fresh.cmd.Process.Signal(syscall.SIGTERM)
|
||||
fresh = current
|
||||
}
|
||||
c = fresh
|
||||
mu.Unlock()
|
||||
}
|
||||
return c.ask(module, method, params, timeout)
|
||||
}
|
||||
|
||||
restart = func(after time.Duration) {
|
||||
go func() {
|
||||
time.Sleep(after)
|
||||
mu.Lock()
|
||||
if stopped || current != nil || shuttingDown.Load() {
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
mu.Unlock()
|
||||
fresh, err := start()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
if !stopped && !shuttingDown.Load() {
|
||||
wait := pause
|
||||
if pause *= 2; pause > pauseMost {
|
||||
pause = pauseMost
|
||||
}
|
||||
logf("[mesh-tools] %s's bundle did not start again: %v; trying in %s", module, err, wait)
|
||||
go restart(wait)
|
||||
}
|
||||
return
|
||||
}
|
||||
if stopped {
|
||||
_ = fresh.cmd.Process.Signal(syscall.SIGTERM)
|
||||
return
|
||||
}
|
||||
if current == nil {
|
||||
current = fresh
|
||||
startedAt = time.Now()
|
||||
} else {
|
||||
_ = fresh.cmd.Process.Signal(syscall.SIGTERM)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
first, err := start()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mu.Lock()
|
||||
current = first
|
||||
startedAt = time.Now()
|
||||
mu.Unlock()
|
||||
|
||||
raw, err := asking("tools/list", map[string]any{}, HandshakeTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var listed struct {
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
InputSchema json.RawMessage `json:"inputSchema"`
|
||||
} `json:"tools"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &listed); err != nil {
|
||||
return nil, fmt.Errorf("%s's bundle listed its tools in a shape that is not MCP's: %w", module, err)
|
||||
}
|
||||
order := []string{}
|
||||
groups := map[string][]Tool{}
|
||||
for _, t := range listed.Tools {
|
||||
under, name := module, t.Name
|
||||
if dot := strings.Index(t.Name, "."); dot >= 0 {
|
||||
under, name = t.Name[:dot], t.Name[dot+1:]
|
||||
}
|
||||
full := t.Name
|
||||
input := t.InputSchema
|
||||
if len(input) == 0 || string(input) == "null" {
|
||||
input = json.RawMessage("{}")
|
||||
}
|
||||
tool := Tool{Name: name, Description: t.Description, Input: input,
|
||||
Run: func(caller string, args json.RawMessage) (json.RawMessage, error) {
|
||||
if len(args) == 0 || string(args) == "null" {
|
||||
args = json.RawMessage("{}")
|
||||
}
|
||||
res, err := asking("tools/call", callParams(full, caller, args), CallTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var called struct {
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
_ = json.Unmarshal(res, &called)
|
||||
text := ""
|
||||
for _, c := range called.Content {
|
||||
if c.Type == "text" {
|
||||
text = c.Text
|
||||
break
|
||||
}
|
||||
}
|
||||
if called.IsError {
|
||||
if text == "" {
|
||||
text = module + "." + full + " failed"
|
||||
}
|
||||
return nil, errors.New(text)
|
||||
}
|
||||
// The bundle's answer is JSON as text; handed back as the value it encodes.
|
||||
if json.Valid([]byte(text)) && text != "" {
|
||||
return json.RawMessage(text), nil
|
||||
}
|
||||
b, _ := json.Marshal(text)
|
||||
return b, nil
|
||||
}}
|
||||
if _, seen := groups[under]; !seen {
|
||||
order = append(order, under)
|
||||
}
|
||||
groups[under] = append(groups[under], tool)
|
||||
}
|
||||
out := &Launched{Stop: func() {
|
||||
mu.Lock()
|
||||
stopped = true
|
||||
c := current
|
||||
current = nil
|
||||
mu.Unlock()
|
||||
if c != nil && c.cmd.Process != nil {
|
||||
_ = c.cmd.Process.Signal(syscall.SIGTERM)
|
||||
// Returns once the bundle's exit is handled, so nothing of it runs on after Stop
|
||||
// (novox/hq issue 476); bounded, for a bundle that does not exit on SIGTERM.
|
||||
select {
|
||||
case <-c.handled:
|
||||
case <-time.After(StopWait):
|
||||
}
|
||||
}
|
||||
}}
|
||||
for _, under := range order {
|
||||
out.Registrations = append(out.Registrations, Registration{Module: under, Tools: groups[under]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *child) ask(module, method string, params any, timeout time.Duration) (json.RawMessage, error) {
|
||||
c.mu.Lock()
|
||||
if c.why != nil {
|
||||
c.mu.Unlock()
|
||||
return nil, c.why
|
||||
}
|
||||
id := c.next
|
||||
c.next++
|
||||
ch := make(chan message, 1)
|
||||
c.pending[id] = ch
|
||||
c.mu.Unlock()
|
||||
if err := c.write(map[string]any{"jsonrpc": "2.0", "id": id, "method": method, "params": params}); err != nil {
|
||||
c.mu.Lock()
|
||||
delete(c.pending, id)
|
||||
c.mu.Unlock()
|
||||
select {
|
||||
case <-c.dead:
|
||||
return nil, c.deathReason()
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case m := <-ch:
|
||||
if m.Error != nil {
|
||||
msg := m.Error.Message
|
||||
if msg == "" {
|
||||
msg = "the bundle refused the request"
|
||||
}
|
||||
return nil, &Refused{Method: method, Message: msg}
|
||||
}
|
||||
return m.Result, nil
|
||||
case <-c.dead:
|
||||
return nil, c.deathReason()
|
||||
case <-timer.C:
|
||||
c.mu.Lock()
|
||||
delete(c.pending, id)
|
||||
c.mu.Unlock()
|
||||
return nil, fmt.Errorf("%s's bundle did not answer %s in %ds", module, method, int(timeout/time.Second))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *child) deathReason() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.why != nil {
|
||||
return c.why
|
||||
}
|
||||
return errors.New("the bundle exited")
|
||||
}
|
||||
|
||||
// CodeChanged is the error code a bundle is answered a lost compare-and-set with (mesh-sdk's
|
||||
// stdio.CodeChanged): told by its code, never by its words (novox/hq ADR 0259).
|
||||
const CodeChanged = -32010
|
||||
|
||||
// errorCode is the code an error is answered to a bundle with: a lost compare-and-set's own, else the
|
||||
// general one.
|
||||
func errorCode(err error) int {
|
||||
var coded interface{ ErrorCode() int }
|
||||
if errors.As(err, &coded) {
|
||||
return coded.ErrorCode()
|
||||
}
|
||||
return -32000
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package launch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A bundle that asks for one of its values and watches it, and writes each change it is handed to $OUT.
|
||||
const provisionBundle = `#!/bin/sh
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*)
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
||||
printf '{"jsonrpc":"2.0","id":"g1","method":"mesh/provision.get","params":{"kind":"config","key":"role"}}\n'
|
||||
printf '{"jsonrpc":"2.0","id":"w1","method":"mesh/provision.watch","params":{"kind":"config","key":"role"}}\n' ;;
|
||||
*'"method":"mesh/provision"'*)
|
||||
printf '%s\n' "$line" >> "$OUT"
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"id":"g1"'*) printf '%s\n' "$line" >> "$OUT" ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
type provisioning struct {
|
||||
subscribing
|
||||
mu sync.Mutex
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (b *provisioning) Provision(verb string, params json.RawMessage) (json.RawMessage, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.asked = append(b.asked, verb+" "+string(params))
|
||||
return json.RawMessage(`"the build machine"`), nil
|
||||
}
|
||||
|
||||
func (b *provisioning) ProvisionWatch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
b.mu.Lock()
|
||||
b.asked = append(b.asked, "watch "+string(params))
|
||||
b.mu.Unlock()
|
||||
return func() {}, deliver(json.RawMessage(`{"kind":"config","key":"role","value":"the build machine"}`))
|
||||
}
|
||||
|
||||
// novox/hq ADR 0349 §14, ADR 0370 §4: a bundle's ask for its values reaches the runtime's bus from its own channel,
|
||||
// answered there, and a watch's change is handed to it naming the watch it asked.
|
||||
func TestABundleAsksForItsValuesAndIsHandedItsWatchsChanges(t *testing.T) {
|
||||
entry := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.WriteFile(entry, []byte(provisionBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := filepath.Join(t.TempDir(), "out")
|
||||
b := &provisioning{}
|
||||
l, err := Start("claude-code", entry, append(os.Environ(), "OUT="+out), b, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
said, _ := os.ReadFile(out)
|
||||
if strings.Count(string(said), "\n") >= 2 {
|
||||
// Each line as the bundle read it, by what it is: the get's answer, and the watch's change.
|
||||
got := map[string]string{}
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(said)), "\n") {
|
||||
var m struct {
|
||||
ID any `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(line), &m); err != nil {
|
||||
t.Fatalf("the bundle read %s", line)
|
||||
}
|
||||
if m.Method != "" {
|
||||
got[m.Method] = string(m.Params)
|
||||
} else {
|
||||
got["answer"] = string(m.Result)
|
||||
}
|
||||
}
|
||||
if got["answer"] != `"the build machine"` {
|
||||
t.Errorf("the get was answered %s", got["answer"])
|
||||
}
|
||||
if got["mesh/provision"] != `{"change":{"kind":"config","key":"role","value":"the build machine"},"watch":"w1"}` {
|
||||
t.Errorf("the watch was handed %s", got["mesh/provision"])
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the bundle was handed %q", said)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
slices.Sort(b.asked)
|
||||
if len(b.asked) != 2 || b.asked[0] != `get {"kind":"config","key":"role"}` || b.asked[1] != `watch {"kind":"config","key":"role"}` {
|
||||
t.Errorf("the runtime's bus was asked %q", b.asked)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package launch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A bundle that asks to take a worker's work and answers each piece it is handed.
|
||||
const takingBundle = `#!/bin/sh
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*)
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
||||
printf '{"jsonrpc":"2.0","id":"s1","method":"mesh/seat.take","params":{"worker":"SEAT_CHANNEL_TELEGRAM_worker"}}\n' ;;
|
||||
*'"method":"mesh/work"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"taken":true}}\n' "$id" ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
type seating struct {
|
||||
subscribing
|
||||
mu sync.Mutex
|
||||
verb string
|
||||
params string
|
||||
handOn func(string, any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
func (b *seating) Seat(verb string, params json.RawMessage, handOn func(string, any) (json.RawMessage, error)) (json.RawMessage, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.verb, b.params, b.handOn = verb, string(params), handOn
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
|
||||
// novox/hq ADR 0259 §3: a bundle's seat traffic reaches the runtime's bus from its own channel, and the work
|
||||
// it takes is handed back to it.
|
||||
func TestABundleTakesASeatsWorkThroughItsOwnChannel(t *testing.T) {
|
||||
entry := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.WriteFile(entry, []byte(takingBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := &seating{}
|
||||
l, err := Start("telegram", entry, os.Environ(), b, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
var handOn func(string, any) (json.RawMessage, error)
|
||||
for i := 0; handOn == nil; i++ {
|
||||
if i > 100 {
|
||||
t.Fatal("the bundle never asked to take its work")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
b.mu.Lock()
|
||||
handOn = b.handOn
|
||||
b.mu.Unlock()
|
||||
}
|
||||
if b.verb != "take" || b.params != `{"worker":"SEAT_CHANNEL_TELEGRAM_worker"}` {
|
||||
t.Fatalf("asked %s %s", b.verb, b.params)
|
||||
}
|
||||
got, err := handOn("mesh/work", map[string]any{"work": map[string]any{"subject": "mesh.seat.channel.accept.show.telegram"}})
|
||||
if err != nil || string(got) != `{"taken":true}` {
|
||||
t.Fatalf("the work was not handed to the bundle: %s %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A bundle that pulls one ask from its worker when it is listed.
|
||||
const pullingBundle = `#!/bin/sh
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*)
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
||||
printf '{"jsonrpc":"2.0","id":"p1","method":"mesh/seat.pull","params":{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker","wait_ms":0}}\n' ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
// 1887/A11: a bundle's pull reaches the runtime's bus from its own channel, as its other seat traffic does.
|
||||
func TestABundlePullsASeatsWorkThroughItsOwnChannel(t *testing.T) {
|
||||
entry := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.WriteFile(entry, []byte(pullingBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := &seating{}
|
||||
l, err := Start("mesh-build-go", entry, os.Environ(), b, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
for i := 0; ; i++ {
|
||||
b.mu.Lock()
|
||||
verb, params := b.verb, b.params
|
||||
b.mu.Unlock()
|
||||
if verb == "pull" {
|
||||
if params != `{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker","wait_ms":0}` {
|
||||
t.Fatalf("pulled with %s", params)
|
||||
}
|
||||
return
|
||||
}
|
||||
if i > 100 {
|
||||
t.Fatal("the bundle's pull never reached the runtime")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package meshtest
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// bundlesWhy returns the reason bundles cannot be launched from a checkout rooted at root, or "" when they can: the
|
||||
// TypeScript and Python bundles the runtime and the console launch need node and the package
|
||||
// @novox/mesh-sdk, which comes from the package registry and is installed beside node-tools/package.json
|
||||
// (novox/hq ADR 0529 clause 1: what merge-check.sh said "NOT TESTED HERE" about is a skip with its reason).
|
||||
func bundlesWhy(root string, haveNode bool) string {
|
||||
if !haveNode {
|
||||
return "node is not installed"
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "node_modules", "@novox", "mesh-sdk")); err != nil {
|
||||
return "@novox/mesh-sdk is not installed in " + filepath.Join(root, "node_modules")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// NeedBundles skips the test, saying why, where the bundles it launches cannot run here.
|
||||
func NeedBundles(t *testing.T) {
|
||||
t.Helper()
|
||||
_, err := exec.LookPath("node")
|
||||
if why := bundlesWhy(moduleRoot(), err == nil); why != "" {
|
||||
t.Skip("NOT TESTED HERE: this test launches bundles, and " + why)
|
||||
}
|
||||
}
|
||||
|
||||
// moduleRoot is node-tools/, where package.json and node_modules are.
|
||||
func moduleRoot() string {
|
||||
_, here, _, _ := runtime.Caller(0)
|
||||
return filepath.Join(filepath.Dir(here), "..", "..")
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
// Package meshtest raises what the controller would, for tests against a real bus: the ASSIGNMENTS
|
||||
// and EVENTS streams, memberships issued by hand, and a fixture's path.
|
||||
//
|
||||
// **Every test has a bus of its own** (novox/hq ADR 0237, the controller's internal/testbus): a server
|
||||
// linked in at the release go.mod pins — the release the mesh runs, held so by a test beside this one —
|
||||
// started for the one test and gone after it. The packages shared one bus and had to run one at a time:
|
||||
// each test raised the streams afresh, and the console discovers every runtime that announces itself on
|
||||
// the bus (ADR 0197), so a runtime from another package's test was, correctly, found. Now the suite runs
|
||||
// as Go runs it, in parallel and under the race detector. A person may still point a run at a bus of
|
||||
// their own with MESH_TEST_NATS_EXTERNAL=1 and MESH_TEST_NATS; then the run is theirs to serialise.
|
||||
package meshtest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// Version is the release of the server the tests run.
|
||||
const Version = server.VERSION
|
||||
|
||||
// URL is the bus of this test alone: started at its first ask, the same one at every ask after — a test
|
||||
// that dials twice, or hands the address to the code it tests, reaches one bus — and shut down when the
|
||||
// test ends.
|
||||
func URL(t *testing.T) string {
|
||||
t.Helper()
|
||||
if os.Getenv("MESH_TEST_NATS_EXTERNAL") == "1" {
|
||||
if url := os.Getenv("MESH_TEST_NATS"); url != "" {
|
||||
return url
|
||||
}
|
||||
t.Fatal("MESH_TEST_NATS_EXTERNAL=1 and MESH_TEST_NATS names no bus")
|
||||
}
|
||||
if url, ok := buses.Load(t); ok {
|
||||
return url.(string)
|
||||
}
|
||||
opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(),
|
||||
NoLog: true, NoSigs: true}
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("a bus for this test could not be made: %v", err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(30 * time.Second) {
|
||||
s.Shutdown()
|
||||
t.Fatal("a bus for this test did not come up within 30s")
|
||||
}
|
||||
url := s.ClientURL()
|
||||
buses.Store(t, url)
|
||||
t.Cleanup(func() {
|
||||
buses.Delete(t)
|
||||
s.Shutdown()
|
||||
s.WaitForShutdown()
|
||||
})
|
||||
return url
|
||||
}
|
||||
|
||||
// buses are the running tests' buses, by test.
|
||||
var buses sync.Map
|
||||
|
||||
// Mesh is the controller's job, done by hand.
|
||||
type Mesh struct {
|
||||
nc *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
}
|
||||
|
||||
// New raises the streams afresh.
|
||||
func New(t *testing.T) *Mesh {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, _ := nc.JetStream()
|
||||
for _, s := range []string{"ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.DeleteStream(s)
|
||||
}
|
||||
if _, err := js.AddStream(&nats.StreamConfig{Name: "ASSIGNMENTS", Subjects: []string{"mesh.assignment.>"},
|
||||
MaxMsgsPerSubject: 1, AllowDirect: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.AddStream(&nats.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
return &Mesh{nc: nc, js: js}
|
||||
}
|
||||
|
||||
// Issue publishes a membership.
|
||||
func (m *Mesh) Issue(t *testing.T, mem bus.Membership) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(mem)
|
||||
if _, err := m.js.Publish(bus.MembershipSubject(mem.Node, mem.Module), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// NextEvent is the subject the next event under a pattern lands on.
|
||||
func (m *Mesh) NextEvent(t *testing.T, pattern string) <-chan string {
|
||||
t.Helper()
|
||||
ch := make(chan string, 1)
|
||||
sub, err := m.nc.Subscribe(pattern, func(msg *nats.Msg) {
|
||||
select {
|
||||
case ch <- msg.Subject:
|
||||
default:
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
_ = m.nc.Flush()
|
||||
return ch
|
||||
}
|
||||
|
||||
// MembershipOf is a membership as the controller issues one on a machine.
|
||||
func MembershipOf(module, node string, plain bool, seats map[string][]string) bus.Membership {
|
||||
own := "mesh.mod." + module
|
||||
serves := []bus.Served{{Subject: own + ".tool.{tool}." + node}}
|
||||
if plain {
|
||||
serves = append(serves, bus.Served{Subject: own + ".tool.{tool}", Queue: "serve." + module})
|
||||
}
|
||||
var verbs []bus.SeatVerb
|
||||
for seat, vs := range seats {
|
||||
for _, v := range vs {
|
||||
verbs = append(verbs, bus.SeatVerb{Seat: seat, Verb: v, Subject: "mesh.seat." + seat + ".tool." + v + "." + node})
|
||||
}
|
||||
}
|
||||
return bus.Membership{Node: node, Module: module, Serves: serves, Seats: verbs,
|
||||
Emits: own + ".event.{event}", Tools: own + ".tool.tools"}
|
||||
}
|
||||
|
||||
// Fixture is a path in node-tools/test/fixtures, which these tests share with the TypeScript ones.
|
||||
func Fixture(name string) string {
|
||||
_, here, _, _ := runtime.Caller(0)
|
||||
return filepath.Join(filepath.Dir(here), "..", "..", "test", "fixtures", name)
|
||||
}
|
||||
|
||||
// Until retries while the bus answers "no responders" — something not yet served.
|
||||
func Until(t *testing.T, try func() error) {
|
||||
t.Helper()
|
||||
var err error
|
||||
for i := 0; i < 50; i++ {
|
||||
if err = try(); err == nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Logs collects what the runtime says.
|
||||
type Logs struct{ lines []string }
|
||||
|
||||
// Logf is a logger that keeps the lines.
|
||||
func (l *Logs) Logf(format string, args ...any) {
|
||||
l.lines = append(l.lines, sprintf(format, args...))
|
||||
}
|
||||
|
||||
// Has says whether a line contains every fragment.
|
||||
func (l *Logs) Has(fragments ...string) bool {
|
||||
for _, line := range l.lines {
|
||||
all := true
|
||||
for _, f := range fragments {
|
||||
all = all && strings.Contains(line, f)
|
||||
}
|
||||
if all {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// All is every line.
|
||||
func (l *Logs) All() string { return strings.Join(l.lines, "\n") }
|
||||
|
||||
// Consumer makes a module's durable consumer on a machine as the controller does: pull, on the
|
||||
// EVENTS stream, filtered to what the module consumes, with a short ack wait so a test sees a
|
||||
// redelivery in seconds rather than the mesh's minutes.
|
||||
func (m *Mesh) Consumer(t *testing.T, node, module string, filters []string, ackWait time.Duration) {
|
||||
t.Helper()
|
||||
cfg := &nats.ConsumerConfig{Durable: node + "_" + module, AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: ackWait, MaxDeliver: 10, DeliverPolicy: nats.DeliverNewPolicy}
|
||||
if len(filters) == 1 {
|
||||
cfg.FilterSubject = filters[0]
|
||||
} else {
|
||||
cfg.FilterSubjects = filters
|
||||
}
|
||||
if _, err := m.js.AddConsumer("EVENTS", cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Emit publishes an event as a module would, into the EVENTS stream.
|
||||
func (m *Mesh) Emit(t *testing.T, subject string, body any) {
|
||||
t.Helper()
|
||||
data, _ := json.Marshal(body)
|
||||
if _, err := m.js.Publish(subject, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Pending is what a module's consumer still holds: delivered and not acknowledged, and not yet
|
||||
// delivered.
|
||||
func (m *Mesh) Pending(t *testing.T, node, module string) (ackPending, notDelivered uint64) {
|
||||
t.Helper()
|
||||
info, err := m.js.ConsumerInfo("EVENTS", node+"_"+module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return uint64(info.NumAckPending), info.NumPending
|
||||
}
|
||||
|
||||
// BucketKeeping makes a module's state afresh as Bucket does, keeping up to history values a key.
|
||||
func (m *Mesh) BucketKeeping(t *testing.T, name string, history uint8) nats.KeyValue {
|
||||
t.Helper()
|
||||
_ = m.js.DeleteKeyValue(name)
|
||||
kv, err := m.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: name, History: history, MaxValueSize: 256 * 1024})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// Bucket makes a module's state afresh as the controller does from the catalogue (novox/hq ADR 0201),
|
||||
// and answers it for writing what is there before a bundle starts.
|
||||
func (m *Mesh) Bucket(t *testing.T, name string) nats.KeyValue {
|
||||
t.Helper()
|
||||
_ = m.js.DeleteKeyValue(name)
|
||||
kv, err := m.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: name, History: 1, MaxValueSize: 256 * 1024})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// Log makes a module's log afresh as the controller does from the catalogue (novox/hq ADR 0297): a file
|
||||
// stream named LOG_<module>_<name> on mesh.log.<module>.<name>.>, limits retention, no age, no cap per
|
||||
// key, the declared cap, 260 KiB a message (a 256 KiB value and its headers), refusing new entries when full, direct get, no delete or
|
||||
// purge. It answers the membership entry the controller issues for it.
|
||||
func (m *Mesh) Log(t *testing.T, module, name string, maxBytes int64) bus.LogIssued {
|
||||
t.Helper()
|
||||
stream := "LOG_" + module + "_" + name
|
||||
subject := "mesh.log." + module + "." + name
|
||||
_ = m.js.DeleteStream(stream)
|
||||
if _, err := m.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{subject + ".>"},
|
||||
Storage: nats.FileStorage, Retention: nats.LimitsPolicy, MaxAge: 0, MaxMsgsPerSubject: -1,
|
||||
MaxBytes: maxBytes, MaxMsgSize: 260 * 1024, Discard: nats.DiscardNew, AllowDirect: true,
|
||||
DenyDelete: true, DenyPurge: true, Replicas: 1,
|
||||
Description: module + "'s log " + name + ", kept as its record (novox/hq ADR 0297)"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return bus.LogIssued{Name: name, Stream: stream, Subject: subject, Writes: true}
|
||||
}
|
||||
|
||||
// Consumers is how many consumers a stream has.
|
||||
func (m *Mesh) Consumers(t *testing.T, stream string) int {
|
||||
t.Helper()
|
||||
info, err := m.js.StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.State.Consumers
|
||||
}
|
||||
|
||||
// ModuleStream makes a module's own stream afresh as the controller does from the catalogue (novox/hq ADR
|
||||
// 0363 §1): named in capitals, taking `mesh.stream.<module>.event.>`, a subject space EVENTS does not take.
|
||||
func (m *Mesh) ModuleStream(t *testing.T, module, name string) {
|
||||
t.Helper()
|
||||
_ = m.js.DeleteStream(name)
|
||||
if _, err := m.js.AddStream(&nats.StreamConfig{Name: name, Subjects: []string{"mesh.stream." + module + ".event.>"},
|
||||
Retention: nats.LimitsPolicy, Discard: nats.DiscardOld, AllowDirect: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ConsumerOn makes a module's durable consumer on a machine on any stream, as Consumer does on EVENTS.
|
||||
func (m *Mesh) ConsumerOn(t *testing.T, stream, node, module string, filters []string, ackWait time.Duration) {
|
||||
t.Helper()
|
||||
cfg := &nats.ConsumerConfig{Durable: node + "_" + module, AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: ackWait, MaxDeliver: 10, DeliverPolicy: nats.DeliverNewPolicy}
|
||||
if len(filters) == 1 {
|
||||
cfg.FilterSubject = filters[0]
|
||||
} else {
|
||||
cfg.FilterSubjects = filters
|
||||
}
|
||||
if _, err := m.js.AddConsumer(stream, cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// PendingOn is what a module's consumer on a stream still holds, as Pending is on EVENTS.
|
||||
func (m *Mesh) PendingOn(t *testing.T, stream, node, module string) (ackPending, notDelivered uint64) {
|
||||
t.Helper()
|
||||
info, err := m.js.ConsumerInfo(stream, node+"_"+module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return uint64(info.NumAckPending), info.NumPending
|
||||
}
|
||||
|
||||
// Messages is how many messages a stream holds.
|
||||
func (m *Mesh) Messages(t *testing.T, stream string) uint64 {
|
||||
t.Helper()
|
||||
info, err := m.js.StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.State.Msgs
|
||||
}
|
||||
|
||||
// RemoveConsumer deletes a module's consumer on a machine on a stream, as the controller does when a grant
|
||||
// it cannot narrow is withdrawn (novox/hq ADR 0363 §6).
|
||||
func (m *Mesh) RemoveConsumer(t *testing.T, stream, node, module string) {
|
||||
t.Helper()
|
||||
if err := m.js.DeleteConsumer(stream, node+"_"+module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package meshtest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The bus the tests run is the bus the mesh runs** (novox/hq ADR 0227 rule 9, ADR 0237): the server linked
|
||||
// into the tests is held to the release the catalogue's bus image is — read from beside this checkout, as
|
||||
// the build seat clones it for a merge check — and, given the facts snapshot, to the release the mesh's bus
|
||||
// server says it runs. A bus upgrade moves the catalogue's pin; this then fails until go.mod's moves with it.
|
||||
func TestTheBusTheTestsRunIsTheBusTheMeshRuns(t *testing.T) {
|
||||
beside := os.Getenv("MESH_CHECK_BESIDE")
|
||||
if beside == "" {
|
||||
beside = filepath.Join("..", "..", "..", "..")
|
||||
}
|
||||
dockerfile := filepath.Join(beside, "mesh-catalog", "modules", "nats", "Dockerfile")
|
||||
raw, err := os.ReadFile(dockerfile)
|
||||
switch {
|
||||
case err == nil:
|
||||
pinned := regexp.MustCompile(`upstream: nats ([0-9.]+)-alpine`).FindSubmatch(raw)
|
||||
if pinned == nil {
|
||||
t.Fatalf("%s says no release its digest is", dockerfile)
|
||||
}
|
||||
if string(pinned[1]) != Version {
|
||||
t.Errorf("the tests run bus %s and the catalogue's bus image is %s: move go.mod's nats-server pin with the image",
|
||||
Version, pinned[1])
|
||||
}
|
||||
case os.IsNotExist(err):
|
||||
t.Logf("the catalogue is not beside this checkout, so its bus image is not compared")
|
||||
default:
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := os.Getenv("MESH_FACTS")
|
||||
if path == "" {
|
||||
t.Logf("no MESH_FACTS: the bus the mesh runs is compared in a merge check, which has it")
|
||||
return
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var f struct {
|
||||
Taken string `json:"taken"`
|
||||
Versions struct {
|
||||
Bus string `json:"bus"`
|
||||
} `json:"versions"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Versions.Bus != "" && strings.TrimPrefix(f.Versions.Bus, "v") != Version {
|
||||
t.Errorf("the tests run bus %s and the mesh's bus runs %s (facts of %s)", Version, f.Versions.Bus, f.Taken)
|
||||
}
|
||||
}
|
||||
|
||||
// Each test is given a bus of its own, and the same one at every ask.
|
||||
func TestEachTestHasABusOfItsOwn(t *testing.T) {
|
||||
var first string
|
||||
t.Run("one", func(t *testing.T) {
|
||||
first = URL(t)
|
||||
if URL(t) != first {
|
||||
t.Fatal("two buses for one test")
|
||||
}
|
||||
})
|
||||
t.Run("two", func(t *testing.T) {
|
||||
if URL(t) == first {
|
||||
t.Fatal("two tests were given one bus")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// What merge-check.sh said was "NOT TESTED HERE" is a skip with its reason: no node, or no package, each says so.
|
||||
func TestBundlesCannotBeLaunchedWithoutNodeOrTheSDKAndTheSkipSaysWhy(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if why := bundlesWhy(root, false); !strings.Contains(why, "node is not installed") {
|
||||
t.Errorf("no node said %q", why)
|
||||
}
|
||||
if why := bundlesWhy(root, true); !strings.Contains(why, "@novox/mesh-sdk") {
|
||||
t.Errorf("no package said %q", why)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "node_modules", "@novox", "mesh-sdk"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := bundlesWhy(root, true); why != "" {
|
||||
t.Errorf("node and the package present said %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// The checker runs node-tools' Go level by Go's convention and its Node level by package.json's test script
|
||||
// (novox/hq ADR 0529): no script of the repository's own runs beside them, and no module but node-tools is built from
|
||||
// here (ADR 0524 §5, the toolchain image retired).
|
||||
func TestTheRepositoryHoldsNoCheckScriptAndNodeToolsIsItsOnlyModule(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "..")
|
||||
if _, err := os.Stat(filepath.Join(root, "merge-check.sh")); err == nil {
|
||||
t.Errorf("merge-check.sh is back: the checker's levels run the tests")
|
||||
}
|
||||
var found []string
|
||||
err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && (d.Name() == "node_modules" || d.Name() == ".git") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == "module.json" {
|
||||
found = append(found, filepath.ToSlash(p))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || !strings.HasSuffix(found[0], "/node-tools/module.json") {
|
||||
t.Errorf("the module manifests are %v, want node-tools' alone", found)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package meshtest
|
||||
|
||||
import "fmt"
|
||||
|
||||
func sprintf(format string, args ...any) string { return fmt.Sprintf(format, args...) }
|
||||
@@ -0,0 +1,102 @@
|
||||
// Package npmlock says whether a package's lock file still describes its package.json (novox/hq issue 624).
|
||||
//
|
||||
// `npm ci` installs only from a lock file in step with package.json, and refuses the whole install when it
|
||||
// is not. node-tools' lock fell behind twice — the SDK's range moved and the toolchain's development
|
||||
// dependencies were added to package.json alone — and nothing here noticed, because the image's build runs
|
||||
// `npm install` and never reads the lock; the lab's runs, which run `npm ci`, stopped at setup instead.
|
||||
// The check is offline and needs no node: it reads the two files the way `npm ci` compares them first.
|
||||
package npmlock
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
DevDependencies map[string]string `json:"devDependencies"`
|
||||
OptionalDependencies map[string]string `json:"optionalDependencies"`
|
||||
PeerDependencies map[string]string `json:"peerDependencies"`
|
||||
}
|
||||
|
||||
type lock struct {
|
||||
LockfileVersion int `json:"lockfileVersion"`
|
||||
Packages map[string]manifest `json:"packages"`
|
||||
}
|
||||
|
||||
// Check reads dir's package.json and package-lock.json and names every way the lock is out of step: a
|
||||
// declared range the lock's root entry records differently or not at all, a range the lock records that
|
||||
// package.json no longer declares, and a declared dependency the lock resolved no package for. None means
|
||||
// in step.
|
||||
func Check(dir string) ([]string, error) {
|
||||
var pkg manifest
|
||||
if err := readJSON(filepath.Join(dir, "package.json"), &pkg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lk lock
|
||||
if err := readJSON(filepath.Join(dir, "package-lock.json"), &lk); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lk.LockfileVersion < 2 {
|
||||
return []string{fmt.Sprintf("package-lock.json is lockfileVersion %d; npm ci needs 2 or later here", lk.LockfileVersion)}, nil
|
||||
}
|
||||
root, ok := lk.Packages[""]
|
||||
if !ok {
|
||||
return []string{`package-lock.json has no root entry (packages[""])`}, nil
|
||||
}
|
||||
var out []string
|
||||
for _, kind := range []struct {
|
||||
name string
|
||||
want, have map[string]string
|
||||
}{
|
||||
{"dependencies", pkg.Dependencies, root.Dependencies},
|
||||
{"devDependencies", pkg.DevDependencies, root.DevDependencies},
|
||||
{"optionalDependencies", pkg.OptionalDependencies, root.OptionalDependencies},
|
||||
{"peerDependencies", pkg.PeerDependencies, root.PeerDependencies},
|
||||
} {
|
||||
for _, name := range sortedKeys(kind.want) {
|
||||
have, ok := kind.have[name]
|
||||
switch {
|
||||
case !ok:
|
||||
out = append(out, fmt.Sprintf("%s %s %q is in package.json and not in the lock", kind.name, name, kind.want[name]))
|
||||
case have != kind.want[name]:
|
||||
out = append(out, fmt.Sprintf("%s %s is %q in package.json and %q in the lock", kind.name, name, kind.want[name], have))
|
||||
}
|
||||
if kind.name == "peerDependencies" {
|
||||
continue
|
||||
}
|
||||
if _, ok := lk.Packages["node_modules/"+name]; !ok {
|
||||
out = append(out, fmt.Sprintf("%s %s has no resolved package in the lock", kind.name, name))
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(kind.have) {
|
||||
if _, ok := kind.want[name]; !ok {
|
||||
out = append(out, fmt.Sprintf("%s %s %q is in the lock and no longer in package.json", kind.name, name, kind.have[name]))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func readJSON(path string, v any) error {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(b, v); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]string) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package npmlock
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestNodeToolsLockIsInStepWithItsPackage: the lab installs node-tools with `npm ci`, which refuses a lock
|
||||
// out of step with package.json, so a change to package.json without its lock fails here and not at a
|
||||
// lab run's setup (novox/hq issue 624).
|
||||
func TestNodeToolsLockIsInStepWithItsPackage(t *testing.T) {
|
||||
out, err := Check(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(out) > 0 {
|
||||
t.Fatalf("node-tools/package-lock.json is out of step with package.json; run `npm install --package-lock-only` in node-tools:\n%s", strings.Join(out, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestALockBehindItsPackageIsNamed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, dir, "package.json", `{"dependencies":{"@novox/mesh-sdk":"^0.1.6","nats":"^2.29.0"},"devDependencies":{"pg":"^8"}}`)
|
||||
write(t, dir, "package-lock.json", `{"lockfileVersion":3,"packages":{
|
||||
"":{"dependencies":{"@novox/mesh-sdk":"^0.1.0","nats":"^2.29.0","left":"^1"}},
|
||||
"node_modules/@novox/mesh-sdk":{"version":"0.1.1"},
|
||||
"node_modules/nats":{"version":"2.29.3"}}}`)
|
||||
out, err := Check(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(out, "\n")
|
||||
for _, want := range []string{
|
||||
`dependencies @novox/mesh-sdk is "^0.1.6" in package.json and "^0.1.0" in the lock`,
|
||||
`dependencies left "^1" is in the lock and no longer in package.json`,
|
||||
`devDependencies pg "^8" is in package.json and not in the lock`,
|
||||
`devDependencies pg has no resolved package in the lock`,
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if len(out) != 4 {
|
||||
t.Errorf("want 4 findings, got %d:\n%s", len(out), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALockInStepIsQuiet(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, dir, "package.json", `{"dependencies":{"nats":"^2.29.0"}}`)
|
||||
write(t, dir, "package-lock.json", `{"lockfileVersion":3,"packages":{"":{"dependencies":{"nats":"^2.29.0"}},"node_modules/nats":{"version":"2.29.3"}}}`)
|
||||
out, err := Check(dir)
|
||||
if err != nil || len(out) != 0 {
|
||||
t.Fatalf("want quiet, got %v %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOldLockfileVersionIsNamed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, dir, "package.json", `{}`)
|
||||
write(t, dir, "package-lock.json", `{"lockfileVersion":1,"dependencies":{}}`)
|
||||
out, err := Check(dir)
|
||||
if err != nil || len(out) != 1 {
|
||||
t.Fatalf("want one finding, got %v %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func write(t *testing.T, dir, name, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
// Package provide is the runtime's half of node-tools being a provider of `mcp-endpoint`: it runs the mesh SDK's
|
||||
// provider helper, so a consumer handed to the provisioner (claude-code's `${bound:mcp-endpoint:port}`) is answered
|
||||
// from an entry node-tools writes in its own bucket, not from the controller's binding file (novox/hq ADR 0349
|
||||
// clause 12, issue 791).
|
||||
//
|
||||
// The offer carries no credential: the console's authority is the machine's login (ADR 0152, 0175), so the helper
|
||||
// reads nothing from the vault, needs no consumers' file and the entry names none. node-tools is not marked moved:
|
||||
// its manifest says `"answered-by": "helper"` for the provision, and the helper runs whenever the runtime does
|
||||
// (issue 791), as route-proxy's does (issue 792).
|
||||
package provide
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
const (
|
||||
// Module is the provider: the runtime's own module.
|
||||
Module = "node-tools"
|
||||
// Provision is what node-tools provides, as its manifest says.
|
||||
Provision = "mcp-endpoint"
|
||||
// ServedPort is the port its manifest serves (`serves.mcp-endpoint.port`): where the console listens.
|
||||
ServedPort = 4270
|
||||
)
|
||||
|
||||
// Offers is what node-tools serves, per provision.
|
||||
func Offers() map[string]provider.Offer {
|
||||
return map[string]provider.Offer{Provision: {
|
||||
Credential: provider.NoCredential,
|
||||
Serves: map[string]any{"port": ServedPort},
|
||||
}}
|
||||
}
|
||||
|
||||
// busOf is the helper's bus over the runtime's own state: the provider's bucket is its state `bound`, and the
|
||||
// controller's assignments are the state it reads, `mesh-controller.assignments` (both declared in the manifest).
|
||||
type busOf struct{ conn *bus.Conn }
|
||||
|
||||
func (b busOf) state(bucket string) (string, error) {
|
||||
switch bucket {
|
||||
case provider.BucketOf(Module):
|
||||
return "bound", nil
|
||||
case provider.AssignmentsBucket:
|
||||
return "mesh-controller.assignments", nil
|
||||
}
|
||||
return "", fmt.Errorf("%s is not a bucket this provider keeps or reads", bucket)
|
||||
}
|
||||
|
||||
func (b busOf) Watch(ctx context.Context, bucket string, handle func(key string, value []byte)) error {
|
||||
name, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stop, err := b.conn.StateWatch(Module, name, "", func(ch bus.StateChange) error {
|
||||
if ch.Op == "delete" {
|
||||
handle(ch.Key, nil)
|
||||
} else {
|
||||
handle(ch.Key, ch.Value)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { <-ctx.Done(); stop() }()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b busOf) Put(_ context.Context, bucket, key string, value []byte) error {
|
||||
name, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = b.conn.StatePut(Module, name, key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busOf) Delete(_ context.Context, bucket, key string) error {
|
||||
name, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.conn.StateDelete(Module, name, key)
|
||||
}
|
||||
|
||||
// New is the helper answering node-tools's consumers on node.
|
||||
func New(conn *bus.Conn, node string, every time.Duration, logf func(string, ...any)) *provider.Provider {
|
||||
return provider.New(provider.Options{
|
||||
Node: node,
|
||||
Module: Module,
|
||||
Offers: Offers(),
|
||||
Bus: busOf{conn: conn},
|
||||
Every: every,
|
||||
Logf: logf,
|
||||
})
|
||||
}
|
||||
|
||||
// Run answers the consumers until ctx ends. The state is not reachable until the mesh has issued the runtime its
|
||||
// membership and made the buckets, so a start that cannot watch is tried again, not given up.
|
||||
func Run(ctx context.Context, conn *bus.Conn, node string, every time.Duration, logf func(string, ...any)) {
|
||||
for delay := 2 * time.Second; ctx.Err() == nil; delay = min(delay*2, time.Minute) {
|
||||
attempt, cancel := context.WithCancel(ctx)
|
||||
err := New(conn, node, every, logf).Run(attempt)
|
||||
cancel()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logf("[mesh-tools] the provider helper stopped: %v; trying again in %s", err, delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start runs the helper when this process is the machine's runtime (the node-tools credential, on a machine's
|
||||
// name), with no condition on the environment, and returns what ends it. A runtime on another credential, a module's
|
||||
// own in a test, is not the provider and answers nobody.
|
||||
func Start(conn *bus.Conn, module, node string, every time.Duration, logf func(string, ...any)) (stop func()) {
|
||||
if module != Module || node == "" {
|
||||
return func() {}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
logf("[mesh-tools] answering %s's consumers of %s through the provider helper", Module, Provision)
|
||||
go Run(ctx, conn, node, every, logf)
|
||||
return cancel
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package provide
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// claude-code's facts.json template, as the catalogue keeps it (modules/claude-code/files/facts.json).
|
||||
const factsTemplate = `{
|
||||
"node": "${machine:name}",
|
||||
"console": "http://127.0.0.1:${bound:mcp-endpoint:port}/mcp"
|
||||
}
|
||||
`
|
||||
|
||||
// factsToday is that file as the controller composes it today, from the port node-tools serves.
|
||||
const factsToday = `{
|
||||
"node": "anchor",
|
||||
"console": "http://127.0.0.1:4270/mcp"
|
||||
}
|
||||
`
|
||||
|
||||
// fill is what the consumer's client does with the template: the machine's name, and each ${bound:<provision>:<key>}
|
||||
// from the provider's entry.
|
||||
func fill(t *testing.T, template string, entry map[string]any) string {
|
||||
t.Helper()
|
||||
values, _ := entry["values"].(map[string]any)
|
||||
out := strings.ReplaceAll(template, "${machine:name}", "anchor")
|
||||
for key, v := range values {
|
||||
raw, _ := json.Marshal(v)
|
||||
out = strings.ReplaceAll(out, "${bound:mcp-endpoint:"+key+"}", string(raw))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTheRuntimeAnswersAHandedConsumersMcpEndpointThroughTheHelper(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
bound := mesh.Bucket(t, "node-tools_bound")
|
||||
assignments := mesh.Bucket(t, "mesh-controller_assignments")
|
||||
|
||||
mem := mt.MembershipOf("node-tools", "anchor", false, nil)
|
||||
mem.State = []bus.StateIssued{
|
||||
{Name: "bound", Bucket: "node-tools_bound", Writes: true},
|
||||
{Name: "mesh-controller.assignments", Bucket: "mesh-controller_assignments"},
|
||||
}
|
||||
mesh.Issue(t, mem)
|
||||
|
||||
// claude-code, handed on this machine, is bound to node-tools here for mcp-endpoint (issue 788's plan entry).
|
||||
plan := `{"node":"anchor","module":"claude-code","provisioning":{"provisions":[{"provision":"mcp-endpoint",` +
|
||||
`"provider":"anchor","provider-module":"node-tools","at":"127.0.0.1","from":"anchor","as":"mesh_anchor_claude_code"}]}}`
|
||||
if _, err := assignments.Put("anchor.claude-code", []byte(plan)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("node-tools")
|
||||
|
||||
said := &mt.Logs{}
|
||||
t.Cleanup(Start(conn, "node-tools", "anchor", 20*time.Millisecond, said.Logf))
|
||||
|
||||
var entry map[string]any
|
||||
mt.Until(t, func() error {
|
||||
kept, err := bound.Get("anchor.claude-code.mcp-endpoint")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(kept.Value(), &entry)
|
||||
})
|
||||
values, _ := entry["values"].(map[string]any)
|
||||
for key, want := range map[string]any{"port": float64(4270), "at": "127.0.0.1", "from": "anchor"} {
|
||||
if values[key] != want {
|
||||
t.Errorf("the entry's %s is %v, want %v (entry %v)", key, values[key], want, entry)
|
||||
}
|
||||
}
|
||||
if entry["as"] != "mesh_anchor_claude_code" {
|
||||
t.Errorf("the entry's as is %v", entry["as"])
|
||||
}
|
||||
if c, has := entry["credential"]; has {
|
||||
t.Errorf("an offer with no credential names one: %v", c)
|
||||
}
|
||||
if got := fill(t, factsTemplate, entry); got != factsToday {
|
||||
t.Errorf("claude-code's facts.json fills to\n%s\nwant today's\n%s", got, factsToday)
|
||||
}
|
||||
}
|
||||
|
||||
// The port the offer serves is the port the manifest serves and the console listens on.
|
||||
func TestTheOfferServesThePortTheManifestServes(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Serves map[string]map[string]any `json:"serves"`
|
||||
Moved bool `json:"moved"`
|
||||
Provides []struct {
|
||||
Name string `json:"name"`
|
||||
AnsweredBy string `json:"answered-by"`
|
||||
} `json:"provides"`
|
||||
Resources []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Env map[string]string `json:"env"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := m.Serves[Provision]["port"]; got != float64(ServedPort) {
|
||||
t.Errorf("the manifest serves %v for %s, the offer %d", got, Provision, ServedPort)
|
||||
}
|
||||
if m.Moved {
|
||||
t.Error("node-tools is marked moved: that also hands the runtime's bus account to the vault (issue 791 round 1)")
|
||||
}
|
||||
if len(m.Provides) != 1 || m.Provides[0].Name != Provision || m.Provides[0].AnsweredBy != "helper" {
|
||||
t.Errorf("node-tools does not say its %s is answered by the helper: %+v", Provision, m.Provides)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r.ID == "consumers" {
|
||||
t.Error("the manifest still carries a consumers file the helper does not need")
|
||||
}
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Name == "files" || len(a.Env) > 0 {
|
||||
t.Errorf("the build still carries %+v, which only a moved module needs", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only the machine's runtime is the provider: a process on another credential, or with no machine, answers nobody.
|
||||
func TestOnlyTheMachinesRuntimeStartsTheHelper(t *testing.T) {
|
||||
said := &mt.Logs{}
|
||||
for _, c := range []struct{ module, node string }{{"claude-code", "anchor"}, {"node-tools", ""}} {
|
||||
Start(nil, c.module, c.node, time.Second, said.Logf)()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0197: what a runtime serves, it announces — the NATS services protocol's discovery,
|
||||
// read here with NATS's own types, each endpoint a subject actually served — and the announcement
|
||||
// follows a re-issued membership.
|
||||
func TestTheRuntimeAnnouncesWhatItServesInTheServicesProtocol(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", false, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "anchor", false, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
stop, err := Run(nodeTools, []Served{
|
||||
{"alpha", []string{mt.Fixture("many-alpha.serve.mjs")}},
|
||||
{"beta", []string{mt.Fixture("many-beta.serve.mjs")}},
|
||||
}, nil, (&mt.Logs{}).Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
info := func() micro.Info {
|
||||
t.Helper()
|
||||
msg, err := nc.Request("$SRV.INFO.node-tools.anchor", nil, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var i micro.Info
|
||||
if err := json.Unmarshal(msg.Data, &i); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return i
|
||||
}
|
||||
subjects := func(i micro.Info) string {
|
||||
var out []string
|
||||
for _, e := range i.Endpoints {
|
||||
out = append(out, e.Subject+"|"+e.QueueGroup+"|"+e.Metadata["kind"]+"|"+e.Metadata["module"]+"|"+e.Metadata["seat"]+"|"+e.Metadata["scope"]+"|"+e.Metadata["interchangeable"])
|
||||
}
|
||||
sort.Strings(out)
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
got := info()
|
||||
if got.Type != micro.InfoResponseType || got.Name != "node-tools" || got.ID != "anchor" || got.Version == "" {
|
||||
t.Errorf("identity: %+v", got.ServiceIdentity)
|
||||
}
|
||||
want := strings.Join([]string{
|
||||
"mesh.mod.alpha.tool.one.anchor||tool|alpha|||false",
|
||||
"mesh.mod.alpha.tool.two.anchor||tool|alpha|||false",
|
||||
"mesh.mod.beta.tool.five.anchor||tool|beta|||false",
|
||||
"mesh.mod.beta.tool.four.anchor||tool|beta|||false",
|
||||
"mesh.mod.beta.tool.three.anchor||tool|beta|||false",
|
||||
"mesh.seat.node-shelf.tool.clear.anchor||seat|beta|node-shelf|node|false",
|
||||
"mesh.seat.node-shelf.tool.list.anchor||seat|beta|node-shelf|node|false",
|
||||
}, "\n")
|
||||
if s := subjects(got); s != want {
|
||||
t.Errorf("announced:\n%s\nwant:\n%s", s, want)
|
||||
}
|
||||
for _, e := range got.Endpoints {
|
||||
if e.Metadata["node"] != "anchor" || e.Metadata["description"] == "" {
|
||||
t.Errorf("endpoint metadata: %+v", e)
|
||||
}
|
||||
// A seat's verb carries its schema; a module's tool does not — it outgrew the bus (2026-10-04).
|
||||
_, hasSchema := e.Metadata["schema"]
|
||||
if e.Metadata["kind"] == "seat" && !json.Valid([]byte(e.Metadata["schema"])) {
|
||||
t.Errorf("a seat's verb without a schema: %+v", e)
|
||||
}
|
||||
if e.Metadata["kind"] == "tool" && hasSchema {
|
||||
t.Errorf("a module's tool still announces its schema: %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
// Every subject announced is answered.
|
||||
asker := connect(t, "console", "workstation")
|
||||
for _, e := range got.Endpoints {
|
||||
if _, err := asker.Ask("", map[string]any{}, e.Subject); err != nil {
|
||||
t.Errorf("announced %s and does not answer it: %v", e.Subject, err)
|
||||
}
|
||||
}
|
||||
|
||||
// PING answers with the same identity, and a request for another service is not answered.
|
||||
if msg, err := nc.Request("$SRV.PING", nil, 2*time.Second); err != nil || !strings.Contains(string(msg.Data), micro.PingResponseType) {
|
||||
t.Errorf("ping: %v %v", msg, err)
|
||||
}
|
||||
if _, err := nc.Request("$SRV.INFO.somebody-else", nil, 300*time.Millisecond); err == nil {
|
||||
t.Error("answered a request for another service")
|
||||
}
|
||||
|
||||
// alpha is re-issued a plain subject: the announcement says so, and says it is interchangeable.
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", true, nil))
|
||||
var after string
|
||||
for i := 0; i < 40; i++ {
|
||||
after = subjects(info())
|
||||
if strings.Contains(after, "mesh.mod.alpha.tool.one|serve.alpha|tool|alpha|||true") {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if !strings.Contains(after, "mesh.mod.alpha.tool.one|serve.alpha|tool|alpha|||true") ||
|
||||
!strings.Contains(after, "mesh.mod.alpha.tool.one.anchor||tool|alpha|||true") {
|
||||
t.Errorf("after a re-issued membership:\n%s", after)
|
||||
}
|
||||
_ = bus.Served{}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
type saidCaller struct {
|
||||
Caller string `json:"caller"`
|
||||
Arguments map[string]any `json:"arguments"`
|
||||
}
|
||||
|
||||
// **A module is told who called it, from the subject the call arrived on** (novox/hq issue 365): a call on the
|
||||
// caller-named subject reaches the bundle with that caller in the call's `_meta`, for a module's tool, on its
|
||||
// machine's address, and for a seat's verb; a caller named in the arguments is only an argument.
|
||||
func TestACallOnTheCallerNamedSubjectReachesTheModuleWithItsCaller(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", true, map[string][]string{"node-desk": {"who"}}))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
stop, err := Run(nodeTools, []Served{{"ledger", []string{mt.Fixture("caller-echo.py")}}}, nil, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop() // returns once the bundle's exit is handled (novox/hq issue 476)
|
||||
asker, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "shanks",
|
||||
User: "shanks.node-tools", Password: "unused"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asker.Logf = func(string, ...any) {}
|
||||
defer asker.Close()
|
||||
|
||||
for _, key := range []string{"ledger.who", "ledger.who@anchor", "seat:node-desk.who@anchor"} {
|
||||
var got saidCaller
|
||||
mt.Until(t, func() error {
|
||||
answered, err := asker.Ask(key, map[string]any{"caller": "novox.node-tools",
|
||||
"_meta": map[string]any{"mesh/caller": "novox.controller"}}, "")
|
||||
if err == nil {
|
||||
err = json.Unmarshal(answered.Result, &got)
|
||||
}
|
||||
return err
|
||||
})
|
||||
if got.Caller != "shanks.node-tools" {
|
||||
t.Errorf("%s reached the module with caller %q, want the asker's bus user", key, got.Caller)
|
||||
}
|
||||
if got.Arguments["caller"] != "novox.node-tools" {
|
||||
t.Errorf("%s: the arguments did not reach the module as sent: %v", key, got.Arguments)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The subject that names no caller is still served, for one release (retired by hq issue 464), and says so:
|
||||
// the module is named no caller, whatever the body claims.
|
||||
func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", true, map[string][]string{"node-desk": {"who"}}))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
stop, err := Run(nodeTools, []Served{{"ledger", []string{mt.Fixture("caller-echo.py")}}}, nil, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop() // returns once the bundle's exit is handled (novox/hq issue 476)
|
||||
old, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer old.Close()
|
||||
body, _ := json.Marshal(map[string]any{"_meta": map[string]any{"mesh/caller": "novox.controller"}})
|
||||
for _, subject := range []string{"mesh.mod.ledger.tool.who", "mesh.mod.ledger.tool.who.anchor", "mesh.seat.node-desk.tool.who.anchor"} {
|
||||
var reply struct {
|
||||
Result saidCaller `json:"result"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
msg, err := old.Request(subject, body, 5*time.Second)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(msg.Data, &reply)
|
||||
}
|
||||
return err
|
||||
})
|
||||
if reply.Error != "" || reply.Result.Caller != "" {
|
||||
t.Errorf("%s: the module was named caller %q (error %q), want none", subject, reply.Result.Caller, reply.Error)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A logger safe to write from the runtime's goroutines.
|
||||
type lines struct {
|
||||
mu sync.Mutex
|
||||
l []string
|
||||
}
|
||||
|
||||
func (s *lines) logf(format string, args ...any) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.l = append(s.l, strings.TrimSpace(sprintf(format, args...)))
|
||||
}
|
||||
|
||||
func (s *lines) all() string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return strings.Join(s.l, "\n")
|
||||
}
|
||||
|
||||
func sprintf(format string, args ...any) string { return fmtSprintf(format, args...) }
|
||||
|
||||
func countLines(t *testing.T, path, want string) int {
|
||||
t.Helper()
|
||||
b, _ := os.ReadFile(path)
|
||||
n := 0
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if l == want {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// novox/hq ADR 0198: a module's long-running code is launched by the node's runtime, and the runtime
|
||||
// is its bus — it binds the module's own consumer, hands each event to the bundle, and acknowledges it
|
||||
// only when the bundle has taken it; one the bundle failed or died on is offered again.
|
||||
func TestTheRuntimeHandsAModulesEventsToItsBundleAndAcknowledgesThemOnlyWhenTaken(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
restore := bus.NakDelay
|
||||
bus.NakDelay = 300 * time.Millisecond
|
||||
t.Cleanup(func() { bus.NakDelay = restore })
|
||||
firstPause := launch.RestartFirst
|
||||
launch.RestartFirst = 100 * time.Millisecond
|
||||
t.Cleanup(func() { launch.RestartFirst = firstPause })
|
||||
|
||||
mesh.Issue(t, mt.MembershipOf("watcher", "anchor", false, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "anchor", true, nil))
|
||||
// The controller's consumer for the module, as its own runtime bound it: anchor_watcher on EVENTS.
|
||||
mesh.Consumer(t, "anchor", "watcher", []string{"mesh.mod.alpha.event.>"}, 2*time.Second)
|
||||
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
asker := connect(t, "console", "workstation")
|
||||
dir := t.TempDir()
|
||||
log := filepath.Join(dir, "watch.log")
|
||||
said := &lines{}
|
||||
stop, err := Run(nodeTools, []Served{
|
||||
{Module: "watcher", Entrypoints: []string{mt.Fixture("watcher.serve.mjs")}},
|
||||
{Module: "beta", Entrypoints: []string{mt.Fixture("many-beta.serve.mjs")}},
|
||||
}, map[string]map[string]string{"watcher": {"WATCH_LOG": log}}, said.logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
if !strings.Contains(said.all(), "watcher's events arrive on its consumer anchor_watcher") {
|
||||
t.Fatalf("the module's consumer was not bound:\n%s", said.all())
|
||||
}
|
||||
|
||||
// Handled once, acknowledged once.
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error {
|
||||
if countLines(t, log, "handled 1") != 1 {
|
||||
return errorf("event 1 not handled yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// The handler fails the first time: not acknowledged, offered again, then handled.
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 2, "fail": true})
|
||||
// The bundle dies on the first offer: not acknowledged, the bundle is started again and the
|
||||
// event is offered again once its ack wait has passed.
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 3, "die": true})
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if countLines(t, log, "handled 2") == 1 && countLines(t, log, "handled 3") == 1 {
|
||||
break
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
if countLines(t, log, "handled 2") != 1 || countLines(t, log, "handled 3") != 1 {
|
||||
b, _ := os.ReadFile(log)
|
||||
t.Fatalf("a failed or interrupted event was not offered again and handled once:\n%s\nruntime:\n%s", b, said.all())
|
||||
}
|
||||
if countLines(t, log, "started") < 2 {
|
||||
t.Errorf("the bundle that died was not started again: %s", said.all())
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
ack, notYet := mesh.Pending(t, "anchor", "watcher")
|
||||
if ack != 0 || notYet != 0 {
|
||||
return errorf("still pending: %d unacknowledged, %d undelivered", ack, notYet)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if countLines(t, log, "handled 1") != 1 {
|
||||
t.Errorf("event 1 was handled more than once")
|
||||
}
|
||||
|
||||
// A tool asks another module's tool through the runtime, as the module.
|
||||
got, err := call(t, asker, "watcher.relay@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same(t, got, `{"beta":3}`)
|
||||
}
|
||||
|
||||
// A long-running bundle that exits is started again, without waiting for a call (ADR 0198).
|
||||
func TestALongRunningBundleThatExitsIsStartedAgain(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
firstPause := launch.RestartFirst
|
||||
launch.RestartFirst = 100 * time.Millisecond
|
||||
t.Cleanup(func() { launch.RestartFirst = firstPause })
|
||||
mesh.Issue(t, mt.MembershipOf("flaky", "anchor", false, nil))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
log := filepath.Join(t.TempDir(), "flaky.log")
|
||||
said := &lines{}
|
||||
stop, err := Run(nodeTools, []Served{{Module: "flaky", Entrypoints: []string{mt.Fixture("flaky.serve.mjs")}}},
|
||||
map[string]map[string]string{"flaky": {"FLAKY_LOG": log}}, said.logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mt.Until(t, func() error {
|
||||
if countLines(t, log, "started") < 3 {
|
||||
return errorf("started %d time(s)", countLines(t, log, "started"))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if !strings.Contains(said.all(), "flaky's bundle exited (3)") || !strings.Contains(said.all(), "started again in") {
|
||||
t.Errorf("the restart was not said:\n%s", said.all())
|
||||
}
|
||||
}
|
||||
|
||||
func fmtSprintf(format string, args ...any) string { return fmt.Sprintf(format, args...) }
|
||||
func errorf(format string, args ...any) error { return fmt.Errorf(format, args...) }
|
||||
@@ -0,0 +1,377 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// logBuses is a module, issues, that keeps a log `changes` on a real bus, and a module, other, on the
|
||||
// same runtime that was issued no log — each reached as a bundle reaches it: mesh/log.<verb> with its
|
||||
// JSON, through its module's bus (novox/hq ADR 0297).
|
||||
func logBuses(t *testing.T, maxBytes int64) (issues, other launch.Bus, mesh *mt.Mesh) {
|
||||
t.Helper()
|
||||
mesh = mt.New(t)
|
||||
m := mt.MembershipOf("issues", "anchor", false, nil)
|
||||
m.Logs = []bus.LogIssued{mesh.Log(t, "issues", "changes", maxBytes)}
|
||||
mesh.ModuleStream(t, "issues", "ISSUE_MOVES")
|
||||
m.Streams = []bus.StreamIssued{{Stream: "ISSUE_MOVES", Events: []string{"moved.>"}, Subject: "mesh.stream.issues.event.{event}"}}
|
||||
mesh.Issue(t, m)
|
||||
mesh.Issue(t, mt.MembershipOf("other", "anchor", false, nil))
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("issues")
|
||||
conn.Follow("other")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("issues") == nil || conn.Membership("other") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
return all.forModule("issues"), all.forModule("other"), mesh
|
||||
}
|
||||
|
||||
// logAsk is a bundle's mesh/log.<verb>, with its params exactly as they go over the wire.
|
||||
func logAsk(b launch.Bus, verb, params string) (json.RawMessage, error) {
|
||||
return b.Log(verb, json.RawMessage(params))
|
||||
}
|
||||
|
||||
func appendOf(t *testing.T, b launch.Bus, params string) uint64 {
|
||||
t.Helper()
|
||||
raw, err := logAsk(b, "append", params)
|
||||
if err != nil {
|
||||
t.Fatalf("append %s: %v", params, err)
|
||||
}
|
||||
var r struct {
|
||||
Sequence *uint64 `json:"sequence"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &r); err != nil || r.Sequence == nil {
|
||||
t.Fatalf("append answered %s, not {sequence}", raw)
|
||||
}
|
||||
return *r.Sequence
|
||||
}
|
||||
|
||||
// wirePage is a read's answer with exactly the contract's field names.
|
||||
type wirePage struct {
|
||||
Entries []struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
At string `json:"at"`
|
||||
} `json:"entries"`
|
||||
Next uint64 `json:"next"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
func readOf(t *testing.T, b launch.Bus, params string) wirePage {
|
||||
t.Helper()
|
||||
raw, err := logAsk(b, "read", params)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", params, err)
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
_ = json.Unmarshal(raw, &fields)
|
||||
for _, f := range []string{"entries", "next", "last"} {
|
||||
if _, ok := fields[f]; !ok {
|
||||
t.Fatalf("read answered %s, without %q", raw, f)
|
||||
}
|
||||
}
|
||||
var p wirePage
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func infoOf(t *testing.T, b launch.Bus) map[string]uint64 {
|
||||
t.Helper()
|
||||
raw, err := logAsk(b, "info", `{"log":"changes"}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var info map[string]uint64
|
||||
if err := json.Unmarshal(raw, &info); err != nil {
|
||||
t.Fatalf("info answered %s: %v", raw, err)
|
||||
}
|
||||
for _, f := range []string{"entries", "bytes", "max_bytes", "first", "last"} {
|
||||
if _, ok := info[f]; !ok {
|
||||
t.Fatalf("info answered %s, without %q", raw, f)
|
||||
}
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// Each verb against a real bus: append answers the sequence, read pages from a sequence on, of a key
|
||||
// pattern, with the sequence to continue from (0 at the end) and the log's last, each entry's time in
|
||||
// RFC 3339; last is a key's last entry or null; info says entries, bytes, cap, first and last — and no
|
||||
// consumer is made for any of it.
|
||||
func TestALogAnswersEachVerbOnTheBus(t *testing.T) {
|
||||
b, other, mesh := logBuses(t, 1<<20)
|
||||
before := time.Now().Add(-time.Second)
|
||||
|
||||
var seqs []uint64
|
||||
for i, key := range []string{"12", "13", "12", "14.notes", "12"} {
|
||||
seqs = append(seqs, appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":%q,"value":{"op":"move","n":%d}}`, key, i)))
|
||||
}
|
||||
for i := 1; i < len(seqs); i++ {
|
||||
if seqs[i] != seqs[i-1]+1 {
|
||||
t.Fatalf("sequences %v are not the log's order", seqs)
|
||||
}
|
||||
}
|
||||
|
||||
// Everything, no keys and no limit: the default page holds all five.
|
||||
all := readOf(t, b, `{"log":"changes","from":0}`)
|
||||
if len(all.Entries) != 5 || all.Next != 0 || all.Last != seqs[4] {
|
||||
t.Fatalf("a read of everything: %d entries, next %d, last %d", len(all.Entries), all.Next, all.Last)
|
||||
}
|
||||
for i, e := range all.Entries {
|
||||
at, err := time.Parse(time.RFC3339, e.At)
|
||||
if err != nil || at.Before(before) || at.After(time.Now().Add(time.Second)) {
|
||||
t.Errorf("entry %d's time %q is not RFC 3339 and now: %v", e.Sequence, e.At, err)
|
||||
}
|
||||
if e.Sequence != seqs[i] || string(e.Value) != fmt.Sprintf(`{"op":"move","n":%d}`, i) {
|
||||
t.Errorf("entry %d: %d %s %s", i, e.Sequence, e.Key, e.Value)
|
||||
}
|
||||
}
|
||||
if all.Entries[3].Key != "14.notes" {
|
||||
t.Errorf("a key of two names read back as %q", all.Entries[3].Key)
|
||||
}
|
||||
|
||||
// One key's entries, a page of two at a time.
|
||||
first := readOf(t, b, `{"log":"changes","from":1,"keys":"12","limit":2}`)
|
||||
if len(first.Entries) != 2 || first.Entries[0].Sequence != seqs[0] || first.Entries[1].Sequence != seqs[2] ||
|
||||
first.Next != seqs[2]+1 {
|
||||
t.Fatalf("the first page of key 12: %+v", first)
|
||||
}
|
||||
second := readOf(t, b, fmt.Sprintf(`{"log":"changes","from":%d,"keys":"12","limit":2}`, first.Next))
|
||||
if len(second.Entries) != 1 || second.Entries[0].Sequence != seqs[4] || second.Next != 0 {
|
||||
t.Fatalf("the second page of key 12: %+v", second)
|
||||
}
|
||||
// A pattern: `*` one name, `>` the rest.
|
||||
if p := readOf(t, b, `{"log":"changes","from":1,"keys":"14.*"}`); len(p.Entries) != 1 || p.Entries[0].Key != "14.notes" {
|
||||
t.Fatalf("14.*: %+v", p)
|
||||
}
|
||||
if p := readOf(t, b, `{"log":"changes","from":1,"keys":"14.>"}`); len(p.Entries) != 1 {
|
||||
t.Fatalf("14.>: %+v", p)
|
||||
}
|
||||
// Past the end: nothing, nothing follows.
|
||||
if p := readOf(t, b, fmt.Sprintf(`{"log":"changes","from":%d}`, seqs[4]+1)); len(p.Entries) != 0 || p.Next != 0 || p.Last != seqs[4] {
|
||||
t.Fatalf("a read past the end: %+v", p)
|
||||
}
|
||||
if p := readOf(t, b, `{"log":"changes","from":1,"keys":"99"}`); len(p.Entries) != 0 || p.Next != 0 {
|
||||
t.Fatalf("a key with no entries: %+v", p)
|
||||
}
|
||||
|
||||
raw, err := logAsk(b, "last", `{"log":"changes","key":"12"}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var last map[string]json.RawMessage
|
||||
_ = json.Unmarshal(raw, &last)
|
||||
if string(last["sequence"]) != fmt.Sprint(seqs[4]) || string(last["key"]) != `"12"` ||
|
||||
string(last["value"]) != `{"op":"move","n":4}` || len(last["at"]) == 0 {
|
||||
t.Fatalf("the last entry of 12: %s", raw)
|
||||
}
|
||||
if raw, err := logAsk(b, "last", `{"log":"changes","key":"99"}`); err != nil || string(raw) != "null" {
|
||||
t.Fatalf("the last entry of a key with none: %s %v", raw, err)
|
||||
}
|
||||
|
||||
info := infoOf(t, b)
|
||||
if info["entries"] != 5 || info["first"] != seqs[0] || info["last"] != seqs[4] || info["max_bytes"] != 1<<20 ||
|
||||
info["bytes"] == 0 {
|
||||
t.Fatalf("info: %v", info)
|
||||
}
|
||||
if n := mesh.Consumers(t, "LOG_issues_changes"); n != 0 {
|
||||
t.Fatalf("the runtime made %d consumers of the log; it uses none", n)
|
||||
}
|
||||
|
||||
// A module reads its own stream back as it reads its log (mesh/stream.read): by its local name, an event pattern,
|
||||
// in pages; another module's stream is refused with the reason.
|
||||
for i, ev := range []string{"moved.a.1", "moved.b.1", "moved.a.1"} {
|
||||
if err := b.Publish(json.RawMessage(fmt.Sprintf(`{"key":%q,"body":{"n":%d}}`, ev, i))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
streamOf := func(b launch.Bus, params string) (wireStream, error) {
|
||||
var p wireStream
|
||||
raw, err := b.StreamRead(json.RawMessage(params))
|
||||
if err == nil {
|
||||
err = json.Unmarshal(raw, &p)
|
||||
}
|
||||
return p, err
|
||||
}
|
||||
page, err := streamOf(b, `{"stream":"issue-moves","subject":"moved.a.>","limit":1}`)
|
||||
if err != nil || len(page.Entries) != 1 || page.Entries[0].Event != "moved.a.1" || string(page.Entries[0].Body) != `{"n":0}` || page.Next == 0 || page.Last != 3 {
|
||||
t.Fatalf("the first page of moved.a: %+v, %v", page, err)
|
||||
}
|
||||
page, err = streamOf(b, fmt.Sprintf(`{"stream":"ISSUE_MOVES","subject":"moved.a.>","from":%d}`, page.Next))
|
||||
if err != nil || len(page.Entries) != 1 || string(page.Entries[0].Body) != `{"n":2}` || page.Next != 0 {
|
||||
t.Fatalf("the second page of moved.a: %+v, %v", page, err)
|
||||
}
|
||||
if _, err := streamOf(other, `{"stream":"issue-moves"}`); err == nil || !strings.Contains(err.Error(), "keeps no stream") {
|
||||
t.Fatalf("another module's stream: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// wireStream is a stream read's answer with exactly the contract's field names (mesh-sdk stdio.StreamPage).
|
||||
type wireStream struct {
|
||||
Entries []struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Event string `json:"event"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
At string `json:"at"`
|
||||
} `json:"entries"`
|
||||
Next uint64 `json:"next"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
// A page holds about 4 MiB at most, whatever its limit, and says where to continue.
|
||||
func TestALogPageIsBoundedByItsBytes(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 64<<20)
|
||||
big := `"` + strings.Repeat("x", 250*1024) + `"`
|
||||
for i := 0; i < 20; i++ {
|
||||
appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":"k%d","value":%s}`, i, big))
|
||||
}
|
||||
p := readOf(t, b, `{"log":"changes","from":1,"limit":1000}`)
|
||||
if len(p.Entries) == 0 || len(p.Entries) >= 20 || p.Next != p.Entries[len(p.Entries)-1].Sequence+1 {
|
||||
t.Fatalf("a page of 20 entries of 250 KiB: %d entries, next %d", len(p.Entries), p.Next)
|
||||
}
|
||||
got := len(p.Entries)
|
||||
for p.Next != 0 {
|
||||
p = readOf(t, b, fmt.Sprintf(`{"log":"changes","from":%d,"limit":1000}`, p.Next))
|
||||
got += len(p.Entries)
|
||||
}
|
||||
if got != 20 {
|
||||
t.Fatalf("paging read %d of 20 entries", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An append with `expect` is written only when the key's last entry is the one expected — 0 when it
|
||||
// has none — and is otherwise refused with the error a lost mesh/state.update answers: the same code,
|
||||
// the same error, and nothing written. An append without `expect` is not checked.
|
||||
func TestAnAppendAtAStaleExpectIsRefusedAsALostUpdate(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 1<<20)
|
||||
isLost := func(err error) bool {
|
||||
var coded interface{ ErrorCode() int }
|
||||
return errors.As(err, &coded) && coded.ErrorCode() == launch.CodeChanged && errors.Is(err, bus.ErrStateChanged)
|
||||
}
|
||||
|
||||
first := appendOf(t, b, `{"log":"changes","key":"7","value":{"op":"open"},"expect":0}`)
|
||||
if _, err := logAsk(b, "append", `{"log":"changes","key":"7","value":{"op":"open"},"expect":0}`); !isLost(err) {
|
||||
t.Fatalf("a second open of one key at expect 0: %v", err)
|
||||
}
|
||||
second := appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"move"},"expect":%d}`, first))
|
||||
if _, err := logAsk(b, "append", fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"note"},"expect":%d}`, first)); !isLost(err) {
|
||||
t.Fatalf("an append at a stale expect: %v", err)
|
||||
}
|
||||
if _, err := logAsk(b, "append", fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"note"},"expect":%d}`, second+100)); !isLost(err) {
|
||||
t.Fatalf("an append at an expect not yet written: %v", err)
|
||||
}
|
||||
// Another key's entries do not make this key's expect stale.
|
||||
appendOf(t, b, `{"log":"changes","key":"8","value":{"op":"open"}}`)
|
||||
third := appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"note"},"expect":%d}`, second))
|
||||
// No expect: no check.
|
||||
appendOf(t, b, `{"log":"changes","key":"7","value":{"op":"set"}}`)
|
||||
|
||||
if info := infoOf(t, b); info["entries"] != 5 {
|
||||
t.Fatalf("refused appends wrote: %d entries, not 5", info["entries"])
|
||||
}
|
||||
p := readOf(t, b, `{"log":"changes","from":1,"keys":"7"}`)
|
||||
if len(p.Entries) != 4 || p.Entries[2].Sequence != third {
|
||||
t.Fatalf("key 7 holds %+v", p.Entries)
|
||||
}
|
||||
|
||||
// The refusal is the very error a lost state update is: its code reaches a bundle as CodeChanged.
|
||||
_, err := logAsk(b, "append", `{"log":"changes","key":"7","value":{},"expect":0}`)
|
||||
var coded interface{ ErrorCode() int }
|
||||
if !errors.As(err, &coded) || coded.ErrorCode() != -32010 {
|
||||
t.Fatalf("the refusal carries no lost-update code: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A log is its owner's alone: another module on the same runtime is refused it with the reason, and so
|
||||
// is a log of a name the owner was not issued — before anything is sent.
|
||||
func TestALogNotIssuedIsRefusedWithTheReason(t *testing.T) {
|
||||
issues, other, _ := logBuses(t, 1<<20)
|
||||
for verb, params := range map[string]string{
|
||||
"append": `{"log":"changes","key":"1","value":{}}`,
|
||||
"read": `{"log":"changes","from":1}`,
|
||||
"last": `{"log":"changes","key":"1"}`,
|
||||
"info": `{"log":"changes"}`,
|
||||
} {
|
||||
_, err := logAsk(other, verb, params)
|
||||
if err == nil || !strings.Contains(err.Error(), `other keeps no log called "changes"`) ||
|
||||
!strings.Contains(err.Error(), "issued: none") {
|
||||
t.Errorf("%s of a log not issued: %v", verb, err)
|
||||
}
|
||||
}
|
||||
_, err := logAsk(issues, "append", `{"log":"moves","key":"1","value":{}}`)
|
||||
if err == nil || !strings.Contains(err.Error(), `issues keeps no log called "moves"`) ||
|
||||
!strings.Contains(err.Error(), "issued: changes") {
|
||||
t.Errorf("an append to a log of a name not issued: %v", err)
|
||||
}
|
||||
if info := infoOf(t, issues); info["entries"] != 0 {
|
||||
t.Fatalf("a refused append wrote: %v", info)
|
||||
}
|
||||
}
|
||||
|
||||
// What the bus cannot hold, or the contract does not allow, is refused with the reason and not sent:
|
||||
// a value that is not JSON or over 256 KiB, a key that is not names joined by dots or over 255 bytes, a
|
||||
// key pattern with `>` not last, a limit over 1000.
|
||||
func TestWhatALogCannotHoldIsRefusedWithTheReason(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 1<<20)
|
||||
over := `"` + strings.Repeat("x", 256*1024) + `"`
|
||||
for params, reason := range map[string]string{
|
||||
`{"log":"changes","key":"1","value":nope}`: "names no log",
|
||||
`{"log":"changes","key":"1"}`: "is JSON",
|
||||
`{"log":"changes","key":"1","value":` + over + `}`: "at most 262144 bytes",
|
||||
`{"log":"changes","key":"a b","value":1}`: "is not a log key",
|
||||
`{"log":"changes","key":"a..b","value":1}`: "is not a log key",
|
||||
`{"log":"changes","key":"a.*","value":1}`: "is not a log key",
|
||||
`{"log":"changes","key":"` + strings.Repeat("k", 256) + `","value":1}`: "is not a log key",
|
||||
`{"key":"1","value":1}`: "names no log",
|
||||
} {
|
||||
if _, err := logAsk(b, "append", params); err == nil || !strings.Contains(err.Error(), reason) {
|
||||
t.Errorf("append %.80s: %v, want %q", params, err, reason)
|
||||
}
|
||||
}
|
||||
for params, reason := range map[string]string{
|
||||
`{"log":"changes","from":1,"keys":"a.>.b"}`: "is not a key pattern",
|
||||
`{"log":"changes","from":1,"limit":1001}`: "1 to 1000",
|
||||
`{"log":"changes","from":1,"limit":-1}`: "1 to 1000",
|
||||
} {
|
||||
if _, err := logAsk(b, "read", params); err == nil || !strings.Contains(err.Error(), reason) {
|
||||
t.Errorf("read %s: %v, want %q", params, err, reason)
|
||||
}
|
||||
}
|
||||
if info := infoOf(t, b); info["entries"] != 0 {
|
||||
t.Fatalf("a refused append wrote: %v", info)
|
||||
}
|
||||
// The largest value allowed is taken, also with the header an expect carries.
|
||||
largest := `"` + strings.Repeat("x", 256*1024-2) + `"`
|
||||
appendOf(t, b, `{"log":"changes","key":"1","value":`+largest+`,"expect":0}`)
|
||||
appendOf(t, b, `{"log":"changes","key":"2","value":`+largest+`}`)
|
||||
}
|
||||
|
||||
// A value naming a credential anywhere in it is refused, as mesh/state.put refuses one (novox/hq ADR
|
||||
// 0201 §13): a log keeps what it is given for good. A value that only counts tokens is taken.
|
||||
func TestALogRefusesAValueNamingACredential(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 1<<20)
|
||||
for _, value := range []string{`{"op":"set","api_key":"s"}`, `{"op":"note","with":{"headers":{"Authorization":"Bearer s"}}}`} {
|
||||
_, err := logAsk(b, "append", `{"log":"changes","key":"1","value":`+value+`}`)
|
||||
if err == nil || !strings.Contains(err.Error(), "names a credential") {
|
||||
t.Errorf("append of %s: %v", value, err)
|
||||
}
|
||||
}
|
||||
if info := infoOf(t, b); info["entries"] != 0 {
|
||||
t.Fatalf("a refused append wrote: %v", info)
|
||||
}
|
||||
appendOf(t, b, `{"log":"changes","key":"1","value":{"op":"note","maxTokens":4096}}`)
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0259 §8: a runtime on a module's own account serves that module alone, and the machine's
|
||||
// runtime never serves itself.
|
||||
func TestARuntimeOnAModulesOwnAccountServesThatModuleAlone(t *testing.T) {
|
||||
if got, err := ServedModulesFrom("telegram=/b/telegram/tools/telegram", "telegram"); err != nil || len(got) != 1 {
|
||||
t.Fatalf("a module's own runtime could not serve it: %v", err)
|
||||
}
|
||||
if _, err := ServedModulesFrom("telegram=/b/t,dunst=/b/d", "telegram"); err == nil || !strings.Contains(err.Error(), "serves telegram alone") {
|
||||
t.Errorf("a module's own runtime served another: %v", err)
|
||||
}
|
||||
if _, err := ServedModulesFrom("node-tools=/b/n", "node-tools"); err == nil {
|
||||
t.Error("the machine's runtime served itself")
|
||||
}
|
||||
}
|
||||
|
||||
// No bus word reaches a bundle (security review of 2026-10-08), not even one its module's words name.
|
||||
func TestNoBusWordReachesABundle(t *testing.T) {
|
||||
base := []string{"MESH_BROKER_FILE=/etc/mesh/broker", "MESH_BROKER_URL=nats://x", "MESH_TOOL_ENV={}",
|
||||
"MESH_TOOL_MODULES=a=/b", "MESH_CONSOLE_LISTEN=127.0.0.1:1", "PATH=/usr/bin"}
|
||||
env := strings.Join(BundleEnv(base, map[string]string{"MESH_BROKER_FILE": "/again", "OWN": "1"}, "telegram", "anchor"), "\n")
|
||||
for _, never := range []string{"MESH_BROKER_FILE", "MESH_BROKER_URL", "MESH_TOOL_ENV", "MESH_TOOL_MODULES", "MESH_CONSOLE_LISTEN"} {
|
||||
if strings.Contains(env, never+"=") {
|
||||
t.Errorf("%s reached the bundle", never)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"PATH=/usr/bin", "OWN=1", "MESH_MODULE=telegram", "MESH_NODE=anchor"} {
|
||||
if !strings.Contains(env, want) {
|
||||
t.Errorf("%s did not reach the bundle", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A tool call reaches only a tool's subject: whatever key a caller names, it is never a seat's event, accept
|
||||
// or proof — so no tool call says a choice, a link or a code, or submits an ask, in anybody's name.
|
||||
func TestAToolCallNeverReachesASeatsEventAcceptOrProof(t *testing.T) {
|
||||
for _, key := range []string{"seat:intake.event.choice.telegram", "seat:intake.proof.code.telegram",
|
||||
"seat:operator-channel.accept.ask.mesh-delivery", "intake.event", "seat:operator-channel.event.decided.x@anchor",
|
||||
"telegram.anything", "x"} {
|
||||
subject, err := bus.ToolSubject(key, "node-tools")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
tokens := strings.Split(subject, ".")
|
||||
if len(tokens) < 4 || tokens[3] != "tool" {
|
||||
t.Errorf("%q reaches %s, which is not a tool's subject", key, subject)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// A module's code reads its values at runtime (novox/hq ADR 0349 clause 14 as ADR 0370 changes it, issue 685).
|
||||
//
|
||||
// A bundle asks `mesh/provision.get` and `mesh/provision.watch` with `{kind, key}`; the runtime asks its machine's
|
||||
// provisioner client **as the module whose bundle asks** — the module this bus was made for, from the bundle it
|
||||
// launched, never a name the bundle sent (ADR 0370 §4) — and hands the answer back. A watch hands the current value
|
||||
// first and then each change: the client says which value moved, and the runtime asks again. A secret is refused
|
||||
// here, with the reason, and never asked: the client serves no secret on the tool runner's word (§2), and a module
|
||||
// under an account of its own asks the vault itself, which waits on issue 569.
|
||||
|
||||
// The kinds a bundle may ask.
|
||||
const (
|
||||
provisionConfig = "config"
|
||||
provisionBound = "bound"
|
||||
provisionSecret = "secret"
|
||||
)
|
||||
|
||||
// provisionAsked is what a bundle names. Anything else it sends, a module's name among it, is not read.
|
||||
type provisionAsked struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// ProvisionChange is what a watch hands a bundle: the value, or, when it is no longer answered, why.
|
||||
type ProvisionChange struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
// Refused is why the value is no longer answered: the module taken off the machine, the key no longer declared.
|
||||
Refused string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
func readProvisionAsked(verb string, params json.RawMessage) (provisionAsked, error) {
|
||||
var asked provisionAsked
|
||||
if err := json.Unmarshal(params, &asked); err != nil || asked.Kind == "" || asked.Key == "" {
|
||||
return asked, fmt.Errorf("mesh/provision.%s names no value: {kind, key}", verb)
|
||||
}
|
||||
switch asked.Kind {
|
||||
case provisionConfig, provisionBound:
|
||||
return asked, nil
|
||||
case provisionSecret:
|
||||
return asked, fmt.Errorf("the secret %s is not asked for at runtime while the module's code runs in the "+
|
||||
"tool runner: the vault hands a secret only to a caller the bus proves, and the bus proves only the tool "+
|
||||
"runner here (novox/hq ADR 0370 §2, ADR 0349 §11); the module's secrets reach it through its files, "+
|
||||
"until it runs under a bus account of its own (issue 569)", asked.Key)
|
||||
}
|
||||
return asked, fmt.Errorf("mesh/provision.%s: %q is not a kind of value a module reads at runtime: config or bound",
|
||||
verb, asked.Kind)
|
||||
}
|
||||
|
||||
// Provision answers a bundle's `get` of one of its module's values.
|
||||
func (b *moduleBus) Provision(verb string, params json.RawMessage) (json.RawMessage, error) {
|
||||
if verb != "get" {
|
||||
return nil, fmt.Errorf("the runtime answers no mesh/provision.%s", verb)
|
||||
}
|
||||
asked, err := readProvisionAsked(verb, params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.all.conn.AskProvisioner(b.module, asked.Kind, asked.Key)
|
||||
}
|
||||
|
||||
// ProvisionWatch hands deliver the value now and each change after, until stopped. A value not answered now is
|
||||
// refused now, with the client's reason.
|
||||
func (b *moduleBus) ProvisionWatch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
asked, err := readProvisionAsked("watch", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := b.all.conn.AskProvisioner(b.module, asked.Kind, asked.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w := &provisionWatch{bus: b, asked: asked, deliver: deliver, last: value}
|
||||
if err := w.hand(ProvisionChange{Kind: asked.Kind, Key: asked.Key, Value: value}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stop, err := b.all.provisioned.add(b.all, b.module, w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stop, nil
|
||||
}
|
||||
|
||||
// provisionWatch is one watch a bundle holds: what it asked, and what it was handed last.
|
||||
type provisionWatch struct {
|
||||
bus *moduleBus
|
||||
asked provisionAsked
|
||||
deliver func(json.RawMessage) error
|
||||
mu sync.Mutex // one change at a time, in order
|
||||
last json.RawMessage
|
||||
refused string
|
||||
stopped bool
|
||||
}
|
||||
|
||||
func (w *provisionWatch) hand(change ProvisionChange) error {
|
||||
raw, err := json.Marshal(change)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return w.deliver(raw)
|
||||
}
|
||||
|
||||
// again asks once more after the client said something moved, and hands the bundle what changed.
|
||||
func (w *provisionWatch) again() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped {
|
||||
return
|
||||
}
|
||||
value, err := w.bus.all.conn.AskProvisioner(w.bus.module, w.asked.Kind, w.asked.Key)
|
||||
change := ProvisionChange{Kind: w.asked.Kind, Key: w.asked.Key}
|
||||
var refused *bus.ProvisionRefused
|
||||
switch {
|
||||
case errors.As(err, &refused):
|
||||
if w.refused == refused.Reason {
|
||||
return
|
||||
}
|
||||
change.Refused = refused.Reason
|
||||
case err != nil:
|
||||
// The client not reached: asked again on its next word, which it says whenever it comes up.
|
||||
w.bus.all.logf("[mesh-tools] %s's watch of its %s %s could not ask again: %v", w.bus.module, w.asked.Kind,
|
||||
w.asked.Key, err)
|
||||
return
|
||||
default:
|
||||
if w.refused == "" && bytes.Equal(value, w.last) {
|
||||
return
|
||||
}
|
||||
change.Value = value
|
||||
}
|
||||
if err := w.hand(change); err != nil {
|
||||
w.bus.all.logf("[mesh-tools] %s did not take a change of its %s %s: %v", w.bus.module, w.asked.Kind,
|
||||
w.asked.Key, err)
|
||||
return
|
||||
}
|
||||
w.last, w.refused = value, change.Refused
|
||||
}
|
||||
|
||||
// provisionWatches is every watch on this runtime, by module, and the one subscription that hears the client.
|
||||
type provisionWatches struct {
|
||||
mu sync.Mutex
|
||||
of map[string]map[*provisionWatch]bool
|
||||
stop func()
|
||||
}
|
||||
|
||||
func (p *provisionWatches) add(all *consumers, module string, w *provisionWatch) (func(), error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.stop == nil {
|
||||
stop, err := all.conn.OnProvisionChanged(func(module string, changed bus.ProvisionChanged) {
|
||||
for _, w := range p.told(module, changed) {
|
||||
go w.again()
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the runtime cannot hear its machine's provisioner client: %w", err)
|
||||
}
|
||||
p.stop = stop
|
||||
}
|
||||
if p.of == nil {
|
||||
p.of = map[string]map[*provisionWatch]bool{}
|
||||
}
|
||||
if p.of[module] == nil {
|
||||
p.of[module] = map[*provisionWatch]bool{}
|
||||
}
|
||||
p.of[module][w] = true
|
||||
return func() {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
delete(p.of[module], w)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// told is the watches of module that what moved concerns.
|
||||
func (p *provisionWatches) told(module string, changed bus.ProvisionChanged) []*provisionWatch {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var out []*provisionWatch
|
||||
for w := range p.of[module] {
|
||||
if w.asked.Kind == changed.Kind && (changed.Key == "" || changed.Key == w.asked.Key) {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stopModule stops every watch of one module, whose bundle is loaded again.
|
||||
func (p *provisionWatches) stopModule(module string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for w := range p.of[module] {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
}
|
||||
delete(p.of, module)
|
||||
}
|
||||
|
||||
func (p *provisionWatches) stopAll() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for _, ws := range p.of {
|
||||
for w := range ws {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
}
|
||||
}
|
||||
p.of = nil
|
||||
if p.stop != nil {
|
||||
p.stop()
|
||||
p.stop = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A module's code reads its values at runtime through moduleBus.Provision and ProvisionWatch (novox/hq ADR 0349
|
||||
// §14, ADR 0370, issue 685), against a real bus with a stand-in for the machine's provisioner client: the runtime
|
||||
// asks as the module whose bundle asks, whatever the bundle names; a watch hands the current value, then each change
|
||||
// the client says moved, and a refusal once; a secret is refused without asking.
|
||||
|
||||
// standInClient answers prov.ask.<node> from its values, keyed `<module>.<kind>.<key>`, and keeps every ask.
|
||||
type standInClient struct {
|
||||
mu sync.Mutex
|
||||
values map[string]string
|
||||
asks []string
|
||||
nc *nats.Conn
|
||||
}
|
||||
|
||||
func aStandInClient(t *testing.T, node string, values map[string]string) *standInClient {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
c := &standInClient{values: values, nc: nc}
|
||||
if _, err := nc.Subscribe("prov.ask."+node, func(m *nats.Msg) {
|
||||
var a struct{ Module, Kind, Key string }
|
||||
_ = json.Unmarshal(m.Data, &a)
|
||||
c.mu.Lock()
|
||||
c.asks = append(c.asks, string(m.Data))
|
||||
v, ok := c.values[a.Module+"."+a.Kind+"."+a.Key]
|
||||
c.mu.Unlock()
|
||||
if !ok {
|
||||
_ = m.Respond([]byte(`{"error":"` + a.Module + ` declares no config value ` + a.Key + `"}`))
|
||||
return
|
||||
}
|
||||
_ = m.Respond([]byte(`{"value":` + v + `}`))
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = nc.Flush()
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *standInClient) set(key, value string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if value == "" {
|
||||
delete(c.values, key)
|
||||
return
|
||||
}
|
||||
c.values[key] = value
|
||||
}
|
||||
|
||||
func (c *standInClient) asked() []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return append([]string(nil), c.asks...)
|
||||
}
|
||||
|
||||
func (c *standInClient) say(t *testing.T, node, module, body string) {
|
||||
t.Helper()
|
||||
if err := c.nc.Publish("prov.changed."+node+"."+module, []byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundleAsksForItsValuesAsItsOwnModule(t *testing.T) {
|
||||
client := aStandInClient(t, "anchor", map[string]string{
|
||||
"claude-code.config.managed_settings": `{"spinnerTipsEnabled":false}`,
|
||||
"dunst.config.managed_settings": `"dunst's"`,
|
||||
})
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
b := all.forModule("claude-code").(*moduleBus)
|
||||
|
||||
// Its own value, though the bundle names another module: the runtime names the module from the bundle.
|
||||
got, err := b.Provision("get", json.RawMessage(`{"kind":"config","key":"managed_settings","module":"dunst"}`))
|
||||
if err != nil || string(got) != `{"spinnerTipsEnabled":false}` {
|
||||
t.Fatalf("answered %s, %v", got, err)
|
||||
}
|
||||
if asks := client.asked(); len(asks) != 1 || asks[0] != `{"key":"managed_settings","kind":"config","module":"claude-code"}` {
|
||||
t.Fatalf("the client was asked %v", asks)
|
||||
}
|
||||
// The client's refusal, in its words.
|
||||
if _, err := b.Provision("get", json.RawMessage(`{"kind":"config","key":"nothing"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "declares no config value nothing") {
|
||||
t.Errorf("an undeclared key: %v", err)
|
||||
}
|
||||
// A secret, refused here and never asked; and what is not an ask.
|
||||
before := len(client.asked())
|
||||
for params, want := range map[string]string{
|
||||
`{"kind":"secret","key":"gitea-read"}`: "the vault hands a secret only to a caller the bus proves",
|
||||
`{"kind":"setting","key":"role"}`: "not a kind of value",
|
||||
`{"key":"role"}`: "names no value",
|
||||
} {
|
||||
if _, err := b.Provision("get", json.RawMessage(params)); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%s: %v, want %q", params, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := b.ProvisionWatch(json.RawMessage(`{"kind":"secret","key":"gitea-read"}`), func(json.RawMessage) error {
|
||||
return nil
|
||||
}); err == nil {
|
||||
t.Error("a secret was watched")
|
||||
}
|
||||
if n := len(client.asked()); n != before {
|
||||
t.Errorf("the client was asked for what the runtime refuses: %v", client.asked()[before:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundlesWatchIsHandedTheValueAndEachChange(t *testing.T) {
|
||||
client := aStandInClient(t, "anchor", map[string]string{"claude-code.config.managed_settings": `{"a":1}`})
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
t.Cleanup(all.stopAll)
|
||||
b := all.forModule("claude-code").(*moduleBus)
|
||||
|
||||
handed := make(chan ProvisionChange, 16)
|
||||
stop, err := b.ProvisionWatch(json.RawMessage(`{"kind":"config","key":"managed_settings","module":"dunst"}`),
|
||||
func(raw json.RawMessage) error {
|
||||
var c ProvisionChange
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
t.Errorf("handed %s", raw)
|
||||
}
|
||||
handed <- c
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := func(what string) ProvisionChange {
|
||||
t.Helper()
|
||||
select {
|
||||
case c := <-handed:
|
||||
return c
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("nothing handed: %s", what)
|
||||
}
|
||||
return ProvisionChange{}
|
||||
}
|
||||
nothing := func(what string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case c := <-handed:
|
||||
t.Errorf("%s: handed %+v", what, c)
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
if c := next("the current value"); string(c.Value) != `{"a":1}` || c.Kind != "config" || c.Key != "managed_settings" {
|
||||
t.Fatalf("first handed %+v", c)
|
||||
}
|
||||
|
||||
// A change, said by the client: asked again, handed.
|
||||
client.set("claude-code.config.managed_settings", `{"a":2}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config","key":"managed_settings"}`)
|
||||
if c := next("the change"); string(c.Value) != `{"a":2}` {
|
||||
t.Fatalf("after the change, handed %+v", c)
|
||||
}
|
||||
// Every key of the kind said to have moved, the value the same: nothing handed. Another module's or another
|
||||
// key's word: not even asked.
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
nothing("the same value again")
|
||||
before := len(client.asked())
|
||||
client.say(t, "anchor", "dunst", `{"kind":"config","key":"managed_settings"}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config","key":"role"}`)
|
||||
nothing("another's word")
|
||||
if n := len(client.asked()); n != before {
|
||||
t.Errorf("asked again for another's word: %v", client.asked()[before:])
|
||||
}
|
||||
// No longer answered: handed the refusal, once.
|
||||
client.set("claude-code.config.managed_settings", "")
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
if c := next("the refusal"); c.Refused == "" || len(c.Value) != 0 {
|
||||
t.Fatalf("handed %+v, want the refusal", c)
|
||||
}
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
nothing("the same refusal again")
|
||||
// Answered again, and then stopped: nothing after.
|
||||
client.set("claude-code.config.managed_settings", `{"a":3}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
if c := next("answered again"); string(c.Value) != `{"a":3}` {
|
||||
t.Fatalf("handed %+v", c)
|
||||
}
|
||||
stop()
|
||||
client.set("claude-code.config.managed_settings", `{"a":4}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
nothing("a stopped watch")
|
||||
for _, a := range client.asked() {
|
||||
if !strings.Contains(a, `"module":"claude-code"`) {
|
||||
t.Errorf("asked as another module: %s", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A runtime serves a module's bundle as soon as it is in place, again when it is replaced, and no
|
||||
// more once it is gone, with no restart (issue 1945, note 23: mesh-files' bundle arrived on a machine
|
||||
// a quarter of an hour after its runtime started, and its tools were missing until a hand restart).
|
||||
|
||||
// syncLogs is a logger that keeps the lines, safe to read while the runtime's watcher writes.
|
||||
type syncLogs struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (l *syncLogs) Logf(format string, args ...any) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.lines = append(l.lines, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *syncLogs) Has(fragments ...string) bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
for _, line := range l.lines {
|
||||
all := true
|
||||
for _, f := range fragments {
|
||||
all = all && strings.Contains(line, f)
|
||||
}
|
||||
if all {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (l *syncLogs) All() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return strings.Join(l.lines, "\n")
|
||||
}
|
||||
|
||||
// bundleServing is a tools bundle, MCP over stdio with no SDK, whose one tool is named tool.
|
||||
func bundleServing(tool string) string { return bundleScript(tool, false, false) }
|
||||
|
||||
// bundleScript is a tools bundle whose one tool is named tool. With subscribe it asks the runtime for its
|
||||
// module's events and writes "<tool> subscribed" and "<tool> handled <n>" to $EVENT_LOG; with writeBeside it
|
||||
// writes a file beside itself as it starts, as a bundle keeping a cache or a socket there does.
|
||||
func bundleScript(tool string, subscribe, writeBeside bool) string {
|
||||
py := func(b bool) string {
|
||||
if b {
|
||||
return "True"
|
||||
}
|
||||
return "False"
|
||||
}
|
||||
return `#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
TOOL = "` + tool + `"
|
||||
SUBSCRIBE = ` + py(subscribe) + `
|
||||
if ` + py(writeBeside) + `:
|
||||
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "cache"), "w") as f:
|
||||
f.write(TOOL + " " + str(os.getpid()))
|
||||
def say(m):
|
||||
m["jsonrpc"] = "2.0"; sys.stdout.write(json.dumps(m) + "\n"); sys.stdout.flush()
|
||||
def log(line):
|
||||
with open(os.environ["EVENT_LOG"], "a") as f:
|
||||
f.write(line + "\n")
|
||||
TOOLS = [{"name": TOOL, "description": "answers its name", "inputSchema": {"type": "object", "properties": {}}}]
|
||||
for line in sys.stdin:
|
||||
req = json.loads(line); rid = req.get("id"); m = req.get("method"); p = req.get("params") or {}
|
||||
if m is None:
|
||||
if rid == "subscribe":
|
||||
log(TOOL + " subscribed")
|
||||
continue
|
||||
if m == "initialize":
|
||||
say({"id": rid, "result": {"protocolVersion": "2025-03-26", "capabilities": {"tools": {}}, "serverInfo": {"name": "ledger", "version": "1"}}})
|
||||
elif m == "tools/list":
|
||||
say({"id": rid, "result": {"tools": TOOLS}})
|
||||
if SUBSCRIBE:
|
||||
say({"id": "subscribe", "method": "mesh/subscribe", "params": {}})
|
||||
elif m == "tools/call":
|
||||
say({"id": rid, "result": {"content": [{"type": "text", "text": json.dumps({"tool": TOOL})}]}})
|
||||
elif m == "mesh/event":
|
||||
body = (p.get("envelope") or {}).get("body") or {}
|
||||
log(TOOL + " handled " + str(body.get("n")))
|
||||
say({"id": rid, "result": {}})
|
||||
elif rid is not None:
|
||||
say({"id": rid, "result": {}})
|
||||
`
|
||||
}
|
||||
|
||||
// place puts a bundle serving tool at path the way a delivery does: written beside it, made
|
||||
// executable, and renamed into place, its directories made as needed.
|
||||
func place(t *testing.T, path, tool string) {
|
||||
t.Helper()
|
||||
placeScript(t, path, bundleServing(tool))
|
||||
}
|
||||
|
||||
func placeScript(t *testing.T, path, script string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmp := path + ".new"
|
||||
if err := os.WriteFile(tmp, []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// fastSettle makes the watcher act on a burst of changes quickly in a test.
|
||||
func fastSettle(t *testing.T) {
|
||||
was := Settle
|
||||
Settle = 50 * time.Millisecond
|
||||
t.Cleanup(func() { Settle = was })
|
||||
}
|
||||
|
||||
type reloadBed struct {
|
||||
mesh *mt.Mesh
|
||||
nodeTools *bus.Conn
|
||||
asker *bus.Conn
|
||||
logs *syncLogs
|
||||
entry string
|
||||
}
|
||||
|
||||
func newReloadBed(t *testing.T) *reloadBed {
|
||||
t.Helper()
|
||||
fastSettle(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
return &reloadBed{
|
||||
mesh: mesh,
|
||||
nodeTools: connect(t, "node-tools", "anchor"),
|
||||
asker: connect(t, "console", "workstation"),
|
||||
logs: &syncLogs{},
|
||||
// Below directories that do not exist yet, as on a machine before the module's first delivery.
|
||||
entry: filepath.Join(t.TempDir(), "bundles", "ledger", "tools", "serve.py"),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *reloadBed) run(t *testing.T) {
|
||||
t.Helper()
|
||||
stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
|
||||
func (b *reloadBed) answers(t *testing.T, tool string) {
|
||||
t.Helper()
|
||||
mt.Until(t, func() error {
|
||||
got, err := call(t, b.asker, "ledger."+tool+"@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ledger.%s is not served: %v\n%s", tool, err, b.logs.All())
|
||||
}
|
||||
if !strings.Contains(got, `"tool":"`+tool+`"`) {
|
||||
return fmt.Errorf("ledger.%s answered %s", tool, got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (b *reloadBed) silent(t *testing.T, tool string) {
|
||||
t.Helper()
|
||||
mt.Until(t, func() error {
|
||||
if _, err := call(t, b.asker, "ledger."+tool+"@anchor", map[string]any{}); err == nil {
|
||||
return fmt.Errorf("ledger.%s is still served\n%s", tool, b.logs.All())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func announced(t *testing.T) string {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
msg, err := nc.Request("$SRV.INFO.node-tools.anchor", nil, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var i micro.Info
|
||||
if err := json.Unmarshal(msg.Data, &i); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, e := range i.Endpoints {
|
||||
out = append(out, e.Subject)
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
// 1945/AC19: a bundle placed after the runtime started is served, and announced, with no restart.
|
||||
func TestABundlePlacedAfterStartIsServed(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
b.run(t)
|
||||
if !b.logs.Has("ledger's bundle", "does not exist") {
|
||||
t.Errorf("the missing bundle was not said as missing:\n%s", b.logs.All())
|
||||
}
|
||||
place(t, b.entry, "first")
|
||||
b.answers(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
if a := announced(t); !strings.Contains(a, "mesh.mod.ledger.tool.first.anchor") {
|
||||
return fmt.Errorf("discovery does not announce the placed bundle's tool:\n%s", a)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// 1945/AC20: a bundle replaced while served is loaded again: its new tool is served, its old one no more.
|
||||
func TestAReplacedBundleIsReloaded(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
place(t, b.entry, "second")
|
||||
b.answers(t, "second")
|
||||
b.silent(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
a := announced(t)
|
||||
if !strings.Contains(a, "mesh.mod.ledger.tool.second.anchor") || strings.Contains(a, "mesh.mod.ledger.tool.first.anchor") {
|
||||
return fmt.Errorf("discovery does not follow the replaced bundle:\n%s", a)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// The bundle's whole directory swapped for a new one, as an archive unpacked beside it and renamed in.
|
||||
dir := filepath.Dir(b.entry)
|
||||
fresh := dir + ".unpacked"
|
||||
place(t, filepath.Join(fresh, filepath.Base(b.entry)), "third")
|
||||
if err := os.Rename(dir, dir+".old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(fresh, dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.answers(t, "third")
|
||||
b.silent(t, "second")
|
||||
|
||||
// And changed again inside the directory swapped in: the watch follows the new directory.
|
||||
place(t, b.entry, "fourth")
|
||||
b.answers(t, "fourth")
|
||||
}
|
||||
|
||||
// A replaced bundle's old tool stays gone when the module's membership is issued again: a stopped tool
|
||||
// leaves no membership handler behind to bind it once more, and a call on it starts no unmanaged second
|
||||
// copy of the bundle (issue 1945, review round 2).
|
||||
func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
place(t, b.entry, "second")
|
||||
b.answers(t, "second")
|
||||
b.silent(t, "first")
|
||||
|
||||
b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
b.answers(t, "second")
|
||||
// Long enough for the reissued membership to have been heard, before the old tool is asked.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil {
|
||||
t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All())
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: a bundle removed while served is no longer served, and its module says why.
|
||||
func TestARemovedBundleIsNoLongerServed(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
if err := os.RemoveAll(filepath.Dir(b.entry)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.silent(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
got, err := call(t, b.asker, "ledger.tools@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var answer ToolsAnswer
|
||||
if err := json.Unmarshal([]byte(got), &answer); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(answer.Tools) != 0 || !strings.Contains(answer.Failed, "does not exist") {
|
||||
return fmt.Errorf("ledger's tools after its bundle was removed: %s", got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if a := announced(t); strings.Contains(a, "mesh.mod.ledger.tool.first") {
|
||||
t.Errorf("discovery still announces the removed bundle's tool:\n%s", a)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC22: a bundle that cannot be started is said with its real reason: a missing file does not
|
||||
// exist, and a file without an execute bit is not executable with its mode.
|
||||
func TestABundleThatCannotStartSaysTheRealReason(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "nowhere", "serve.py")
|
||||
if why := notStartable(missing); !strings.Contains(why, missing+" does not exist") || strings.Contains(why, "not executable") {
|
||||
t.Errorf("a missing bundle said: %q", why)
|
||||
}
|
||||
plain := filepath.Join(dir, "serve.py")
|
||||
if err := os.WriteFile(plain, []byte(bundleServing("first")), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := notStartable(plain); !strings.Contains(why, plain+" is not executable (mode -rw-r--r--)") {
|
||||
t.Errorf("a bundle without an execute bit said: %q", why)
|
||||
}
|
||||
if why := notStartable(dir); !strings.Contains(why, "is a directory") {
|
||||
t.Errorf("a directory said: %q", why)
|
||||
}
|
||||
if err := os.Chmod(plain, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := notStartable(plain); why != "" {
|
||||
t.Errorf("a startable bundle said: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC23: an event a module's consumer hears while the module is loaded again is offered again, never
|
||||
// a panic of the runtime, and the bundle loaded in its place is handed the next offer.
|
||||
func TestAnEventHeardDuringAReloadIsOfferedAgainNotAPanic(t *testing.T) {
|
||||
c := &consumers{logf: func(string, ...any) {}, of: map[string]*moduleEvents{}}
|
||||
var got []string
|
||||
before := func(json.RawMessage) error { got = append(got, "before"); return nil }
|
||||
old := &moduleEvents{delivers: []*func(json.RawMessage) error{&before}}
|
||||
c.of["ledger"] = old
|
||||
c.dropModule("ledger")
|
||||
env := bus.Envelope{Key: "alpha.happened", Body: json.RawMessage(`{"n":1}`)}
|
||||
hand := func(m *moduleEvents) (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("an event heard during a reload panicked the runtime: %v", r)
|
||||
}
|
||||
}()
|
||||
return c.handOut("ledger", m, env)
|
||||
}
|
||||
if err := hand(old); err == nil {
|
||||
t.Error("an event heard while the module is loaded again was taken, not offered again")
|
||||
}
|
||||
after := func(json.RawMessage) error { got = append(got, "after"); return nil }
|
||||
fresh := &moduleEvents{delivers: []*func(json.RawMessage) error{&after}}
|
||||
c.of["ledger"] = fresh
|
||||
if err := hand(fresh); err != nil {
|
||||
t.Errorf("the bundle loaded again was not handed the event: %v", err)
|
||||
}
|
||||
if err := hand(old); err == nil {
|
||||
t.Error("the stopped consumer's handler handed an event on after the reload")
|
||||
}
|
||||
if strings.Join(got, ",") != "after" {
|
||||
t.Errorf("handed to %v, want the bundle loaded again alone", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC23, whole: a module's events reach its bundle before a reload, around it, and after it.
|
||||
func TestAModulesEventsReachItsBundleLoadedAgain(t *testing.T) {
|
||||
fastSettle(t)
|
||||
restore := bus.NakDelay
|
||||
bus.NakDelay = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.NakDelay = restore })
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
mesh.Consumer(t, "anchor", "ledger", []string{"mesh.mod.alpha.event.>"}, 2*time.Second)
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
logs := &syncLogs{}
|
||||
dir := t.TempDir()
|
||||
log := filepath.Join(dir, "events.log")
|
||||
entry := filepath.Join(dir, "bundles", "ledger", "tools", "serve.py")
|
||||
has := func(line string) error {
|
||||
b, _ := os.ReadFile(log)
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if l == line {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("no %q yet in:\n%s\nruntime:\n%s", line, b, logs.All())
|
||||
}
|
||||
placeScript(t, entry, bundleScript("first", true, false))
|
||||
stop, err := Run(nodeTools, []Served{{"ledger", []string{entry}}}, map[string]map[string]string{"ledger": {"EVENT_LOG": log}}, logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mt.Until(t, func() error { return has("first subscribed") })
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error { return has("first handled 1") })
|
||||
|
||||
placeScript(t, entry, bundleScript("second", true, false))
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 2})
|
||||
mt.Until(t, func() error {
|
||||
if has("first handled 2") != nil && has("second handled 2") != nil {
|
||||
return has("second handled 2")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
mt.Until(t, func() error { return has("second subscribed") })
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 3})
|
||||
mt.Until(t, func() error { return has("second handled 3") })
|
||||
}
|
||||
|
||||
// 1945/AC24: a bundle that writes a file beside itself as it starts is loaded once at start and once
|
||||
// when it is replaced, never again by its own writing.
|
||||
func TestABundleWritingBesideItselfIsNotLoadedAgainByIt(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
t.Setenv("EVENT_LOG", filepath.Join(t.TempDir(), "unused.log"))
|
||||
placeScript(t, b.entry, bundleScript("first", false, true))
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
reloads := func() int {
|
||||
return strings.Count(b.logs.All(), "loading it again")
|
||||
}
|
||||
time.Sleep(20 * Settle)
|
||||
if n := reloads(); n != 0 {
|
||||
t.Fatalf("the bundle was loaded again %d time(s) by its own writing:\n%s", n, b.logs.All())
|
||||
}
|
||||
placeScript(t, b.entry, bundleScript("second", false, true))
|
||||
b.answers(t, "second")
|
||||
time.Sleep(20 * Settle)
|
||||
if n := reloads(); n != 1 {
|
||||
t.Errorf("replaced once, loaded again %d time(s):\n%s", n, b.logs.All())
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,272 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
func connect(t *testing.T, module, node string) *bus.Conn {
|
||||
t.Helper()
|
||||
c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: module, Node: node})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Logf = func(string, ...any) {}
|
||||
t.Cleanup(c.Close)
|
||||
return c
|
||||
}
|
||||
|
||||
func call(t *testing.T, asker *bus.Conn, key string, args any) (string, error) {
|
||||
t.Helper()
|
||||
got, err := asker.Ask(key, args, "")
|
||||
return string(got.Result), err
|
||||
}
|
||||
|
||||
func same(t *testing.T, got, want string) {
|
||||
t.Helper()
|
||||
var a, b any
|
||||
if json.Unmarshal([]byte(got), &a) != nil || json.Unmarshal([]byte(want), &b) != nil {
|
||||
t.Fatalf("not JSON: got %s want %s", got, want)
|
||||
}
|
||||
ga, _ := json.Marshal(a)
|
||||
gb, _ := json.Marshal(b)
|
||||
if string(ga) != string(gb) {
|
||||
t.Errorf("got %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The node's runtime serves five modules' bundles on one credential — two TypeScript, one broken,
|
||||
// one Python, one written against the protocol — and follows a re-issued membership (ADR 0175, 0193).
|
||||
func TestTheNodesRuntimeServesFiveModulesAndFollowsAReissuedMembership(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", true, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "anchor", false, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
mesh.Issue(t, mt.MembershipOf("gamma", "anchor", false, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("delta", "anchor", false, map[string][]string{"node-lamp": {"on"}}))
|
||||
mesh.Issue(t, mt.MembershipOf("epsilon", "anchor", false, nil))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
asker := connect(t, "console", "workstation")
|
||||
logs := &mt.Logs{}
|
||||
t.Setenv("MESH_OPERATOR_ACCOUNT", "somebody")
|
||||
t.Setenv("MESH_OPERATOR_HOME", "/home/somebody")
|
||||
stop, err := Run(nodeTools, []Served{
|
||||
{"alpha", []string{mt.Fixture("many-alpha.serve.mjs")}},
|
||||
{"beta", []string{mt.Fixture("many-beta.serve.mjs")}},
|
||||
{"gamma", []string{mt.Fixture("many-broken.serve.mjs")}},
|
||||
{"delta", []string{mt.Fixture("many-delta.py")}},
|
||||
{"epsilon", []string{mt.Fixture("many-epsilon.mjs")}},
|
||||
}, nil, logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
if !logs.Has("the operator's account here is somebody (home /home/somebody)") {
|
||||
t.Errorf("the operator was not said:\n%s", logs.All())
|
||||
}
|
||||
if !logs.Has("gamma's bundle", "many-broken.serve.mjs failed to load: gamma's bundle exited (1): Error: gamma's bundle cannot find its client; its tools are not served here") {
|
||||
t.Errorf("the broken bundle was not named with its own words:\n%s", logs.All())
|
||||
}
|
||||
if !logs.Has("serving 8 tool(s) for 5 module(s): alpha.one, alpha.two, beta.three, beta.four, beta.five, delta.greet, delta.die, epsilon.seven; not serving gamma") {
|
||||
t.Errorf("not serving what it should:\n%s", logs.All())
|
||||
}
|
||||
|
||||
for key, want := range map[string]string{
|
||||
"alpha.one": `{"alpha":1}`, "alpha.one@anchor": `{"alpha":1}`, "beta.three@anchor": `{"beta":3}`,
|
||||
"beta.four@anchor": `{"beta":4}`, "beta.five@anchor": `{"beta":5}`,
|
||||
"seat:node-shelf.list@anchor": `{"shelf":["a","b"]}`, "seat:node-shelf.clear@anchor": `{"cleared":true}`,
|
||||
"seat:node-lamp.on@anchor": `{"on":true,"language":"python"}`, "epsilon.seven@anchor": `{"epsilon":7,"via":"stdio"}`,
|
||||
} {
|
||||
got, err := call(t, asker, key, map[string]any{})
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", key, err)
|
||||
continue
|
||||
}
|
||||
same(t, got, want)
|
||||
}
|
||||
if _, err := call(t, asker, "beta.three", map[string]any{}); err == nil || !strings.Contains(err.Error(), "no responders") {
|
||||
t.Errorf("beta was not issued the plain subject, and answered on it: %v", err)
|
||||
}
|
||||
got, err := call(t, asker, "delta.greet@anchor", map[string]any{"who": "mesh"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same(t, got, `{"greeting":"hello mesh","language":"python"}`)
|
||||
|
||||
// A launched tool that emits does so as its module, through the runtime.
|
||||
landed := mesh.NextEvent(t, "mesh.mod.*.event.>")
|
||||
got, err = call(t, asker, "alpha.two", map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same(t, got, `{"alpha":2}`)
|
||||
select {
|
||||
case subject := <-landed:
|
||||
if subject != "mesh.mod.alpha.event.happened" {
|
||||
t.Errorf("the event landed on %s", subject)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("the event never landed")
|
||||
}
|
||||
|
||||
// A bundle that dies mid-call is said, and started again on its next call.
|
||||
if _, err := call(t, asker, "delta.die@anchor", map[string]any{}); err == nil {
|
||||
t.Error("a bundle that died answered")
|
||||
}
|
||||
got, err = call(t, asker, "delta.greet@anchor", map[string]any{"who": "again"})
|
||||
if err != nil {
|
||||
t.Fatalf("not started again: %v", err)
|
||||
}
|
||||
same(t, got, `{"greeting":"hello again","language":"python"}`)
|
||||
|
||||
// `tools` answers for each, and why gamma serves nothing.
|
||||
gamma, err := call(t, asker, "gamma.tools@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same(t, gamma, `{"module":"gamma","tools":[],"failed":"gamma's bundle exited (1): Error: gamma's bundle cannot find its client"}`)
|
||||
beta, _ := call(t, asker, "beta.tools@anchor", map[string]any{})
|
||||
var answer ToolsAnswer
|
||||
_ = json.Unmarshal([]byte(beta), &answer)
|
||||
if len(answer.Tools) != 3 || answer.Tools[0].Name != "three" || strings.Join(answer.Tools[0].Subjects, ",") != "mesh.mod.beta.tool.three.anchor" {
|
||||
t.Errorf("beta's tools answer: %s", beta)
|
||||
}
|
||||
|
||||
// Re-issued mid-run, now answering for the module anywhere: served without a restart.
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "anchor", true, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
mt.Until(t, func() error { _, err := call(t, asker, "beta.three", map[string]any{}); return err })
|
||||
got, err = call(t, asker, "seat:node-shelf.list@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same(t, got, `{"shelf":["a","b"]}`)
|
||||
}
|
||||
|
||||
// Each bundle is given its own environment and none of another's (ADR 0192), and the composed
|
||||
// environments are not left in the runtime's.
|
||||
func TestEachBundleIsGivenItsOwnEnvironment(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
for _, m := range []string{"gamma", "delta", "zeta"} {
|
||||
mesh.Issue(t, mt.MembershipOf(m, "anchor", false, nil))
|
||||
}
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
asker := connect(t, "console", "workstation")
|
||||
t.Setenv("MESH_OPERATOR_ACCOUNT", "somebody")
|
||||
t.Setenv(ToolEnv, `{"gamma":{"GAMMA_CONFIG_FILE":"/var/lib/mesh/gamma/config.json"},"delta":{"DELTA_TOKEN_FILE":"/var/lib/mesh/delta/token"},"zeta":{"ZETA_URL":"http://127.0.0.1:3000"}}`)
|
||||
envs, err := TakeToolEnvs()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, left := os.LookupEnv(ToolEnv); left {
|
||||
t.Error("the composed environments were left in the runtime's")
|
||||
}
|
||||
logs := &mt.Logs{}
|
||||
stop, err := Run(nodeTools, []Served{
|
||||
{"gamma", []string{mt.Fixture("env-gamma.serve.mjs")}},
|
||||
{"delta", []string{mt.Fixture("env-delta.serve.mjs")}},
|
||||
{"zeta", []string{mt.Fixture("env-zeta.mjs")}},
|
||||
}, envs, logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
for key, want := range map[string]string{
|
||||
"gamma.given@anchor": `{"mine":"/var/lib/mesh/gamma/config.json","theirs":null,"runtime":"somebody","composed":null}`,
|
||||
"delta.given@anchor": `{"mine":"/var/lib/mesh/delta/token","theirs":null}`,
|
||||
"zeta.given@anchor": `{"mine":"http://127.0.0.1:3000","theirs":null,"composed":null}`,
|
||||
} {
|
||||
got, err := call(t, asker, key, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v\n%s", key, err, logs.All())
|
||||
}
|
||||
same(t, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// An entrypoint that is not executable is refused by name, and the others serve (ADR 0193).
|
||||
func TestAnEntrypointThatIsNotExecutableIsRefused(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", false, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("plain", "anchor", false, nil))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
asker := connect(t, "console", "workstation")
|
||||
logs := &mt.Logs{}
|
||||
stop, err := Run(nodeTools, []Served{
|
||||
{"alpha", []string{mt.Fixture("many-alpha.serve.mjs")}},
|
||||
{"plain", []string{mt.Fixture("many-alpha.mjs")}},
|
||||
}, nil, logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
if !logs.Has("plain's bundle", "many-alpha.mjs failed to load:", "is not executable (mode -", "); a bundle the runtime serves is started, never imported") {
|
||||
t.Errorf("the non-executable entrypoint was not refused by name:\n%s", logs.All())
|
||||
}
|
||||
got, err := call(t, asker, "alpha.one@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same(t, got, `{"alpha":1}`)
|
||||
}
|
||||
|
||||
// A launched bundle is told the module it serves, so its seat's verbs stay the seat's (ADR 0193).
|
||||
func TestALaunchedBundleRegisteringItsSeatFirstServesTheSeat(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("theta", "anchor", false, map[string][]string{"node-shelf": {"list"}}))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
asker := connect(t, "console", "workstation")
|
||||
stop, err := Run(nodeTools, []Served{{"theta", []string{mt.Fixture("served-seat-first.mjs")}}},
|
||||
map[string]map[string]string{"theta": {"THETA_WORD": "given"}}, (&mt.Logs{}).Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
got, err := call(t, asker, "theta.own@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same(t, got, `{"theta":"given"}`)
|
||||
got, err = call(t, asker, "seat:node-shelf.list@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same(t, got, `{"shelf":["x"]}`)
|
||||
}
|
||||
|
||||
func TestToolModulesNamesOtherModulesOnly(t *testing.T) {
|
||||
got, err := ServedModulesFrom(" alpha=/a/tools/index.serve.mjs, beta=/b/one, beta=/b/two ", "node-tools")
|
||||
if err != nil || len(got) != 2 || got[1].Module != "beta" || len(got[1].Entrypoints) != 2 {
|
||||
t.Fatalf("%v %v", got, err)
|
||||
}
|
||||
for _, bad := range []string{"/mine/index.js", "node-tools=/own.js", "=/x"} {
|
||||
if _, err := ServedModulesFrom(bad, "node-tools"); err == nil {
|
||||
t.Errorf("%q was accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A line about an event a bundle did not take says who failed: the handler in its own words, or the
|
||||
// runtime not reaching it (hq issue 276).
|
||||
func TestAnUntakenEventSaysWhoFailed(t *testing.T) {
|
||||
handler := whyNotTaken(&launch.Refused{Method: "mesh/event", Message: "Unexpected end of JSON input"})
|
||||
if !strings.Contains(handler, `its handler answered an error, in its own words: "Unexpected end of JSON input"`) {
|
||||
t.Errorf("a handler's error is not said as the handler's: %s", handler)
|
||||
}
|
||||
if got := whyNotTaken(errors.New("plex's bundle did not answer mesh/event in 120s")); got != "plex's bundle did not answer mesh/event in 120s" {
|
||||
t.Errorf("the runtime's own failure is not said as it is: %s", got)
|
||||
}
|
||||
if got := eventRef(bus.Envelope{Headers: map[string]string{"x-event-id": "e1"}}); got != " (event e1)" {
|
||||
t.Errorf("the event is not named by its id: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A bundle's seat traffic through moduleBus.Seat (novox/hq ADR 0259 §3), against a real bus: publish under
|
||||
// its own name with the id prefixed and a duplicate said; the record read without naming the bucket; a
|
||||
// worker and a proof subject bound once however often a restarted child asks; and nothing beyond what the
|
||||
// membership lists.
|
||||
func TestABundlesSeatTrafficGoesThroughItsModulesBus(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, _ := nc.JetStream()
|
||||
if _, err := js.AddStream(&nats.StreamConfig{Name: "SEAT_OPERATOR_CHANNEL", Retention: nats.WorkQueuePolicy,
|
||||
Subjects: []string{"mesh.seat.operator-channel.accept.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.AddConsumer("SEAT_OPERATOR_CHANNEL", &nats.ConsumerConfig{Durable: "SEAT_OPERATOR_CHANNEL_worker",
|
||||
AckPolicy: nats.AckExplicitPolicy, FilterSubject: "mesh.seat.operator-channel.accept.>"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "messenger_asks"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = kv.Put("mesh-delivery.d-1", []byte(`{"state":"open"}`))
|
||||
asker := mt.MembershipOf("mesh-delivery", "anchor", false, nil)
|
||||
asker.SeatTraffic = &bus.SeatTraffic{Publish: []string{"mesh.seat.operator-channel.accept.ask.mesh-delivery"},
|
||||
Records: []string{"$JS.API.DIRECT.GET.KV_messenger_asks.$KV.messenger_asks.mesh-delivery.>"}}
|
||||
router := mt.MembershipOf("messenger", "anchor", false, nil)
|
||||
router.SeatTraffic = &bus.SeatTraffic{Answers: []string{"mesh.seat.intake.proof.code.*"},
|
||||
Workers: []bus.SeatWorker{{Stream: "SEAT_OPERATOR_CHANNEL", Consumer: "SEAT_OPERATOR_CHANNEL_worker",
|
||||
Filter: "mesh.seat.operator-channel.accept.>"}}}
|
||||
mesh.Issue(t, asker)
|
||||
mesh.Issue(t, router)
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("mesh-delivery")
|
||||
conn.Follow("messenger")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("mesh-delivery") == nil || conn.Membership("messenger") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
asking := all.forModule("mesh-delivery").(*moduleBus)
|
||||
routing := all.forModule("messenger").(*moduleBus)
|
||||
|
||||
publish := json.RawMessage(`{"subject":"mesh.seat.operator-channel.accept.ask.mesh-delivery","body":{"id":"d-1"},"id":"d-1"}`)
|
||||
first, err := asking.Seat("publish", publish, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, _ := asking.Seat("publish", publish, nil)
|
||||
if !strings.Contains(string(first), `"duplicate":false`) || !strings.Contains(string(again), `"duplicate":true`) {
|
||||
t.Errorf("the same id twice: %s, then %s", first, again)
|
||||
}
|
||||
if _, err := asking.Seat("publish", json.RawMessage(`{"subject":"mesh.seat.operator-channel.accept.ask.mesh-controller","body":{}}`), nil); err == nil {
|
||||
t.Error("a module submitted under another's name")
|
||||
}
|
||||
got, err := asking.Seat("record", json.RawMessage(`{"key":"d-1"}`), nil)
|
||||
if err != nil || !strings.Contains(string(got), `"state":"open"`) {
|
||||
t.Errorf("the record read without its bucket: %s %v", got, err)
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var taken []string
|
||||
handOn := func(method string, params any) (json.RawMessage, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
raw, _ := json.Marshal(params)
|
||||
taken = append(taken, method+" "+string(raw))
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
for i := 0; i < 2; i++ { // a child that started again asks again
|
||||
if _, err := routing.Seat("take", json.RawMessage(`{"worker":"SEAT_OPERATOR_CHANNEL_worker"}`), handOn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
mu.Lock()
|
||||
n := len(taken)
|
||||
mu.Unlock()
|
||||
if n >= 1 || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
mu.Lock()
|
||||
if len(taken) != 1 || !strings.Contains(taken[0], "mesh/work") || !strings.Contains(taken[0], `"x-event-id":"mesh-delivery.d-1"`) {
|
||||
t.Errorf("taken %v", taken)
|
||||
}
|
||||
mu.Unlock()
|
||||
if _, err := asking.Seat("take", json.RawMessage(`{"worker":"SEAT_OPERATOR_CHANNEL_worker"}`), handOn); err == nil {
|
||||
t.Error("an asker took the router's work")
|
||||
}
|
||||
if _, err := asking.Seat("answer", json.RawMessage(`{"subject":"mesh.seat.intake.proof.code.*"}`), handOn); err == nil {
|
||||
t.Error("an asker answered proofs")
|
||||
}
|
||||
if _, err := asking.Seat("anything", json.RawMessage(`{}`), nil); err == nil {
|
||||
t.Error("an unknown verb was answered")
|
||||
}
|
||||
}
|
||||
|
||||
// A lost compare-and-set is answered with its code, so the SDK knows it without reading words.
|
||||
func TestALostCompareAndSetCarriesItsCode(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Bucket(t, "messenger_asks")
|
||||
m := mt.MembershipOf("messenger", "anchor", false, nil)
|
||||
m.State = []bus.StateIssued{{Name: "asks", Bucket: "messenger_asks", Writes: true}}
|
||||
mesh.Issue(t, m)
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("messenger")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("messenger") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
b := (&consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}).forModule("messenger")
|
||||
if _, err := b.State("create", json.RawMessage(`{"state":"asks","key":"a","value":{}}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := b.State("create", json.RawMessage(`{"state":"asks","key":"a","value":{}}`))
|
||||
var coded interface{ ErrorCode() int }
|
||||
if !errors.As(err, &coded) || coded.ErrorCode() != -32010 || !errors.Is(err, bus.ErrStateChanged) {
|
||||
t.Errorf("a second create: %v", err)
|
||||
}
|
||||
if _, err := b.State("update", json.RawMessage(`{"state":"asks","key":"a","value":{},"revision":0}`)); err == nil {
|
||||
t.Error("an update of revision zero was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A11: a bundle pulls one message from its own worker and settles it through its module's bus; another module may
|
||||
// neither pull that worker nor settle what it did not pull.
|
||||
func TestABundlePullsAndSettlesThroughItsModulesBus(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, _ := nc.JetStream()
|
||||
if _, err := js.AddStream(&nats.StreamConfig{Name: "SEAT_NODE_BUILD_GO", Retention: nats.WorkQueuePolicy,
|
||||
Subjects: []string{"mesh.seat.node-build-go.accept.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.AddConsumer("SEAT_NODE_BUILD_GO", &nats.ConsumerConfig{Durable: "SEAT_NODE_BUILD_GO_DELIVERY_worker",
|
||||
AckPolicy: nats.AckExplicitPolicy, FilterSubject: "mesh.seat.node-build-go.accept.delivery"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Publish("mesh.seat.node-build-go.accept.delivery", []byte(`{"id":"ask1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
holder := mt.MembershipOf("mesh-build-go", "anchor", false, nil)
|
||||
holder.SeatTraffic = &bus.SeatTraffic{Workers: []bus.SeatWorker{{Stream: "SEAT_NODE_BUILD_GO",
|
||||
Consumer: "SEAT_NODE_BUILD_GO_DELIVERY_worker", Filter: "mesh.seat.node-build-go.accept.delivery"}}}
|
||||
other := mt.MembershipOf("mesh-build-node", "anchor", false, nil)
|
||||
other.SeatTraffic = &bus.SeatTraffic{}
|
||||
mesh.Issue(t, holder)
|
||||
mesh.Issue(t, other)
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("mesh-build-go")
|
||||
conn.Follow("mesh-build-node")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("mesh-build-go") == nil || conn.Membership("mesh-build-node") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
building := all.forModule("mesh-build-go").(*moduleBus)
|
||||
elsewhere := all.forModule("mesh-build-node").(*moduleBus)
|
||||
if _, err := elsewhere.Seat("pull", json.RawMessage(`{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker"}`), nil); err == nil {
|
||||
t.Fatal("another module pulled the Go build seat's worker")
|
||||
}
|
||||
raw, err := building.Seat("pull", json.RawMessage(`{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker","wait_ms":1000}`), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var p bus.Pulled
|
||||
if err := json.Unmarshal(raw, &p); err != nil || string(p.Body) != `{"id":"ask1"}` || p.ID == "" {
|
||||
t.Fatalf("pulled %s (%v)", raw, err)
|
||||
}
|
||||
if _, err := elsewhere.Seat("settle", json.RawMessage(`{"id":"`+p.ID+`","fate":"ack"}`), nil); err == nil {
|
||||
t.Fatal("another module settled the pulled ask")
|
||||
}
|
||||
if _, err := building.Seat("settle", json.RawMessage(`{"id":"`+p.ID+`","fate":"ack"}`), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if none, err := building.Seat("pull", json.RawMessage(`{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker"}`), nil); err != nil || string(none) != "null" {
|
||||
t.Fatalf("an empty worker answered %s, %v", none, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// counterBus is a module, issues, that keeps a state numbers in a bucket of a real bus, reached as a bundle
|
||||
// reaches it: mesh/state.<verb> with its JSON, through the module's bus.
|
||||
func counterBus(t *testing.T) launch.Bus {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
mesh.Bucket(t, "issues_numbers")
|
||||
m := mt.MembershipOf("issues", "anchor", false, nil)
|
||||
m.State = []bus.StateIssued{{Name: "numbers", Bucket: "issues_numbers", Writes: true}}
|
||||
mesh.Issue(t, m)
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("issues")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("issues") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return (&consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}).forModule("issues")
|
||||
}
|
||||
|
||||
func ask(b launch.Bus, verb string, params map[string]any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(params)
|
||||
return b.State(verb, raw)
|
||||
}
|
||||
|
||||
// increment is what a bundle's counter does over the wire: get, then create (no value) or update at the
|
||||
// revision read; a lost write — told by its code alone — is read again and tried again.
|
||||
func increment(b launch.Bus, key string, attempts int) (n uint64, lost int, err error) {
|
||||
for i := 0; i < attempts; i++ {
|
||||
raw, err := ask(b, "get", map[string]any{"state": "numbers", "key": key})
|
||||
if err != nil {
|
||||
return 0, lost, err
|
||||
}
|
||||
var e *bus.StateEntry
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
return 0, lost, err
|
||||
}
|
||||
if e == nil {
|
||||
n = 1
|
||||
_, err = ask(b, "create", map[string]any{"state": "numbers", "key": key, "value": n})
|
||||
} else {
|
||||
if err := json.Unmarshal(e.Value, &n); err != nil {
|
||||
return 0, lost, err
|
||||
}
|
||||
n++
|
||||
_, err = ask(b, "update", map[string]any{"state": "numbers", "key": key, "value": n, "revision": e.Revision})
|
||||
}
|
||||
var coded interface{ ErrorCode() int }
|
||||
switch {
|
||||
case err == nil:
|
||||
return n, lost, nil
|
||||
case errors.As(err, &coded) && coded.ErrorCode() == launch.CodeChanged:
|
||||
lost++
|
||||
default:
|
||||
return 0, lost, err
|
||||
}
|
||||
}
|
||||
return 0, lost, fmt.Errorf("%s lost %d times", key, attempts)
|
||||
}
|
||||
|
||||
// Two writers incrementing one counter a thousand times each, against a real bus, end at two thousand:
|
||||
// every number handed out once and none lost, because the runtime writes only at the revision read
|
||||
// (novox/hq research 042: an issue's number is handed out once).
|
||||
func TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate(t *testing.T) {
|
||||
b := counterBus(t)
|
||||
const each = 1000
|
||||
var wg sync.WaitGroup
|
||||
numbers := make([][]uint64, 2)
|
||||
errs := make([]error, 2)
|
||||
lost := make([]int, 2)
|
||||
for w := range numbers {
|
||||
wg.Add(1)
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < each; i++ {
|
||||
n, l, err := increment(b, "issue", 1000)
|
||||
lost[w] += l
|
||||
if err != nil {
|
||||
errs[w] = err
|
||||
return
|
||||
}
|
||||
numbers[w] = append(numbers[w], n)
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
if err := errors.Join(errs...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[uint64]bool{}
|
||||
for w, ns := range numbers {
|
||||
for i, n := range ns {
|
||||
if seen[n] {
|
||||
t.Errorf("the number %d was handed out twice", n)
|
||||
}
|
||||
seen[n] = true
|
||||
if i > 0 && n <= ns[i-1] {
|
||||
t.Errorf("writer %d was handed %d after %d", w, n, ns[i-1])
|
||||
}
|
||||
}
|
||||
}
|
||||
raw, err := ask(b, "get", map[string]any{"state": "numbers", "key": "issue"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var e bus.StateEntry
|
||||
_ = json.Unmarshal(raw, &e)
|
||||
if string(e.Value) != "2000" || len(seen) != 2*each {
|
||||
t.Fatalf("the counter ended at %s with %d numbers handed out, not 2000: an increment was lost", e.Value, len(seen))
|
||||
}
|
||||
if lost[0]+lost[1] == 0 {
|
||||
t.Fatal("the two writers never collided: the test proved nothing about contention")
|
||||
}
|
||||
t.Logf("2000 increments, %d writes lost to the other writer and tried again", lost[0]+lost[1])
|
||||
}
|
||||
|
||||
// A write at a revision that is no longer the key's is refused with the lost-write code and writes
|
||||
// nothing; create is refused for a key with a value and taken for one deleted; an update names the
|
||||
// revision it read, never zero.
|
||||
func TestAStaleWriteIsRefusedAndWritesNothing(t *testing.T) {
|
||||
b := counterBus(t)
|
||||
state := func(verb string, extra map[string]any) (uint64, error) {
|
||||
p := map[string]any{"state": "numbers", "key": "issue"}
|
||||
for k, v := range extra {
|
||||
p[k] = v
|
||||
}
|
||||
raw, err := ask(b, verb, p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var r struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
return r.Revision, nil
|
||||
}
|
||||
isLost := func(err error) bool {
|
||||
var coded interface{ ErrorCode() int }
|
||||
return errors.As(err, &coded) && coded.ErrorCode() == launch.CodeChanged && errors.Is(err, bus.ErrStateChanged)
|
||||
}
|
||||
value := func() string {
|
||||
raw, err := ask(b, "get", map[string]any{"state": "numbers", "key": "issue"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var e *bus.StateEntry
|
||||
_ = json.Unmarshal(raw, &e)
|
||||
if e == nil {
|
||||
return "none"
|
||||
}
|
||||
return string(e.Value)
|
||||
}
|
||||
|
||||
if _, err := state("update", map[string]any{"value": 1, "revision": 0}); err == nil || isLost(err) {
|
||||
t.Fatalf("an update at revision zero: %v", err)
|
||||
}
|
||||
if value() != "none" {
|
||||
t.Fatal("an update at revision zero wrote")
|
||||
}
|
||||
first, err := state("create", map[string]any{"value": 1})
|
||||
if err != nil || first == 0 {
|
||||
t.Fatalf("create of a key with no value: %d %v", first, err)
|
||||
}
|
||||
if _, err := state("create", map[string]any{"value": 9}); !isLost(err) {
|
||||
t.Fatalf("a second create: %v", err)
|
||||
}
|
||||
second, err := state("update", map[string]any{"value": 2, "revision": first})
|
||||
if err != nil || second <= first {
|
||||
t.Fatalf("an update at the revision read: %d %v", second, err)
|
||||
}
|
||||
if _, err := state("update", map[string]any{"value": 9, "revision": first}); !isLost(err) {
|
||||
t.Fatalf("an update at a stale revision: %v", err)
|
||||
}
|
||||
if _, err := state("update", map[string]any{"value": 9, "revision": second + 100}); !isLost(err) {
|
||||
t.Fatalf("an update at a revision not yet written: %v", err)
|
||||
}
|
||||
if v := value(); v != "2" {
|
||||
t.Fatalf("refused writes changed the value to %s", v)
|
||||
}
|
||||
if _, err := ask(b, "delete", map[string]any{"state": "numbers", "key": "issue"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := state("create", map[string]any{"value": 1}); err != nil {
|
||||
t.Fatalf("create of a deleted key: %v", err)
|
||||
}
|
||||
if v := value(); v != "1" {
|
||||
t.Fatalf("after delete and create: %s", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// historyBuses are three modules on one runtime, each reached as a bundle reaches it (novox/hq ADR
|
||||
// 0307): issues keeps a state `issues` in a bucket keeping 8 values a key, triage reads it as
|
||||
// `issues.issues`, and other was issued no state.
|
||||
func historyBuses(t *testing.T) (owner, reader, other launch.Bus) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
mesh.BucketKeeping(t, "issues_issues", 8)
|
||||
m := mt.MembershipOf("issues", "anchor", false, nil)
|
||||
m.State = []bus.StateIssued{{Name: "issues", Bucket: "issues_issues", Writes: true}}
|
||||
mesh.Issue(t, m)
|
||||
r := mt.MembershipOf("triage", "anchor", false, nil)
|
||||
r.State = []bus.StateIssued{{Name: "issues.issues", Bucket: "issues_issues"}}
|
||||
mesh.Issue(t, r)
|
||||
mesh.Issue(t, mt.MembershipOf("other", "anchor", false, nil))
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
for _, module := range []string{"issues", "triage", "other"} {
|
||||
conn.Follow(module)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
for _, module := range []string{"issues", "triage", "other"} {
|
||||
if conn.Membership(module) == nil {
|
||||
return errors.New(module + " not issued yet")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
return all.forModule("issues"), all.forModule("triage"), all.forModule("other")
|
||||
}
|
||||
|
||||
// kept is one value of a history answer, by the field names on the wire.
|
||||
type kept struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Op string `json:"op"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
func historyOf(t *testing.T, b launch.Bus, state, key string) []kept {
|
||||
t.Helper()
|
||||
raw, err := ask(b, "history", map[string]any{"state": state, "key": key})
|
||||
if err != nil {
|
||||
t.Fatalf("history of %s %s: %v", state, key, err)
|
||||
}
|
||||
var h []kept
|
||||
if err := json.Unmarshal(raw, &h); err != nil || h == nil {
|
||||
t.Fatalf("history answered %s, not an array", raw)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// A key's history is every value the bucket keeps, oldest first, each with its revision, op and time
|
||||
// in RFC 3339 — a delete with no value; an unknown key's is []. The owner and a reader are served it,
|
||||
// and reading it writes nothing.
|
||||
func TestAKeysHistoryIsEveryValueKeptOldestFirst(t *testing.T) {
|
||||
owner, reader, _ := historyBuses(t)
|
||||
before := time.Now().Add(-time.Second)
|
||||
var revisions []uint64
|
||||
for _, v := range []string{`{"state":"open"}`, `{"state":"fixing"}`, `{"state":"resolved"}`} {
|
||||
raw, err := ask(owner, "put", map[string]any{"state": "issues", "key": "502", "value": json.RawMessage(v)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var r struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
revisions = append(revisions, r.Revision)
|
||||
}
|
||||
if _, err := ask(owner, "put", map[string]any{"state": "issues", "key": "503", "value": 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ask(owner, "delete", map[string]any{"state": "issues", "key": "502"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for who, b := range map[string]launch.Bus{"the owner": owner, "a reader": reader} {
|
||||
name := "issues"
|
||||
if b == reader {
|
||||
name = "issues.issues"
|
||||
}
|
||||
h := historyOf(t, b, name, "502")
|
||||
if len(h) != 4 {
|
||||
t.Fatalf("%s read %d values of 502, not 4: %+v", who, len(h), h)
|
||||
}
|
||||
for i, v := range []string{`{"state":"open"}`, `{"state":"fixing"}`, `{"state":"resolved"}`} {
|
||||
if h[i].Op != "put" || string(h[i].Value) != v || h[i].Revision != revisions[i] || h[i].Key != "502" {
|
||||
t.Errorf("%s: value %d is %+v", who, i, h[i])
|
||||
}
|
||||
}
|
||||
if h[3].Op != "delete" || len(h[3].Value) != 0 && string(h[3].Value) != "null" || h[3].Revision <= revisions[2] {
|
||||
t.Errorf("%s: the delete reads %+v", who, h[3])
|
||||
}
|
||||
for i, k := range h {
|
||||
at, err := time.Parse(time.RFC3339, k.At)
|
||||
if err != nil || at.Before(before) || at.After(time.Now().Add(time.Second)) {
|
||||
t.Errorf("%s: value %d's time %q is not RFC 3339 and now: %v", who, i, k.At, err)
|
||||
}
|
||||
if i > 0 && k.Revision <= h[i-1].Revision {
|
||||
t.Errorf("%s: not oldest first: %+v", who, h)
|
||||
}
|
||||
}
|
||||
if h := historyOf(t, b, name, "999"); len(h) != 0 {
|
||||
t.Errorf("%s: an unknown key's history is %+v, not []", who, h)
|
||||
}
|
||||
}
|
||||
// The history is read, never written: the key is still deleted and keeps the same four values.
|
||||
if raw, err := ask(owner, "get", map[string]any{"state": "issues", "key": "502"}); err != nil || string(raw) != "null" {
|
||||
t.Fatalf("after history, 502 reads %s %v", raw, err)
|
||||
}
|
||||
if h := historyOf(t, owner, "issues", "502"); len(h) != 4 {
|
||||
t.Fatalf("reading the history changed it: %+v", h)
|
||||
}
|
||||
// A key the bucket holds nothing for answers [] on the wire, not null.
|
||||
if raw, _ := ask(owner, "history", map[string]any{"state": "issues", "key": "999"}); string(raw) != "[]" {
|
||||
t.Fatalf("an unknown key's history is %s on the wire", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// A module the bucket was not issued to is refused its history with the reason, as it is a get.
|
||||
func TestAKeysHistoryIsRefusedToAModuleNotIssuedTheState(t *testing.T) {
|
||||
owner, _, other := historyBuses(t)
|
||||
if _, err := ask(owner, "put", map[string]any{"state": "issues", "key": "1", "value": 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"issues", "issues.issues"} {
|
||||
_, err := ask(other, "history", map[string]any{"state": name, "key": "1"})
|
||||
if err == nil || !strings.Contains(err.Error(), "keeps and reads no state called") ||
|
||||
!strings.Contains(err.Error(), "issued: none") {
|
||||
t.Errorf("history of %s to a module not issued it: %v", name, err)
|
||||
}
|
||||
}
|
||||
if _, err := ask(owner, "history", map[string]any{"state": "issues", "key": "a b"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "not a key") {
|
||||
t.Errorf("history of a key the bus cannot hold: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0201: a module keeps its current state in buckets it declares, and its code reaches
|
||||
// them through the runtime. The owner's instances write and read; a reader only reads; a watch hands
|
||||
// the current values — none that is deleted — and then every change; the runtime refuses, with the
|
||||
// reason, what the module was not issued, a write to state it only reads, and a value naming a
|
||||
// credential.
|
||||
func TestABundleKeepsAndWatchesItsStateThroughTheRuntime(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
kv := mesh.Bucket(t, "keeper_servers")
|
||||
// What was there before anything started: one value, and one key since deleted.
|
||||
if _, err := kv.Put("all.one", []byte(`{"url":"http://one"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put("all.gone", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := kv.Delete("all.gone"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
keeper := mt.MembershipOf("keeper", "anchor", false, nil)
|
||||
keeper.State = []bus.StateIssued{{Name: "servers", Bucket: "keeper_servers", Writes: true}}
|
||||
peeker := mt.MembershipOf("peeker", "anchor", false, nil)
|
||||
peeker.State = []bus.StateIssued{{Name: "keeper.servers", Bucket: "keeper_servers"}}
|
||||
mesh.Issue(t, keeper)
|
||||
mesh.Issue(t, peeker)
|
||||
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
asker := connect(t, "console", "workstation")
|
||||
dir := t.TempDir()
|
||||
keeperLog, peekerLog := filepath.Join(dir, "keeper.log"), filepath.Join(dir, "peeker.log")
|
||||
said := &lines{}
|
||||
stop, err := Run(nodeTools, []Served{
|
||||
{Module: "keeper", Entrypoints: []string{mt.Fixture("state-keeper.mjs")}},
|
||||
{Module: "peeker", Entrypoints: []string{mt.Fixture("state-keeper.mjs")}},
|
||||
}, map[string]map[string]string{
|
||||
"keeper": {"STATE_NAME": "servers", "STATE_LOG": keeperLog},
|
||||
"peeker": {"STATE_NAME": "keeper.servers", "STATE_LOG": peekerLog},
|
||||
}, said.logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
|
||||
// Both read the whole current state at start: the value, not the deleted key, then the end of it.
|
||||
for _, log := range []string{keeperLog, peekerLog} {
|
||||
waitFor(t, log, "current")
|
||||
lines := read(t, log)
|
||||
if lines[0] != `was put all.one {"url":"http://one"}` || lines[1] != "current" || strings.Contains(strings.Join(lines, "\n"), "all.gone") {
|
||||
t.Fatalf("the current state was not handed over as it is:\n%s\nruntime:\n%s", strings.Join(lines, "\n"), said.all())
|
||||
}
|
||||
}
|
||||
|
||||
// The owner writes; every watch sees the change.
|
||||
got, err := call(t, asker, "keeper.put@anchor", map[string]any{"state": "servers", "key": "anchor.two", "value": map[string]any{"url": "http://two"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(got, `"revision"`) {
|
||||
t.Fatalf("a put answered %s", got)
|
||||
}
|
||||
waitFor(t, keeperLog, `now put anchor.two {"url":"http://two"}`)
|
||||
waitFor(t, peekerLog, `now put anchor.two {"url":"http://two"}`)
|
||||
|
||||
// Get and keys, by the owner and by the reader.
|
||||
got, err = call(t, asker, "peeker.get@anchor", map[string]any{"state": "keeper.servers", "key": "anchor.two"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(got, `"value":{"url":"http://two"}`) {
|
||||
t.Fatalf("the reader's get answered %s", got)
|
||||
}
|
||||
got, err = call(t, asker, "peeker.get@anchor", map[string]any{"state": "keeper.servers", "key": "nothing.here"})
|
||||
if err != nil || got != "null" {
|
||||
t.Fatalf("a key with no value answered %s, %v", got, err)
|
||||
}
|
||||
got, err = call(t, asker, "keeper.keys@anchor", map[string]any{"state": "servers"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same(t, got, `["all.one","anchor.two"]`)
|
||||
|
||||
// Refused, with the reason, before anything is sent.
|
||||
for _, c := range []struct {
|
||||
key, why string
|
||||
args map[string]any
|
||||
}{
|
||||
{"peeker.put@anchor", "peeker reads keeper.servers and does not keep it",
|
||||
map[string]any{"state": "keeper.servers", "key": "x", "value": 1}},
|
||||
{"keeper.get@anchor", `keeper keeps and reads no state called "other"`,
|
||||
map[string]any{"state": "other", "key": "x"}},
|
||||
{"keeper.put@anchor", `carries a field "accessToken", which names a credential`,
|
||||
map[string]any{"state": "servers", "key": "x", "value": map[string]any{"headers": map[string]any{"accessToken": "s"}}}},
|
||||
{"keeper.put@anchor", "is not a key the bus can hold",
|
||||
map[string]any{"state": "servers", "key": "a..b", "value": 1}},
|
||||
} {
|
||||
if _, err := call(t, asker, c.key, c.args); err == nil || !strings.Contains(err.Error(), c.why) {
|
||||
t.Errorf("%s %v: want refused for %q, got %v", c.key, c.args, c.why, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A delete reaches every watch.
|
||||
if _, err := call(t, asker, "keeper.del@anchor", map[string]any{"state": "servers", "key": "all.one"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, peekerLog, "now delete all.one")
|
||||
waitFor(t, keeperLog, "now delete all.one")
|
||||
}
|
||||
|
||||
func read(t *testing.T, path string) []string {
|
||||
t.Helper()
|
||||
b, _ := os.ReadFile(path)
|
||||
return strings.Split(strings.TrimSpace(string(b)), "\n")
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, path, line string) {
|
||||
t.Helper()
|
||||
mt.Until(t, func() error {
|
||||
for _, l := range read(t, path) {
|
||||
if l == line {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errorf("%s has no %q yet: %s", filepath.Base(path), line, strings.Join(read(t, path), " | "))
|
||||
})
|
||||
}
|
||||
|
||||
// The TypeScript SDK against the runtime (ADR 0201): two watches of one state, each handed only its
|
||||
// own changes, the current values handled before the module's import goes on.
|
||||
func TestTheSDKsStateReachesTheRuntime(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
kv := mesh.Bucket(t, "sdkkeeper_servers")
|
||||
if _, err := kv.Put("all.one", []byte(`{"url":"http://one"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := mt.MembershipOf("sdkkeeper", "anchor", false, nil)
|
||||
m.State = []bus.StateIssued{{Name: "servers", Bucket: "sdkkeeper_servers", Writes: true}}
|
||||
mesh.Issue(t, m)
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
asker := connect(t, "console", "workstation")
|
||||
log := filepath.Join(t.TempDir(), "sdk.log")
|
||||
said := &lines{}
|
||||
stop, err := Run(nodeTools, []Served{{Module: "sdkkeeper", Entrypoints: []string{mt.Fixture("state-sdk.serve.mjs")}}},
|
||||
map[string]map[string]string{"sdkkeeper": {"STATE_LOG": log}}, said.logf)
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, said.all())
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
waitFor(t, log, "current")
|
||||
if got := strings.Join(read(t, log), " | "); got != "was put all.one | current" {
|
||||
t.Fatalf("the import went on with %q", got)
|
||||
}
|
||||
if _, err := call(t, asker, "sdkkeeper.register@anchor", map[string]any{"key": "anchor.two", "url": "http://two"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, log, "now put anchor.two")
|
||||
waitFor(t, log, "narrow put anchor.two")
|
||||
if _, err := call(t, asker, "sdkkeeper.register@anchor", map[string]any{"key": "all.three", "url": "http://three"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, log, "now put all.three")
|
||||
for _, l := range read(t, log) {
|
||||
if l == "narrow put all.three" {
|
||||
t.Fatalf("a narrowed watch was handed a key outside it: %s", strings.Join(read(t, log), " | "))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Settle is how long the bundles' directories stay quiet before the runtime acts on what changed there:
|
||||
// a delivery unpacks a bundle in many writes, and the runtime loads it once, when they are done
|
||||
// (issue 1945, note 23).
|
||||
var Settle = 300 * time.Millisecond
|
||||
|
||||
// watchMask is what the runtime hears of a watched directory: a file or directory made, written,
|
||||
// renamed in or out, removed or given another mode, and the directory itself removed or renamed.
|
||||
const watchMask = unix.IN_CREATE | unix.IN_CLOSE_WRITE | unix.IN_MOVED_TO | unix.IN_MOVED_FROM | unix.IN_DELETE |
|
||||
unix.IN_ATTRIB | unix.IN_DELETE_SELF | unix.IN_MOVE_SELF | unix.IN_ONLYDIR
|
||||
|
||||
// bundleWatch hears, through inotify, the directory of every entrypoint the runtime serves, or for one
|
||||
// not there yet the nearest directory above it that is, and calls changed once a burst of changes has
|
||||
// settled. Nothing is looked at on a timer: between bursts the watch sleeps in poll(2).
|
||||
type bundleWatch struct {
|
||||
fd int
|
||||
wake [2]int // a pipe: a byte written to wake[1] ends the loop
|
||||
paths []string
|
||||
byWD map[int]string
|
||||
settle time.Duration
|
||||
changed func()
|
||||
logf func(string, ...any)
|
||||
done chan struct{}
|
||||
// what names what is watched, and lost what follows when the watch fails, in the words a journal line carries.
|
||||
what, lost string
|
||||
}
|
||||
|
||||
// WatchFile hears, as the bundles are heard, the directory of the file at path, or the nearest directory above it that
|
||||
// exists, and calls changed once at its start and once after each burst of changes there has settled: a runtime waiting
|
||||
// for its broker credential reads it again when the node-engine writes it, never on a clock (issue 1496). It answers its
|
||||
// stop.
|
||||
func WatchFile(path string, changed func(), logf func(string, ...any), lost string) (func(), error) {
|
||||
return watch([]string{path}, changed, logf, path, lost)
|
||||
}
|
||||
|
||||
// watchBundles starts the watch of the entrypoints at paths. It answers its stop.
|
||||
func watchBundles(paths []string, changed func(), logf func(string, ...any)) (func(), error) {
|
||||
return watch(paths, changed, logf, "the bundles", "a bundle placed or changed there is served only after a restart")
|
||||
}
|
||||
|
||||
// watch starts the watch of the files at paths, named what in its words; lost says what follows when it fails.
|
||||
func watch(paths []string, changed func(), logf func(string, ...any), what, lost string) (func(), error) {
|
||||
fd, err := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inotify: %w", err)
|
||||
}
|
||||
w := &bundleWatch{fd: fd, paths: paths, byWD: map[int]string{}, settle: Settle, changed: changed, logf: logf,
|
||||
done: make(chan struct{}), what: what, lost: lost}
|
||||
if err := unix.Pipe2(w.wake[:], unix.O_CLOEXEC|unix.O_NONBLOCK); err != nil {
|
||||
unix.Close(fd)
|
||||
return nil, fmt.Errorf("pipe: %w", err)
|
||||
}
|
||||
w.arm()
|
||||
go w.loop()
|
||||
return func() {
|
||||
_, _ = unix.Write(w.wake[1], []byte{1})
|
||||
<-w.done
|
||||
unix.Close(w.fd)
|
||||
unix.Close(w.wake[0])
|
||||
unix.Close(w.wake[1])
|
||||
}, nil
|
||||
}
|
||||
|
||||
// arm watches, for every entrypoint, its directory, or the nearest directory above it that exists. A
|
||||
// watch already held on the same directory is kept (inotify answers its descriptor again). A directory
|
||||
// made below the watched one between its stat and its watch would be heard by nobody, so after each watch
|
||||
// the path is looked at again, and a deeper directory that came to be is watched in turn.
|
||||
func (w *bundleWatch) arm() {
|
||||
for _, p := range w.paths {
|
||||
target := filepath.Dir(p)
|
||||
watched := ""
|
||||
for {
|
||||
dir := nearestDir(target)
|
||||
if dir == watched {
|
||||
break
|
||||
}
|
||||
wd, err := unix.InotifyAddWatch(w.fd, dir, watchMask)
|
||||
if err != nil {
|
||||
w.logf("[mesh-tools] cannot watch %s for %s: %v; %s", dir, p, err, w.lost)
|
||||
break
|
||||
}
|
||||
w.byWD[wd] = dir
|
||||
watched = dir
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// nearestDir is dir when it is a directory, else the nearest directory above it.
|
||||
func nearestDir(dir string) string {
|
||||
for {
|
||||
if info, err := os.Stat(dir); err == nil && info.IsDir() {
|
||||
return dir
|
||||
}
|
||||
up := filepath.Dir(dir)
|
||||
if up == dir {
|
||||
return dir
|
||||
}
|
||||
dir = up
|
||||
}
|
||||
}
|
||||
|
||||
func (w *bundleWatch) loop() {
|
||||
defer close(w.done)
|
||||
buf := make([]byte, 64*1024)
|
||||
// Looked at once as the watch begins: a bundle that came between its load and the watch is not missed.
|
||||
pending := true
|
||||
quietAt := time.Now()
|
||||
for {
|
||||
timeout := -1
|
||||
if pending {
|
||||
timeout = int((time.Until(quietAt) + time.Millisecond - 1) / time.Millisecond)
|
||||
if timeout < 0 {
|
||||
timeout = 0
|
||||
}
|
||||
}
|
||||
fds := []unix.PollFd{{Fd: int32(w.fd), Events: unix.POLLIN}, {Fd: int32(w.wake[0]), Events: unix.POLLIN}}
|
||||
n, err := unix.Poll(fds, timeout)
|
||||
if errors.Is(err, unix.EINTR) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
w.logf("[mesh-tools] the watch of %s stopped: %v; %s", w.what, err, w.lost)
|
||||
return
|
||||
}
|
||||
if fds[1].Revents != 0 {
|
||||
return
|
||||
}
|
||||
if n == 0 {
|
||||
// Quiet for the settle: watch the directories that came to be, then act on what changed.
|
||||
pending = false
|
||||
w.arm()
|
||||
w.changed()
|
||||
continue
|
||||
}
|
||||
if fds[0].Revents&unix.POLLIN != 0 {
|
||||
w.drain(buf)
|
||||
pending = true
|
||||
quietAt = time.Now().Add(w.settle)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// drain reads every event queued, dropping the watches of directories that went away or moved: the
|
||||
// next arm watches what is at their path now.
|
||||
func (w *bundleWatch) drain(buf []byte) {
|
||||
for {
|
||||
n, err := unix.Read(w.fd, buf)
|
||||
if err != nil || n <= 0 {
|
||||
return
|
||||
}
|
||||
for off := 0; off+unix.SizeofInotifyEvent <= n; {
|
||||
wd := int(int32(binary.NativeEndian.Uint32(buf[off:])))
|
||||
mask := binary.NativeEndian.Uint32(buf[off+4:])
|
||||
length := int(binary.NativeEndian.Uint32(buf[off+12:]))
|
||||
switch {
|
||||
case mask&unix.IN_IGNORED != 0:
|
||||
delete(w.byWD, wd)
|
||||
case mask&unix.IN_MOVE_SELF != 0:
|
||||
_, _ = unix.InotifyRmWatch(w.fd, uint32(wd))
|
||||
delete(w.byWD, wd)
|
||||
}
|
||||
off += unix.SizeofInotifyEvent + length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// signatureOf is what the runtime compares to tell that an entrypoint changed: the file's identity,
|
||||
// mode, size and times; "" is a file that does not exist. Only the entrypoint: a delivery puts every file
|
||||
// of a bundle in place anew (a directory swapped in, or each file renamed in), so the entrypoint is new
|
||||
// whenever its build is, while a file beside it may be one the bundle itself writes (a cache, a socket),
|
||||
// which would load it again after every quiet spell, and modules sharing a directory each other.
|
||||
func signatureOf(path string) string {
|
||||
info, err := os.Stat(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return ""
|
||||
}
|
||||
if err != nil {
|
||||
return "unreadable: " + err.Error()
|
||||
}
|
||||
return statSignature(info)
|
||||
}
|
||||
|
||||
func statSignature(info fs.FileInfo) string {
|
||||
sig := fmt.Sprintf("%s %d %d", info.Mode(), info.Size(), info.ModTime().UnixNano())
|
||||
if st, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
sig += fmt.Sprintf(" %d:%d %d.%d", st.Dev, st.Ino, st.Ctim.Sec, st.Ctim.Nsec)
|
||||
}
|
||||
return sig
|
||||
}
|
||||
|
||||
// notStartable says why the runtime cannot start an entrypoint, in its real reason, or "" when it can
|
||||
// (issue 1945, note 23: a missing file was said as "not executable").
|
||||
func notStartable(path string) string {
|
||||
info, err := os.Stat(path)
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
return path + " does not exist; it is served as soon as it is in place"
|
||||
case err != nil:
|
||||
return path + " cannot be read: " + err.Error()
|
||||
case info.IsDir():
|
||||
return path + " is a directory, not a bundle's entrypoint"
|
||||
case info.Mode().Perm()&0o111 == 0:
|
||||
return fmt.Sprintf("%s is not executable (mode %s); a bundle the runtime serves is started, never imported, "+
|
||||
"and its build makes it executable (novox/hq ADR 0193)", path, info.Mode().Perm())
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Package wire is JSON as the TypeScript runtime writes it: no HTML escaping of <, > and &.
|
||||
package wire
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// Marshal encodes v the way JSON.stringify does, without a trailing newline.
|
||||
func Marshal(v any) ([]byte, error) {
|
||||
var b bytes.Buffer
|
||||
enc := json.NewEncoder(&b)
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.TrimRight(b.Bytes(), "\n"), nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
{
|
||||
"module": "node-tools",
|
||||
"version": "1",
|
||||
"slug": "node-tools",
|
||||
"invokes": [
|
||||
"*"
|
||||
],
|
||||
"waits": [
|
||||
"mesh-knowledge.reviewed",
|
||||
"mesh-delivery-gate.transition",
|
||||
"gitea.forge.status",
|
||||
"mesh-controller.checked",
|
||||
"operator-channel.decided.claude-code"
|
||||
],
|
||||
"provides": [
|
||||
{
|
||||
"name": "mcp-endpoint",
|
||||
"scope": "node",
|
||||
"answered-by": "helper"
|
||||
}
|
||||
],
|
||||
"serves": {
|
||||
"mcp-endpoint": {
|
||||
"port": 4270
|
||||
}
|
||||
},
|
||||
"own-secrets": {
|
||||
"broker": "${dir:mesh-state}/broker"
|
||||
},
|
||||
"listens": [
|
||||
{
|
||||
"name": "mcp",
|
||||
"port": 4270,
|
||||
"protocol": "tcp",
|
||||
"from": "machine",
|
||||
"why": "the mesh's tools for whoever is on this machine, over MCP on loopback; the machine's login is the authority (novox/hq ADR 0152, 0175)"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "mesh-state",
|
||||
"type": "directory",
|
||||
"mode": "0755",
|
||||
"place": "mesh"
|
||||
},
|
||||
{
|
||||
"id": "interpreter",
|
||||
"type": "package",
|
||||
"package": "nodejs"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "runtime",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/node-tools",
|
||||
"binary": "node-tools"
|
||||
}
|
||||
]
|
||||
},
|
||||
"state": [
|
||||
"bound"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-controller.assignments"
|
||||
]
|
||||
}
|
||||
Generated
+1845
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "@novox/mesh-tools",
|
||||
"version": "0.1.0",
|
||||
"description": "The Novox Mesh tool runtime \u2014 binds the mesh broker and serves the assigned modules' tools.",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"mesh-tools": "./dist/main.js",
|
||||
"mesh": "./dist/mesh.js"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"pretest": "tsc",
|
||||
"test": "node --test --test-concurrency=1 --experimental-strip-types 'test/*.test.ts'"
|
||||
},
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": "^0.1.6",
|
||||
"nats": "^2.29.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.20.1",
|
||||
"typescript": "^5.9.3",
|
||||
"esbuild": "^0.25.0",
|
||||
"mongodb": "^6.21.0",
|
||||
"mssql": "^11.0.2",
|
||||
"pg": "^8"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
// What a runtime serves, it announces (novox/hq ADR 0197): the NATS services protocol's discovery —
|
||||
// `$SRV.PING`, `$SRV.INFO`, `$SRV.STATS`, and the same followed by the service's name and its id —
|
||||
// answered in the io.nats.micro.v1 format with what is served at the moment of the request. Serving is
|
||||
// unchanged; this only says what is served. One service per runtime process, because the bus admits
|
||||
// one reply per request from each responder: one endpoint per tool per subject, its metadata saying
|
||||
// which module, seat, scope and machine it is. The same shape the Go runtime answers.
|
||||
|
||||
import { StringCodec } from "nats";
|
||||
import { asSchema } from "@novox/mesh-sdk/stdio";
|
||||
import type { ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import type { RuntimeBroker } from "./broker-nats.js";
|
||||
|
||||
const sc = StringCodec();
|
||||
|
||||
export const VERSION = "0.1.0";
|
||||
export const INFO_RESPONSE = "io.nats.micro.v1.info_response";
|
||||
export const PING_RESPONSE = "io.nats.micro.v1.ping_response";
|
||||
export const STATS_RESPONSE = "io.nats.micro.v1.stats_response";
|
||||
|
||||
/** One tool served on one subject, as announced. */
|
||||
export interface Endpoint {
|
||||
kind: "tool" | "seat";
|
||||
module: string;
|
||||
tool: string;
|
||||
seat?: string;
|
||||
scope?: "mesh" | "node";
|
||||
node: string;
|
||||
description: string;
|
||||
schema: unknown;
|
||||
interchangeable: boolean;
|
||||
subject: string;
|
||||
queue?: string;
|
||||
}
|
||||
|
||||
/** A seat's verb as the runtime serves it, with the definition that answers it. */
|
||||
export interface ServedSeatVerb {
|
||||
seat: string;
|
||||
verb: string;
|
||||
subject: string;
|
||||
holder: string;
|
||||
tool: ToolDefinition;
|
||||
}
|
||||
|
||||
export interface Service {
|
||||
name: string;
|
||||
id: string;
|
||||
description: string;
|
||||
metadata: Record<string, string>;
|
||||
}
|
||||
|
||||
/** The endpoint's name as the protocol allows it; the metadata, not the name, identifies it. */
|
||||
function nameOf(e: Endpoint): string {
|
||||
const clean = (s: string) => s.replace(/[^A-Za-z0-9_-]/g, "_");
|
||||
return `${clean(e.kind === "seat" ? e.seat ?? "" : e.module)}__${clean(e.tool)}`;
|
||||
}
|
||||
|
||||
/** The info_response for these endpoints. */
|
||||
export function info(s: Service, endpoints: Endpoint[]): Record<string, unknown> {
|
||||
return {
|
||||
name: s.name, id: s.id, version: VERSION, metadata: s.metadata, type: INFO_RESPONSE, description: s.description,
|
||||
endpoints: endpoints.map((e) => {
|
||||
const metadata: Record<string, string> = {
|
||||
kind: e.kind, module: e.module, tool: e.tool, node: e.node, description: e.description,
|
||||
schema: JSON.stringify(e.schema ?? {}), interchangeable: e.interchangeable ? "true" : "false",
|
||||
};
|
||||
if (e.kind === "seat") {
|
||||
metadata.seat = e.seat ?? "";
|
||||
metadata.scope = e.scope ?? "mesh";
|
||||
}
|
||||
return { name: nameOf(e), subject: e.subject, queue_group: e.queue ?? "", metadata };
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Everything served now: each module's tools on every subject issued for them, and each held
|
||||
* seat's verbs on the seat's subject. */
|
||||
export function endpointsOf(
|
||||
broker: RuntimeBroker,
|
||||
own: { module: string; tools: ToolDefinition[] }[],
|
||||
seats: ServedSeatVerb[],
|
||||
): Endpoint[] {
|
||||
const node = broker.node ?? "";
|
||||
const out: Endpoint[] = [];
|
||||
for (const { module, tools } of own) {
|
||||
for (const t of tools) {
|
||||
const served = broker.servedOn ? broker.servedOn(module, t.name) : [];
|
||||
const interchangeable = served.some((s) => s.subject === `mesh.mod.${module}.tool.${t.name}`);
|
||||
for (const s of served) {
|
||||
out.push({ kind: "tool", module, tool: t.name, node, description: t.description, schema: asSchema(t.input),
|
||||
interchangeable, subject: s.subject, queue: s.queue });
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const v of seats) {
|
||||
out.push({ kind: "seat", module: v.holder, tool: v.verb, seat: v.seat,
|
||||
scope: node && v.subject.endsWith(`.${node}`) ? "node" : "mesh", node, description: v.tool.description,
|
||||
schema: asSchema(v.tool.input), interchangeable: false, subject: v.subject });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Answer discovery for one service until stopped. */
|
||||
export function announce(broker: RuntimeBroker, s: Service, current: () => Endpoint[]): () => void {
|
||||
const started = new Date().toISOString();
|
||||
const identity = { name: s.name, id: s.id, version: VERSION, metadata: s.metadata };
|
||||
const answer = (subject: string): Uint8Array | undefined => {
|
||||
const parts = subject.split(".");
|
||||
if (parts[0] !== "$SRV" || parts.length < 2) return undefined;
|
||||
if (parts.length >= 3 && parts[2] !== s.name) return undefined; // another service's
|
||||
if (parts.length >= 4 && parts[3] !== s.id) return undefined; // another instance's
|
||||
let v: unknown;
|
||||
switch (parts[1]) {
|
||||
case "PING":
|
||||
v = { ...identity, type: PING_RESPONSE };
|
||||
break;
|
||||
case "INFO":
|
||||
v = info(s, current());
|
||||
break;
|
||||
case "STATS":
|
||||
v = { ...identity, type: STATS_RESPONSE, started, endpoints: current().map((e) => ({
|
||||
name: nameOf(e), subject: e.subject, queue_group: e.queue ?? "", num_requests: 0, num_errors: 0,
|
||||
last_error: "", processing_time: 0, average_processing_time: 0 })) };
|
||||
break;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
return sc.encode(JSON.stringify(v));
|
||||
};
|
||||
const stops: (() => void)[] = [];
|
||||
// Exactly the questions asked of every service and of this one by name and instance — what the
|
||||
// grants allow (novox/hq ADR 0197). A wildcard is refused by the bus, and a refused subscription
|
||||
// ends a runtime: on 2026-10-03 it crash-looped every container that announced itself.
|
||||
for (const verb of ["PING", "INFO", "STATS"]) {
|
||||
for (const subject of [`$SRV.${verb}`, `$SRV.${verb}.${s.name}`, `$SRV.${verb}.${s.name}.${s.id}`]) {
|
||||
stops.push(broker.raw!(subject, (subj) => answer(subj)));
|
||||
}
|
||||
}
|
||||
return () => stops.forEach((stop) => stop());
|
||||
}
|
||||
@@ -0,0 +1,680 @@
|
||||
// The tool runtime's broker client, on NATS.
|
||||
//
|
||||
// **The sdk's contract does not change** (novox/hq ADR 0106, ADR 0039): a module is written
|
||||
// against `request`, `handle`, `publish`, `subscribe`, `close`, and the runtime implements them.
|
||||
// That is why a module built before any of this runs on the new runtime without a rebuild, and
|
||||
// why the sdk's own diff for the whole bus change is three comments.
|
||||
//
|
||||
// Underneath, everything is a subject and durability is JetStream (novox/hq design 25,
|
||||
// design 29).
|
||||
//
|
||||
// mesh.mod.<module>.event.<type> an event this module emits
|
||||
// mesh.mod.<module>.tool.<tool> a tool this module serves
|
||||
// mesh.seat.<seat>.accept.<verb> work submitted to a role
|
||||
//
|
||||
// The module never writes one of those: it names its events and tools locally and the mesh
|
||||
// derives the subject (design 29 §1), so reorganising the subject space leaves every module
|
||||
// correct.
|
||||
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { createHash } from "node:crypto";
|
||||
import net from "node:net";
|
||||
import tls from "node:tls";
|
||||
import { connect as natsConnect, headers as natsHeaders, StringCodec, type JsMsg, type Subscription, type TlsOptions } from "nats";
|
||||
import type { Broker, Envelope, EventHeaders } from "@novox/mesh-sdk/messaging";
|
||||
|
||||
const sc = StringCodec();
|
||||
|
||||
/** Requests wait this long for an answer before failing. Unchanged from what modules already
|
||||
* expect, so a module's timeout handling is not something the bus quietly redefines. */
|
||||
const REQUEST_TIMEOUT_MS = 30_000;
|
||||
|
||||
export class PinMismatchError extends Error {}
|
||||
|
||||
/** A broker credential as the mesh delivers it (novox/hq ADR 0120): the bus's address, the
|
||||
* fingerprint of the certificate it must present, and the node and module the account is scoped
|
||||
* to — the runtime derives its subjects from those rather than being told them. */
|
||||
export interface Credential {
|
||||
url: string;
|
||||
fingerprint?: string;
|
||||
node?: string;
|
||||
module?: string;
|
||||
user?: string;
|
||||
password?: string;
|
||||
/** The seats this module claims, with the verbs each promises (novox/hq ADR 0159). The runtime
|
||||
* serves each claimed seat's verbs with its tools of the same name; the bus admits only the
|
||||
* holder's subscription, so claiming and not holding costs a refused subscription and nothing else. */
|
||||
claims?: { seat: string; scope?: string; serves?: string[] }[];
|
||||
}
|
||||
|
||||
/**
|
||||
* What the mesh issued this assignment (novox/hq ADR 0160): where its tools are served, in which
|
||||
* queue, the verbs of the seats it holds, where its events land, what it may reach. Read from the
|
||||
* ASSIGNMENTS stream at `mesh.assignment.<node>.<module>` — the one subject a runtime derives for
|
||||
* itself — and followed live. Absent for a mesh older than the membership, and then the runtime
|
||||
* serves the shape it always derived, and says so.
|
||||
*/
|
||||
export interface Membership {
|
||||
node: string;
|
||||
module: string;
|
||||
/** Addresses a tool is answered on; `{tool}` stands for the tool's name. */
|
||||
serves: { subject: string; queue?: string }[];
|
||||
seats?: { seat: string; verb: string; subject: string }[];
|
||||
emits: string;
|
||||
reaches?: Record<string, string[]>;
|
||||
tools: string;
|
||||
}
|
||||
|
||||
/** What a tool call answers: the module's own result, and which machine answered it
|
||||
* (novox/hq ADR 0159) — a module on several machines is otherwise an answer from nowhere. */
|
||||
export interface Answered<Res> {
|
||||
result: Res;
|
||||
node?: string;
|
||||
}
|
||||
|
||||
/** The bus as the runtime sees it: the sdk's contract, and the few things only the runtime needs —
|
||||
* an answer that says which machine gave it, serving a subject that is not a module's own tool
|
||||
* (a seat's verb), and following the memberships of the modules it serves beside its own. */
|
||||
export interface RuntimeBroker extends Broker {
|
||||
/** Call a tool by key, or — when `on` names a subject the mesh listed for it (ADR 0160) — there. */
|
||||
ask<Req, Res>(key: string, body: Req, on?: string): Promise<Answered<Res>>;
|
||||
handleSubject<Req, Res>(subject: string, handler: (body: Req) => Promise<Res>): Promise<() => void>;
|
||||
/**
|
||||
* Serve another module's tools from this connection: read its membership on this machine and
|
||||
* follow it, so `handle("<module>.<tool>")` is served where the mesh issued that module (novox/hq
|
||||
* ADR 0175: one runtime per node, every assigned module's tools). The account must be allowed
|
||||
* to read that membership and to subscribe its subjects — the node's is; a module's own is not,
|
||||
* and is refused by the bus, not here.
|
||||
*/
|
||||
follow(module: string): Promise<void>;
|
||||
/** What the mesh issued this assignment — this module's when none is named — or undefined when
|
||||
* nothing has been issued yet. */
|
||||
membership(module?: string): Membership | undefined;
|
||||
/** Called when the mesh issues a new membership to any module this connection follows; the
|
||||
* runtime re-serves on it. The membership says which module it is for. */
|
||||
onMembership(handler: (m: Membership) => void): void;
|
||||
/** The modules whose memberships this connection follows: its own and every one `follow` added. */
|
||||
serving(): string[];
|
||||
/** The module this connection is: what its credential named, and what a bare key serves as. */
|
||||
readonly module: string;
|
||||
/** Where a served module's tool is answered right now (ADR 0197: what it serves, it announces). */
|
||||
servedOn?(module: string, tool: string): { subject: string; queue?: string }[];
|
||||
/** Answer a subject in a format of its own, not a tool's reply envelope — the NATS services
|
||||
* protocol's discovery (novox/hq ADR 0197). An undefined answer is no reply. */
|
||||
raw?(subject: string, answer: (subject: string, data: Uint8Array) => Uint8Array | undefined): () => void;
|
||||
/** The machine this connection serves on, when its credential names one. */
|
||||
readonly node?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which module's tool is at work on this connection, when one is (novox/hq ADR 0175). One runtime
|
||||
* carries many modules' tools, and an event a tool emits must land on the emitting *module's*
|
||||
* subject, not the runtime's — so the runtime runs each tool inside this store, and `publish`
|
||||
* reads the module from it. Outside a tool the connection's own module stands.
|
||||
*/
|
||||
export const atWork = new AsyncLocalStorage<{ module: string }>();
|
||||
|
||||
const ASSIGNMENTS_STREAM = "ASSIGNMENTS";
|
||||
|
||||
/** The one address a runtime derives for itself (ADR 0160). */
|
||||
export function membershipSubject(node: string, module: string): string {
|
||||
return `mesh.assignment.${node}.${module}`;
|
||||
}
|
||||
|
||||
/** Whether a connection failure is worth retrying, or is a fact about this configuration that
|
||||
* retrying cannot change. The runtime's supervisor asks this and does not need to know what it
|
||||
* is connected to. */
|
||||
export function fatalBrokerReason(err: unknown): string | null {
|
||||
if (err instanceof PinMismatchError) return "the bus's certificate does not match the pin";
|
||||
const e = err as { code?: string; message?: string };
|
||||
const message = typeof e?.message === "string" ? e.message : String(err);
|
||||
if (e?.code === "ERR_INVALID_URL" || /invalid url/i.test(message)) {
|
||||
return "the bus address is not a usable URL";
|
||||
}
|
||||
if (/authorization violation|user authentication expired|permissions violation/i.test(message)) {
|
||||
return "the bus refused this account";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to the mesh bus and return a Broker.
|
||||
*
|
||||
* **A module's subjects come from its credential, not from its calls.** `node` and `module` name
|
||||
* the account the mesh issued, and every subject this client publishes or subscribes is derived
|
||||
* from them — so a module cannot name another's namespace even by mistake, and what it emits
|
||||
* matches what the mesh authorised (ADR 0074's identity rule).
|
||||
*/
|
||||
export async function connectNats(
|
||||
target: string | Credential,
|
||||
opts: { module?: string } = {},
|
||||
): Promise<RuntimeBroker> {
|
||||
const cred: Credential = typeof target === "string" ? { url: target } : target;
|
||||
const self = cred.module ?? opts.module;
|
||||
if (!self) {
|
||||
throw new Error(
|
||||
"a broker credential with no module: the runtime derives its subjects from the account " +
|
||||
"the mesh issued, and cannot guess which module it is",
|
||||
);
|
||||
}
|
||||
|
||||
const conn = await natsConnect({
|
||||
servers: cred.url,
|
||||
user: cred.user,
|
||||
pass: cred.password,
|
||||
name: `${cred.node ?? "?"}.${self}`,
|
||||
tls: cred.fingerprint ? await pinnedTls(cred.url, cred.fingerprint) : undefined,
|
||||
// Its own inbox, not a random one: every user's inbox is private to it (design 25 §4), and the
|
||||
// grant names `_INBOX.<user>.>` — a reply space the client invented would be refused, and with
|
||||
// it every pull for the next message and every answer to a tool call.
|
||||
inboxPrefix: cred.user ? `_INBOX.${cred.user}` : undefined,
|
||||
// Reconnect forever: the bus being restarted is an upgrade, not a reason for every module on
|
||||
// the mesh to exit. `close()` stays the only thing that ends the connection.
|
||||
maxReconnectAttempts: -1,
|
||||
});
|
||||
const js = conn.jetstream();
|
||||
|
||||
const subs: Subscription[] = [];
|
||||
// Every registration, and the one reader that dispatches to them. A module has one durable
|
||||
// consumer; the loop belongs to the connection rather than to a subscription.
|
||||
const listeners: { pattern: string; handler: (env: Envelope<unknown>) => Promise<void> }[] = [];
|
||||
let reading: Awaited<ReturnType<Awaited<ReturnType<typeof js.consumers.get>>["consume"]>> | undefined;
|
||||
let closed = false;
|
||||
|
||||
const node = cred.node;
|
||||
|
||||
// The memberships, one per module this connection serves, each read once and followed. A
|
||||
// module's own runtime follows one, its own; the node's runtime follows one per assigned module
|
||||
// (novox/hq ADR 0175) — the same subject shape, the same stream, read as many times as there are
|
||||
// modules, and nothing on the bus learns a new shape for it. A direct get is one request on the
|
||||
// stream's API, which is the whole of what an account may ask JetStream for a subject it is
|
||||
// granted; a 404 is a mesh that has not issued one, which is a fact to say and not an error to
|
||||
// retry.
|
||||
const issued = new Map<string, Membership | undefined>();
|
||||
const issuedHandlers: ((m: Membership) => void)[] = [];
|
||||
const follow = async (module: string): Promise<void> => {
|
||||
if (!node || issued.has(module)) return;
|
||||
issued.set(module, undefined);
|
||||
const subjectOfTheirs = membershipSubject(node, module);
|
||||
try {
|
||||
// The subject-addressed form of a direct get — the stream, then the subject, nothing in
|
||||
// the body — because that is the one address the mesh grants this account on the
|
||||
// stream's API; the body form asks the stream's root, which it may not.
|
||||
const got = await conn.request(`$JS.API.DIRECT.GET.${ASSIGNMENTS_STREAM}.${subjectOfTheirs}`,
|
||||
new Uint8Array(0), { timeout: 5_000 });
|
||||
const status = got.headers?.code ?? 0;
|
||||
if (status === 0 && got.data.length > 0) {
|
||||
issued.set(module, JSON.parse(sc.decode(got.data)) as Membership);
|
||||
}
|
||||
} catch {
|
||||
// Not readable here: an older mesh, a stream not yet asserted, or no grant. Said below.
|
||||
}
|
||||
if (!issued.get(module)) {
|
||||
console.log(`[mesh-tools] no membership issued for ${module} on ${node} yet; serving the derived shape until one arrives`);
|
||||
}
|
||||
try {
|
||||
const live = conn.subscribe(subjectOfTheirs);
|
||||
subs.push(live);
|
||||
void (async () => {
|
||||
for await (const msg of live) {
|
||||
try {
|
||||
const m = JSON.parse(sc.decode(msg.data)) as Membership;
|
||||
issued.set(module, m);
|
||||
console.log(`[mesh-tools] ${module} on ${node} was issued a new membership; re-serving on it`);
|
||||
for (const h of issuedHandlers) h(m);
|
||||
} catch (err) {
|
||||
console.log(`[mesh-tools] a membership arrived that is not one: ${err}`);
|
||||
}
|
||||
}
|
||||
})();
|
||||
} catch {
|
||||
// A subscription this account may not make is a mesh older than the membership.
|
||||
}
|
||||
};
|
||||
await follow(self);
|
||||
|
||||
/** The subjects a tool of a served module is served on: from that module's membership when
|
||||
* issued, derived otherwise (the shape the mesh issues on day one, so the two agree). */
|
||||
const servedOn = (module: string, tool: string): { subject: string; queue?: string }[] => {
|
||||
const m = issued.get(module);
|
||||
if (m) {
|
||||
const out = m.serves.map((s) => ({ subject: s.subject.replace("{tool}", tool), queue: s.queue }));
|
||||
// The verb that lists what this module serves is answered on the mesh's plain address for it
|
||||
// whatever the placement — one answer suffices, so a queue — and on this machine's beside it.
|
||||
if (tool === "tools" && m.tools && !out.some((s) => s.subject === m.tools)) {
|
||||
out.unshift({ subject: m.tools, queue: `serve.${module}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
const base = `mesh.mod.${module}.tool.${tool}`;
|
||||
const out: { subject: string; queue?: string }[] = [{ subject: base, queue: `serve.${module}` }];
|
||||
if (node) out.push({ subject: `${base}.${node}` });
|
||||
return out;
|
||||
};
|
||||
|
||||
/** Where a call by key goes: a subject this connection's own membership says it reaches, when it
|
||||
* says one — the machine's when named — else the derived shape. */
|
||||
const reachedAt = (key: string): string => {
|
||||
const [name, wanted] = key.split("@", 2);
|
||||
const reach = issued.get(self)?.reaches?.[name];
|
||||
if (reach && reach.length > 0) {
|
||||
if (wanted) {
|
||||
const at = reach.find((s) => s.endsWith(`.${wanted}`));
|
||||
if (at) return at;
|
||||
} else {
|
||||
return reach[0];
|
||||
}
|
||||
}
|
||||
return toolSubject(key, self);
|
||||
};
|
||||
|
||||
/** Answer one subject with one handler, and say which machine answered (novox/hq ADR 0159). */
|
||||
const answerOn = <Req, Res>(
|
||||
subject: string,
|
||||
queue: string | undefined,
|
||||
handler: (body: Req) => Promise<Res>,
|
||||
): (() => void) => {
|
||||
const sub = conn.subscribe(subject, queue ? { queue } : {});
|
||||
subs.push(sub);
|
||||
void (async () => {
|
||||
// **A refused subscription is said, never fatal** (novox/hq 04-ISSUES/218, as 217 for the
|
||||
// announcements). The grants are the mesh's word on what this account may answer; a subject
|
||||
// they leave out — a seat claimed here and held elsewhere — costs that subject, never the
|
||||
// module's other tools, its handlers and its provisioning. Unhandled, the refusal ended the
|
||||
// process and a module's runtime crash-looped on 2026-10-04.
|
||||
try {
|
||||
for await (const msg of sub) {
|
||||
let reply: { result?: Res; error?: string; node?: string };
|
||||
try {
|
||||
reply = { result: await handler(JSON.parse(sc.decode(msg.data)) as Req) };
|
||||
} catch (err) {
|
||||
// The caller is told, rather than left to time out: a handler that threw is a
|
||||
// different failure from a tool nobody serves, and only one of them is worth retrying.
|
||||
reply = { error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
if (node) reply.node = node;
|
||||
msg.respond(sc.encode(JSON.stringify(reply)));
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(`[mesh-tools] the bus refused ${subject}: ${err instanceof Error ? err.message : String(err)}; ` +
|
||||
"not served here, and the rest serves on");
|
||||
}
|
||||
})();
|
||||
return () => sub.unsubscribe();
|
||||
};
|
||||
|
||||
/**
|
||||
* Ask one question and await one answer, with the machine that gave it.
|
||||
*
|
||||
* Core NATS request/reply, not JetStream: a tool call must never be persisted (design 25 §3),
|
||||
* and a lost one is a timeout the caller already handles. The reply travels on the inbox the
|
||||
* request carries, which the responder may answer because its account has `allow_responses`
|
||||
* — one reply to a message it actually received, and nothing wider.
|
||||
*/
|
||||
const ask = async <Req, Res>(key: string, body: Req, on?: string): Promise<Answered<Res>> => {
|
||||
const msg = await conn.request(on ?? reachedAt(key), sc.encode(JSON.stringify(body)), {
|
||||
timeout: REQUEST_TIMEOUT_MS,
|
||||
});
|
||||
const reply = JSON.parse(sc.decode(msg.data)) as { result?: Res; error?: string; node?: string };
|
||||
if (reply.error) throw new Error(reply.error);
|
||||
return { result: reply.result as Res, node: reply.node };
|
||||
};
|
||||
|
||||
return {
|
||||
async request<Req, Res>(key: string, body: Req): Promise<Res> {
|
||||
return (await ask<Req, Res>(key, body)).result;
|
||||
},
|
||||
|
||||
ask,
|
||||
|
||||
/**
|
||||
* Answer a question, two ways (novox/hq ADR 0159): on the module's subject in a queue group,
|
||||
* so several machines may serve one tool and exactly one of them answers each call; and on the
|
||||
* same subject with this machine as its last token, so a caller that names the machine reaches
|
||||
* this instance and no other. A runtime that does not know its machine serves only the first,
|
||||
* which is how it always behaved.
|
||||
*/
|
||||
async handle<Req, Res>(key: string, handler: (body: Req) => Promise<Res>): Promise<() => void> {
|
||||
// A seat's verb named outright is served on the seat's subject as given, for a holder that
|
||||
// knows its role without a membership; everything else is a served module's own tool, served
|
||||
// where the mesh issued that module (ADR 0160). A key naming a module this connection does
|
||||
// not follow is not served here at all: a module serves its own tools, and the node's
|
||||
// runtime those of the modules it was given (ADR 0175) — never a stranger's.
|
||||
if (key.startsWith("seat:")) {
|
||||
const stop = answerOn(toolSubject(key, self), undefined, handler);
|
||||
return () => stop();
|
||||
}
|
||||
const dot = key.indexOf(".");
|
||||
const module = dot < 0 ? self : key.slice(0, dot);
|
||||
if (!issued.has(module) && module !== self) {
|
||||
throw new Error(
|
||||
`${self} cannot serve ${key}: a module serves its own tools, and a runtime those of the ` +
|
||||
"modules it follows",
|
||||
);
|
||||
}
|
||||
const tool = dot < 0 ? key : key.slice(dot + 1);
|
||||
let stops = servedOn(module, tool).map((s) => answerOn(s.subject, s.queue, handler));
|
||||
// When that module's new membership arrives, serve where it now says and stop serving where
|
||||
// it no longer does.
|
||||
issuedHandlers.push((m) => {
|
||||
if (m.module !== module) return;
|
||||
stops.forEach((stop) => stop());
|
||||
stops = servedOn(module, tool).map((s) => answerOn(s.subject, s.queue, handler));
|
||||
});
|
||||
return () => stops.forEach((stop) => stop());
|
||||
},
|
||||
|
||||
module: self,
|
||||
node,
|
||||
servedOn,
|
||||
raw(subject: string, answer: (subject: string, data: Uint8Array) => Uint8Array | undefined): () => void {
|
||||
const sub = conn.subscribe(subject);
|
||||
subs.push(sub);
|
||||
void (async () => {
|
||||
// **A refusal here is said, never fatal** (novox/hq 04-ISSUES/217). This serves discovery —
|
||||
// what the runtime says about itself — not the work; a bus that refuses it costs the mesh
|
||||
// seeing this runtime, not the runtime's tools and handlers. Unhandled, the refusal ended the
|
||||
// process and every per-module container crash-looped on 2026-10-03.
|
||||
try {
|
||||
for await (const msg of sub) {
|
||||
const body = answer(msg.subject, msg.data);
|
||||
if (body) msg.respond(body);
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(`[mesh-tools] the bus refused ${subject}: ${err instanceof Error ? err.message : String(err)}; ` +
|
||||
"discovery will not see this runtime there, and it serves on");
|
||||
}
|
||||
})();
|
||||
return () => sub.unsubscribe();
|
||||
},
|
||||
follow,
|
||||
serving: () => [...issued.keys()],
|
||||
membership: (module?: string) => issued.get(module ?? self),
|
||||
onMembership: (handler: (m: Membership) => void) => {
|
||||
issuedHandlers.push(handler);
|
||||
},
|
||||
|
||||
async handleSubject<Req, Res>(subject: string, handler: (body: Req) => Promise<Res>): Promise<() => void> {
|
||||
return answerOn(subject, undefined, handler);
|
||||
},
|
||||
|
||||
/**
|
||||
* Emit an event.
|
||||
*
|
||||
* Published into JetStream and awaited, so a publish the bus never accepted fails the emit
|
||||
* rather than vanishing — at-least-once starts at the emitter, not only the consumer
|
||||
* (ADR 0042).
|
||||
*
|
||||
* `msgID` is the event's own id, so a redelivery after a crash between publishing and
|
||||
* acknowledging is de-duplicated by the server inside its window rather than seen twice.
|
||||
*/
|
||||
async publish<T>(env: Envelope<T>): Promise<void> {
|
||||
// **The body is the payload and the metadata rides as headers** (ADR 0042). That shape
|
||||
// is what the conformance suite pins: an implementation that nested the whole envelope in
|
||||
// the body would pass every one of its own tests and agree with nobody.
|
||||
const meta = (env.headers ?? {}) as Record<string, string>;
|
||||
const h = natsHeaders();
|
||||
for (const [k, v] of Object.entries(meta)) {
|
||||
if (v != null) h.set(k, String(v));
|
||||
}
|
||||
if (!meta["content-type"]) h.set("content-type", "application/json");
|
||||
if (env.node) h.set("x-node", env.node);
|
||||
|
||||
// The emitting module's subject: the tool at work's when a served module's tool emits from
|
||||
// the node's runtime (ADR 0175), this connection's own otherwise.
|
||||
await js.publish(eventSubject(env.key, atWork.getStore()?.module ?? self), sc.encode(JSON.stringify(env.body)), {
|
||||
headers: h,
|
||||
// De-duplicated by the server inside its window, so a redelivery after a crash between
|
||||
// publishing and acknowledging is not seen twice. Only the emitter can make this id.
|
||||
msgID: meta["x-event-id"],
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* React to events.
|
||||
*
|
||||
* The durable consumer is the **controller's** to create, from what this module declared it
|
||||
* consumes (design 29 §3) — this binds to it and never creates one. A runtime that created
|
||||
* its own would be a module deciding its own delivery semantics, and its account cannot
|
||||
* reach the JetStream API to do it anyway.
|
||||
*
|
||||
* **One consumer, one loop, however many patterns a module registers.** A module has exactly one
|
||||
* durable consumer, so two loops reading it would each take half the messages — and a loop that
|
||||
* received one its own pattern does not match acknowledges it, which is the right answer for a
|
||||
* filter wider than anything registered and silent loss when it is another handler's. Every
|
||||
* registration is therefore dispatched from one reader, and a message is acknowledged once every
|
||||
* handler it is for has taken it.
|
||||
*/
|
||||
async subscribe<T>(
|
||||
pattern: string,
|
||||
handler: (env: Envelope<T>) => Promise<void>,
|
||||
): Promise<() => void> {
|
||||
const listener = { pattern, handler: handler as (env: Envelope<unknown>) => Promise<void> };
|
||||
listeners.push(listener);
|
||||
if (!reading) {
|
||||
const durable = `${cred.node ?? "?"}_${self}`;
|
||||
const consumer = await js.consumers.get("EVENTS", durable);
|
||||
const messages = await consumer.consume();
|
||||
reading = messages;
|
||||
void (async () => {
|
||||
for await (const msg of messages) {
|
||||
await deliver(msg, listeners);
|
||||
}
|
||||
})();
|
||||
}
|
||||
return () => {
|
||||
const at = listeners.indexOf(listener);
|
||||
if (at >= 0) listeners.splice(at, 1);
|
||||
if (listeners.length === 0 && reading) {
|
||||
void reading.close();
|
||||
reading = undefined;
|
||||
}
|
||||
};
|
||||
},
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
for (const sub of subs) sub.unsubscribe();
|
||||
// Drain rather than close: an in-flight reply is finished instead of dropped, which for a
|
||||
// tool call is the difference between an answer and an unexplained timeout at the caller.
|
||||
await conn.drain();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Deliver one event to every handler it is for, acknowledging only once each has taken it.
|
||||
*
|
||||
* Several registrations share one durable consumer, so matching happens here rather than by having
|
||||
* each registration read the stream: two readers of one consumer would split it between them, and a
|
||||
* message that reached the wrong one would be acknowledged as not-for-me and lost.
|
||||
*/
|
||||
async function deliver(
|
||||
msg: JsMsg,
|
||||
listeners: { pattern: string; handler: (env: Envelope<unknown>) => Promise<void> }[],
|
||||
): Promise<void> {
|
||||
let env: Envelope<unknown>;
|
||||
try {
|
||||
env = toEnvelope<unknown>(msg);
|
||||
} catch {
|
||||
// Unparseable: terminate it. Redelivering a message no version of this code can read is
|
||||
// an infinite loop, and what the controller keeps of a message given up on (DEAD_LETTERS,
|
||||
// novox/hq issue 330) is for handlers that fail, not for bytes that were never an envelope.
|
||||
msg.term();
|
||||
return;
|
||||
}
|
||||
const forThis = listeners.filter((l) => topicMatches(l.pattern, env.key));
|
||||
if (forThis.length === 0) {
|
||||
// The consumer's filters are the controller's, derived from what the module declared it
|
||||
// consumes, and may be wider than anything it registered a handler for. Acknowledge it, or it
|
||||
// would be redelivered until it expired.
|
||||
msg.ack();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
for (const l of forThis) await l.handler(env);
|
||||
msg.ack();
|
||||
} catch {
|
||||
// Negative-acknowledge with a delay, so a handler failing on a transient cause gets another
|
||||
// attempt, and one failing permanently exhausts max-deliver rather than spinning: the consumer
|
||||
// gives the event up, and the controller keeps it in DEAD_LETTERS until a person delivers it
|
||||
// again or drops it (novox/hq issue 330). Delivered again, it arrives under
|
||||
// `mesh.again.<consumer>.…`, and keyFromSubject reads the same key from it. The consumer's
|
||||
// limits are the controller's; this only says "not done".
|
||||
msg.nak(5_000);
|
||||
}
|
||||
}
|
||||
|
||||
/** Rebuild the envelope a module sees, from the subject, the headers and the payload — the
|
||||
* mirror of publish, and the reason both live beside each other. */
|
||||
function toEnvelope<T>(msg: JsMsg): Envelope<T> {
|
||||
const headers: Record<string, string> = {};
|
||||
if (msg.headers) {
|
||||
for (const k of msg.headers.keys()) headers[k] = msg.headers.get(k);
|
||||
}
|
||||
return {
|
||||
// The event's own key, recovered from the subject: `mesh.mod.<module>.event.<key>`. The
|
||||
// module never sees the subject, only the key it declared.
|
||||
key: keyFromSubject(msg.subject),
|
||||
node: headers["x-node"] ?? "",
|
||||
body: JSON.parse(sc.decode(msg.data)) as T,
|
||||
headers: headers as EventHeaders,
|
||||
};
|
||||
}
|
||||
|
||||
/** The event key a module sees: the emitter and the event, which is exactly how its manifest names
|
||||
* what it consumes (novox/hq design 29 §1, 04-ISSUES/127).
|
||||
*
|
||||
* **One vocabulary for the declaration and the handler.** This returned the event name alone, so a
|
||||
* manifest declaring `consumes: builder.built` produced a handler pattern that could never match
|
||||
* the key it was compared against — and a module consuming the same event from two emitters could
|
||||
* not tell them apart except by reading a header. The subject already carries the emitter; naming it
|
||||
* here makes a mismatch between manifest and code a typo rather than a category error. */
|
||||
function keyFromSubject(subject: string): string {
|
||||
const marker = ".event.";
|
||||
const at = subject.indexOf(marker);
|
||||
if (at < 0) return subject;
|
||||
const event = subject.slice(at + marker.length);
|
||||
// `mesh.mod.<emitter>.event.…` — the emitter is the token before the marker.
|
||||
const before = subject.slice(0, at).split(".");
|
||||
const emitter = before[before.length - 1];
|
||||
return emitter ? `${emitter}.${event}` : event;
|
||||
}
|
||||
|
||||
/** A module's own event subject. Derived, never taken from the caller: the module names its
|
||||
* event and the mesh decides where it lands (design 29 §1). */
|
||||
function eventSubject(type: string, self: string): string {
|
||||
return `mesh.mod.${self}.event.${type}`;
|
||||
}
|
||||
|
||||
/** A tool's subject. A bare name is this module's own tool; `<module>.<tool>` addresses
|
||||
* another's, which is how a request reaches a module that is not this one; `seat:<seat>.<verb>`
|
||||
* addresses a role's tool, answered by whoever holds the seat (novox/hq ADR 0132) — with
|
||||
* `seat:<seat>.<verb>@<node>` for a node-scoped seat, whose tool carries the machine (design 33 §4). */
|
||||
function toolSubject(key: string, self: string): string {
|
||||
if (key.startsWith("seat:")) {
|
||||
const rest = key.slice("seat:".length);
|
||||
const dot = rest.indexOf(".");
|
||||
if (dot < 0) throw new Error(`"${key}" names a seat and no verb: seat:<seat>.<verb>`);
|
||||
const seat = rest.slice(0, dot);
|
||||
const [verb, node] = rest.slice(dot + 1).split("@", 2);
|
||||
return node ? `mesh.seat.${seat}.tool.${verb}.${node}` : `mesh.seat.${seat}.tool.${verb}`;
|
||||
}
|
||||
// `<module>.<tool>@<node>` names the machine (novox/hq ADR 0159): the same subject with the
|
||||
// machine as its last token, which is what that instance serves beside the queue.
|
||||
const [name, node] = key.split("@", 2);
|
||||
const dot = name.indexOf(".");
|
||||
const base = dot < 0
|
||||
? `mesh.mod.${self}.tool.${name}`
|
||||
: `mesh.mod.${name.slice(0, dot)}.tool.${name.slice(dot + 1)}`;
|
||||
return node ? `${base}.${node}` : base;
|
||||
}
|
||||
|
||||
/** A seat's verb, as its holder serves it: flat for a mesh seat, carrying the machine for a
|
||||
* node-scoped one (design 33 §4) — the same shape the controller grants. */
|
||||
export function seatToolSubject(seat: string, verb: string, scope: string | undefined, node: string | undefined): string {
|
||||
const base = `mesh.seat.${seat}.tool.${verb}`;
|
||||
return scope === "node" && node ? `${base}.${node}` : base;
|
||||
}
|
||||
|
||||
function normalizeFingerprint(fingerprint: string): string {
|
||||
return fingerprint.replace(/^sha256:/i, "").replace(/:/g, "").toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Dial once to see the certificate, and refuse unless it is exactly the one the mesh pinned.
|
||||
* A certificate authority is not consulted: the mesh issued this and knows its fingerprint,
|
||||
* which is stronger than trusting whoever a machine's trust store happens to contain.
|
||||
*
|
||||
* **The pin is the only check.** What comes back is handed to the client as its TLS options, and
|
||||
* the client's transport spreads them into Node's own `tls.connect` — so the pinned certificate
|
||||
* is the one authority the handshake accepts, and the hostname check beside it is replaced with
|
||||
* one that always passes. Pinning the exact certificate makes verifying its name redundant, and
|
||||
* the bus's certificate names the seat (`mesh-broker`), not the address a machine happens to
|
||||
* dial it by: every module on the mesh met "does not match certificate's altnames" the first time
|
||||
* it reached the handshake (2026-09-28).
|
||||
*/
|
||||
async function pinnedTls(rawUrl: string, fingerprint: string): Promise<TlsOptions> {
|
||||
const url = new URL(rawUrl.includes("://") ? rawUrl : `nats://${rawUrl}`);
|
||||
const port = url.port ? Number(url.port) : 4222;
|
||||
// **The bus speaks first, in the clear.** A NATS server sends its INFO line before TLS begins,
|
||||
// and only then expects the client to start the handshake; a raw TLS connect to that port reads
|
||||
// the INFO line as a TLS record and fails with "wrong version number" — which is what every
|
||||
// module met the first time it dialled the bus being built (2026-09-28). So: connect, wait for
|
||||
// INFO, then start TLS on the same socket, and read the certificate the server presents.
|
||||
const certificate = await new Promise<tls.DetailedPeerCertificate>((resolve, reject) => {
|
||||
const plain = net.connect({ host: url.hostname, port }, () => {});
|
||||
let seenInfo = false;
|
||||
let buffered = "";
|
||||
plain.on("error", reject);
|
||||
plain.on("data", (chunk: Buffer) => {
|
||||
if (seenInfo) return;
|
||||
buffered += chunk.toString("utf8");
|
||||
if (!buffered.includes("\r\n")) return;
|
||||
seenInfo = true;
|
||||
plain.removeAllListeners("data");
|
||||
const secure = tls.connect(
|
||||
{ socket: plain, rejectUnauthorized: false, servername: url.hostname },
|
||||
() => {
|
||||
const peer = secure.getPeerCertificate(true);
|
||||
secure.end();
|
||||
resolve(peer);
|
||||
},
|
||||
);
|
||||
secure.on("error", reject);
|
||||
});
|
||||
});
|
||||
const seen = createHash("sha256").update(certificate.raw).digest("hex");
|
||||
if (seen !== normalizeFingerprint(fingerprint)) {
|
||||
throw new PinMismatchError(
|
||||
`the bus at ${url.hostname}:${port} presented ${seen}, not the pinned ${normalizeFingerprint(fingerprint)}`,
|
||||
);
|
||||
}
|
||||
const pem = `-----BEGIN CERTIFICATE-----\n${certificate.raw.toString("base64").replace(/(.{64})/g, "$1\n")}\n-----END CERTIFICATE-----\n`;
|
||||
// Node's option, not the client's: the transport passes the whole object on. `undefined` from
|
||||
// checkServerIdentity is "the name is fine"; the pin above already decided the rest.
|
||||
return { ca: pem, checkServerIdentity: () => undefined } as TlsOptions;
|
||||
}
|
||||
|
||||
/** The mesh's topic matching: `*` is one token, `#` the rest. This is the module's vocabulary —
|
||||
* a module's `consumes` pattern is matched here, and the subject it becomes is the mesh's
|
||||
* business, not the module's. */
|
||||
export function topicMatches(pattern: string, key: string): boolean {
|
||||
return matchFrom(pattern.split("."), 0, key.split("."), 0);
|
||||
}
|
||||
|
||||
function matchFrom(p: string[], pi: number, k: string[], ki: number): boolean {
|
||||
if (pi === p.length) return ki === k.length;
|
||||
// `**` is the mesh's wildcard for the rest of a name; `#` is the old bus's, accepted so a pattern
|
||||
// written either way behaves the same while both buses ship (novox/hq design 29 §1).
|
||||
if (p[pi] === "#" || p[pi] === "**") {
|
||||
for (let skip = ki; skip <= k.length; skip++) {
|
||||
if (matchFrom(p, pi + 1, k, skip)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (ki === k.length) return false;
|
||||
if (p[pi] !== "*" && p[pi] !== k[ki]) return false;
|
||||
return matchFrom(p, pi + 1, k, ki + 1);
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* The mesh's tools, for whoever is on a machine (novox/hq design 25 §7, design 34).
|
||||
*
|
||||
* Two surfaces over one thing. A command line, for somebody at a terminal; an MCP server, for an
|
||||
* agent. Both are adapters over the same three calls — what tools are there, what does this one take,
|
||||
* call it — because a second way of reaching a tool is a second thing to keep correct.
|
||||
*
|
||||
* **It uses the same client a module's runtime uses.** Not a second protocol and not a bridge: the
|
||||
* caller connects as its own bus user — a person's, or the console's — publishes on the tool subjects
|
||||
* that account permits, and the server refuses anything else. So "what may this ask" is answered by the
|
||||
* same permission list that answers it for a module, and there is nothing here for an audit to read
|
||||
* separately.
|
||||
*
|
||||
* What the caller may NOT do is the more interesting half, and none of it is enforced here — it is the
|
||||
* account (design 25 §4): it cannot publish an event, so it cannot claim a module said something; it
|
||||
* has no consumer, so there is no delivery to acknowledge; and it cannot answer a request, so it cannot
|
||||
* impersonate a module on a bus where anyone may serve a tool.
|
||||
*/
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
import type { Broker } from "@novox/mesh-sdk/messaging";
|
||||
|
||||
import { connectNats, type Answered, type Credential } from "./broker-nats.js";
|
||||
import { TOOLS_VERB, type ToolsAnswer } from "./runtime.js";
|
||||
|
||||
/** Where the catalogue answers which modules the mesh holds. */
|
||||
const CATALOGUE_MODULES = "mesh-catalog.catalog_modules";
|
||||
|
||||
/** Where the mesh answers every role's tools, from its records: the mesh-controller seat's own
|
||||
* `tools` verb (novox/hq ADR 0154, design 33 §5). */
|
||||
const SEAT_TOOLS = "seat:mesh-controller.tools";
|
||||
|
||||
/** A tool as its module describes it. */
|
||||
export interface Tool {
|
||||
/** The module that serves it — or, for a role's tool, the seat. */
|
||||
module: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
/** The JSON schema of what it takes, as the module declared it. */
|
||||
input?: unknown;
|
||||
/** True for a role's tool: addressed to the seat, answered by whoever holds it (ADR 0132). */
|
||||
seat?: boolean;
|
||||
/** A role's scope: `node` for a seat held once per machine, whose verb is asked of one machine
|
||||
* (design 33 §4) and takes `node` for it; `mesh` or absent otherwise. */
|
||||
scope?: string;
|
||||
/** Where the tool is answered, as the mesh issued it (ADR 0160): the plain subject first when the
|
||||
* module answers for itself anywhere, then one per machine. Absent for a runtime older than this. */
|
||||
subjects?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* What the mesh could say about its tools when asked (design 34 §3).
|
||||
*
|
||||
* **Silence is named, never dropped.** A module the catalogue holds and nothing answered for is in
|
||||
* `notAnswering`, because a tool that is not offered looks exactly like a tool that does not exist,
|
||||
* and those need different people to fix them.
|
||||
*/
|
||||
export interface Listing {
|
||||
tools: Tool[];
|
||||
/** Modules the catalogue holds whose runtime did not answer `tools`: not assigned, not up, or built
|
||||
* before the runtime answered it. Each may still be called by name. The mesh's own records are
|
||||
* listed here as `mesh-controller (seat)` when the control plane did not answer. */
|
||||
notAnswering: string[];
|
||||
}
|
||||
|
||||
/** The seats and the verbs each declares, from the last listing, so a call can tell a role's tool
|
||||
* from a module's when the two share a prefix (a module and a seat may share a name). */
|
||||
export type Seats = Map<string, Set<string>>;
|
||||
|
||||
/** The roles' tools, keyed the way `toolKey` names them. */
|
||||
export function seatsIn(have: Listing): Seats {
|
||||
const seats: Seats = new Map();
|
||||
for (const t of have.tools) {
|
||||
if (!t.seat) continue;
|
||||
if (!seats.has(t.module)) seats.set(t.module, new Set());
|
||||
seats.get(t.module)!.add(t.name);
|
||||
}
|
||||
return seats;
|
||||
}
|
||||
|
||||
/** The key a call uses for `<prefix>.<name>`: a role's when the prefix is a seat declaring that
|
||||
* verb, a module's otherwise. Both names for one capability are deliberate and bounded (ADR 0132);
|
||||
* the seat wins only for a verb it actually declares, so a module's own tool is never shadowed. */
|
||||
export function toolKey(name: string, seats?: Seats): string {
|
||||
if (name.startsWith("seat:")) return name;
|
||||
const dot = name.indexOf(".");
|
||||
if (dot < 0) return name;
|
||||
const prefix = name.slice(0, dot);
|
||||
const verb = name.slice(dot + 1);
|
||||
if (seats?.get(prefix)?.has(verb)) return `seat:${prefix}.${verb}`;
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* A person's credential, as `operator issue` prints it.
|
||||
*
|
||||
* The same shape a module is handed, minus the parts a module needs and a person does not: no node,
|
||||
* because a person is not on a machine, and no module, because they are not one.
|
||||
*/
|
||||
export interface PersonCredential extends Credential {
|
||||
person?: string;
|
||||
invokes?: string[];
|
||||
}
|
||||
|
||||
/** Read the credential from the file `operator issue` produced. */
|
||||
export async function credentialFrom(path: string): Promise<PersonCredential> {
|
||||
const raw = await readFile(path, "utf8");
|
||||
let held: PersonCredential;
|
||||
try {
|
||||
held = JSON.parse(raw) as PersonCredential;
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`${path} is not a credential this mesh issued: ${(e as Error).message}. ` +
|
||||
"It is the JSON `operator issue` printed, saved verbatim.",
|
||||
);
|
||||
}
|
||||
if (!held.url || !held.user || !held.password) {
|
||||
throw new Error(
|
||||
`${path} names no bus, user or password. It is the JSON \`operator issue\` printed, saved ` +
|
||||
"verbatim — not an edited copy of it.",
|
||||
);
|
||||
}
|
||||
return held;
|
||||
}
|
||||
|
||||
/** Connect as this person. The module name the runtime wants is their own user, because every subject
|
||||
* it derives is for a tool somebody else serves. */
|
||||
export async function connectAs(held: PersonCredential): Promise<Broker> {
|
||||
return connectNats({ ...held, module: held.user });
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect as the console: the module credential the mesh delivered (novox/hq ADR 0152), read from
|
||||
* the same variable every runtime reads. It names the node and the module, so the account's inbox
|
||||
* and subjects derive from what the mesh authorised and from nothing in this process's environment.
|
||||
*/
|
||||
export async function connectAsTheConsole(path: string): Promise<{ bus: Broker; who: string }> {
|
||||
const raw = await readFile(path, "utf8");
|
||||
let held: Credential;
|
||||
try {
|
||||
held = JSON.parse(raw) as Credential;
|
||||
} catch (e) {
|
||||
throw new Error(`${path} is not a broker credential: ${(e as Error).message}`);
|
||||
}
|
||||
if (!held.url || !held.module || !held.user) {
|
||||
throw new Error(
|
||||
`${path} names no bus, module or user: the console runs on the credential the mesh sealed to ` +
|
||||
"this machine for it, and nothing else",
|
||||
);
|
||||
}
|
||||
return { bus: await connectNats(held), who: `${held.node ?? "?"}.${held.module}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* What tools the mesh has, asked of the modules (design 34 §3).
|
||||
*
|
||||
* The catalogue says which modules the mesh holds; each module says what it serves, through the one
|
||||
* verb its runtime answers for it. **Asked, not configured**: a client carrying its own list would be a
|
||||
* list that goes stale the first time a module is assigned. Every module is asked at once, and the bus
|
||||
* refuses at once a request nothing serves, so the cost is bounded by the modules that are up.
|
||||
*/
|
||||
export async function toolsOn(bus: Broker): Promise<Listing> {
|
||||
const [answered, roles] = await Promise.all([
|
||||
bus.request<Record<string, never>, { modules?: { module: string }[] }>(CATALOGUE_MODULES, {}),
|
||||
// The roles' tools, from the mesh's records (design 33 §5). Asked beside the modules rather
|
||||
// than first: a control plane that is restarting must not hide every module's tools with it.
|
||||
bus
|
||||
.request<Record<string, never>, { seats?: { seat: string; scope?: string; tools?: ToolsAnswer["tools"] }[] }>(
|
||||
SEAT_TOOLS,
|
||||
{},
|
||||
)
|
||||
.catch(() => undefined),
|
||||
]);
|
||||
const names = (answered.modules ?? []).map((m) => m.module).filter((m) => typeof m === "string");
|
||||
|
||||
const asked = await Promise.allSettled(
|
||||
names.map((module) => bus.request<Record<string, never>, ToolsAnswer>(`${module}.${TOOLS_VERB}`, {})),
|
||||
);
|
||||
const tools: Tool[] = [];
|
||||
const notAnswering: string[] = [];
|
||||
if (roles) {
|
||||
for (const s of roles.seats ?? []) {
|
||||
// A node-scoped seat's tool is asked of one machine (design 33 §4): listed with its scope, so
|
||||
// a caller names the machine and the call carries it — `seat:<seat>.<verb>@<node>`. Left out
|
||||
// of the listing, the verb never resolved as a seat's and nothing served it (ADR 0170).
|
||||
for (const t of s.tools ?? []) {
|
||||
tools.push({ module: s.seat, name: t.name, description: t.description, input: t.input, seat: true, scope: s.scope });
|
||||
}
|
||||
}
|
||||
} else {
|
||||
notAnswering.push("mesh-controller (seat)");
|
||||
}
|
||||
asked.forEach((outcome, i) => {
|
||||
const module = names[i]!;
|
||||
if (outcome.status === "fulfilled" && typeof outcome.value?.failed === "string") {
|
||||
// The runtime answered for it and serves nothing: the bundle failed to load (ADR 0175). Said
|
||||
// with the reason, because "not answering" would send somebody to check an assignment that
|
||||
// is fine.
|
||||
notAnswering.push(`${module} (its tools bundle failed to load: ${outcome.value.failed})`);
|
||||
} else if (outcome.status === "fulfilled" && Array.isArray(outcome.value?.tools)) {
|
||||
for (const t of outcome.value.tools) {
|
||||
tools.push({ module, name: t.name, description: t.description, input: t.input, subjects: t.subjects });
|
||||
}
|
||||
} else {
|
||||
notAnswering.push(module);
|
||||
}
|
||||
});
|
||||
tools.sort((a, b) => `${a.module}.${a.name}`.localeCompare(`${b.module}.${b.name}`));
|
||||
notAnswering.sort();
|
||||
return { tools, notAnswering };
|
||||
}
|
||||
|
||||
/** Call one tool. The key is `<module>.<tool>`, which is what a person types and what the account
|
||||
* permits — one vocabulary, so a refusal names the thing they asked for. */
|
||||
/** Call a tool and learn which machine answered (novox/hq ADR 0159). `<module>.<tool>@<node>` asks
|
||||
* the instance on one machine; without it, whichever instance answers first does, and the answer
|
||||
* says which. */
|
||||
export async function callTool(
|
||||
bus: Broker,
|
||||
key: string,
|
||||
args: unknown,
|
||||
seats?: Seats,
|
||||
listing?: Listing,
|
||||
): Promise<Answered<unknown>> {
|
||||
const at = key.indexOf("@");
|
||||
const name = at < 0 ? key : key.slice(0, at);
|
||||
const node = at < 0 ? "" : key.slice(at + 1);
|
||||
if (!name.includes(".")) {
|
||||
throw new Error(
|
||||
`"${key}" does not name a tool: write <module>.<tool>, as \`mesh tools\` lists them, ` +
|
||||
"or <module>.<tool>@<node> for the instance on one machine",
|
||||
);
|
||||
}
|
||||
const resolved = toolKey(name, seats) + (node ? `@${node}` : "");
|
||||
// Where the tool is answered is the module's to say and the mesh's to issue (ADR 0160): when the
|
||||
// listing carried subjects for it, the call goes to one of those and composes nothing.
|
||||
const on = subjectListed(name, node, listing);
|
||||
const asking = bus as Broker & { ask?: <Req, Res>(k: string, b: Req, on?: string) => Promise<Answered<Res>> };
|
||||
if (typeof asking.ask === "function") return asking.ask<unknown, unknown>(resolved, args ?? {}, on);
|
||||
return { result: await bus.request<unknown, unknown>(resolved, args ?? {}) };
|
||||
}
|
||||
|
||||
/** The subject the listing says answers `<module>.<tool>` — the machine's when one is named, else
|
||||
* the plain one — or undefined when the listing said none, and the key is composed as before. */
|
||||
export function subjectListed(name: string, node: string, listing?: Listing): string | undefined {
|
||||
if (!listing) return undefined;
|
||||
const dot = name.indexOf(".");
|
||||
const module = name.slice(0, dot);
|
||||
const tool = name.slice(dot + 1);
|
||||
const found = listing.tools.find((t) => t.module === module && t.name === tool && !t.seat);
|
||||
const subjects = found?.subjects ?? [];
|
||||
if (subjects.length === 0) return undefined;
|
||||
if (node) return subjects.find((s) => s.endsWith(`.${node}`));
|
||||
return subjects[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Why a call failed, said so that the remedy is in the words.
|
||||
*
|
||||
* Three answers a person actually gets, and they need different things done: nobody serves that tool,
|
||||
* the mesh refused this account, or the tool itself failed. Without this they are one timeout and a
|
||||
* stack trace.
|
||||
*/
|
||||
export function whyItFailed(key: string, err: unknown): string {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
if (/no responders|503/i.test(message)) {
|
||||
return `nothing serves ${key}. The module may not be assigned to any machine, or it is down` +
|
||||
(key.startsWith("seat:") ? ", or nothing holds that seat" : "") +
|
||||
" — `mesh tools` lists what answered.";
|
||||
}
|
||||
if (/permissions violation|authorization/i.test(message)) {
|
||||
return `this account may not call ${key}. What it may call was fixed when it was issued — a ` +
|
||||
"person's by `operator issue`, the console's by its manifest.";
|
||||
}
|
||||
if (/timeout/i.test(message)) {
|
||||
return `${key} did not answer in time. Something is serving it, so this is the tool being slow ` +
|
||||
"rather than absent.";
|
||||
}
|
||||
return `${key} failed: ${message}`;
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
/**
|
||||
* The console's endpoint: MCP over HTTP, on a machine's loopback (novox/hq ADR 0152, design 34 §2).
|
||||
*
|
||||
* **Loopback is the authority boundary.** Whoever can connect is on the machine, and whoever is on the
|
||||
* machine is the account that owns the mesh there (ADR 0034, ADR 0144). So there is no token and no
|
||||
* login here, and the one thing this file enforces is that it binds nothing else: a console reachable
|
||||
* from another machine would be authority over the mesh handed to whoever finds the port.
|
||||
*
|
||||
* The transport is the streamable-HTTP shape an agent host speaks: `POST /mcp` with one JSON-RPC
|
||||
* message, answered with one JSON body. No session, because the surface holds nothing per caller; no
|
||||
* event stream, because nothing here has anything to say unasked.
|
||||
*/
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
|
||||
import type { Broker } from "@novox/mesh-sdk/messaging";
|
||||
|
||||
import { mcpSurface, type Reply, type Request } from "./mcp.js";
|
||||
|
||||
/** The most a request body may be. A tool's arguments are small; a megabyte is somebody else's file. */
|
||||
const BODY_LIMIT = 1 << 20;
|
||||
|
||||
export interface Listening {
|
||||
/** Where it listens, as `host:port`, with the port the machine actually gave. */
|
||||
address: string;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
/** Hosts that are this machine and no other. */
|
||||
const loopback = new Set(["127.0.0.1", "::1", "localhost", "[::1]"]);
|
||||
|
||||
/**
|
||||
* Listen on `host:port`. Refused unless the host is loopback — said before binding, so a manifest or
|
||||
* a flag that would open the console to a network is a startup failure rather than something
|
||||
* discovered by whoever finds it.
|
||||
*/
|
||||
export async function serveMcpHttp(bus: Broker, who: string, listen: string): Promise<Listening> {
|
||||
const at = listen.lastIndexOf(":");
|
||||
if (at < 0) {
|
||||
throw new Error(`"${listen}" is not host:port`);
|
||||
}
|
||||
const host = listen.slice(0, at);
|
||||
const port = Number(listen.slice(at + 1));
|
||||
if (!loopback.has(host)) {
|
||||
throw new Error(
|
||||
`the console listens on loopback and nowhere else (novox/hq ADR 0152): "${host}" is not this ` +
|
||||
"machine's own address — whoever is on the machine owns the mesh there, and nobody else may reach this",
|
||||
);
|
||||
}
|
||||
if (!Number.isInteger(port) || port < 0 || port > 65535) {
|
||||
throw new Error(`"${listen.slice(at + 1)}" is not a port`);
|
||||
}
|
||||
|
||||
const surface = mcpSurface(bus, who);
|
||||
const server = createServer((req, res) => {
|
||||
void route(req, res, surface.handle).catch((e) => {
|
||||
json(res, 500, { jsonrpc: "2.0", id: null, error: { code: -32603, message: String(e) } });
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(port, host.replace(/^\[|\]$/g, ""), () => resolve());
|
||||
});
|
||||
const bound = server.address();
|
||||
const address = typeof bound === "object" && bound ? `${host}:${bound.port}` : listen;
|
||||
return {
|
||||
address,
|
||||
close: () =>
|
||||
new Promise<void>((resolve) => {
|
||||
server.close(() => resolve());
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async function route(
|
||||
req: IncomingMessage,
|
||||
res: ServerResponse,
|
||||
handle: (r: Request) => Promise<Reply | undefined>,
|
||||
): Promise<void> {
|
||||
const path = (req.url ?? "/").split("?")[0];
|
||||
if (path === "/") {
|
||||
res.writeHead(200, { "content-type": "text/plain; charset=utf-8" });
|
||||
res.end("the mesh's console: MCP over HTTP at POST /mcp (novox/hq design 34)\n");
|
||||
return;
|
||||
}
|
||||
if (path !== "/mcp") {
|
||||
json(res, 404, { error: "the console serves /mcp and nothing else" });
|
||||
return;
|
||||
}
|
||||
switch (req.method) {
|
||||
case "POST":
|
||||
break;
|
||||
case "DELETE":
|
||||
// A host ending a session. There is no session to end; saying so is the truthful answer.
|
||||
res.writeHead(204).end();
|
||||
return;
|
||||
case "GET":
|
||||
// A host opening an event stream. The console has nothing to say unasked.
|
||||
res.writeHead(405, { allow: "POST, DELETE" }).end();
|
||||
return;
|
||||
default:
|
||||
res.writeHead(405, { allow: "POST, DELETE" }).end();
|
||||
return;
|
||||
}
|
||||
|
||||
let body: string;
|
||||
try {
|
||||
body = await read(req);
|
||||
} catch (e) {
|
||||
json(res, 413, { jsonrpc: "2.0", id: null, error: { code: -32600, message: String(e) } });
|
||||
return;
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(body);
|
||||
} catch {
|
||||
json(res, 400, { jsonrpc: "2.0", id: null, error: { code: -32700, message: "the body is not JSON" } });
|
||||
return;
|
||||
}
|
||||
|
||||
// One message, or a batch of them; a batch is answered as a batch. A notification gets no reply
|
||||
// and, alone, no body: 202 is how the transport says "heard".
|
||||
if (Array.isArray(parsed)) {
|
||||
const replies = (await Promise.all(parsed.map((r) => handle(r as Request)))).filter(Boolean);
|
||||
if (replies.length === 0) {
|
||||
res.writeHead(202).end();
|
||||
} else {
|
||||
json(res, 200, replies);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const reply = await handle(parsed as Request);
|
||||
if (!reply) {
|
||||
res.writeHead(202).end();
|
||||
return;
|
||||
}
|
||||
json(res, 200, reply);
|
||||
}
|
||||
|
||||
function read(req: IncomingMessage): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let size = 0;
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (chunk: Buffer) => {
|
||||
size += chunk.length;
|
||||
if (size > BODY_LIMIT) {
|
||||
reject(new Error(`the request is larger than ${BODY_LIMIT} bytes`));
|
||||
req.destroy();
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
|
||||
req.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function json(res: ServerResponse, status: number, body: unknown): void {
|
||||
const text = JSON.stringify(body);
|
||||
res.writeHead(status, {
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"content-length": Buffer.byteLength(text),
|
||||
});
|
||||
res.end(text);
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
// A tools bundle as a process the runtime launches (novox/hq ADR 0188).
|
||||
//
|
||||
// The runtime does not run a tool's code itself when the bundle is not JavaScript: it starts the
|
||||
// bundle's executable as a child with the runtime's environment and speaks MCP over stdio to it —
|
||||
// `initialize`, `tools/list` once, `tools/call` per call. A Rust binary, a Go binary, a Python
|
||||
// script and a Node script are the same thing from here: a process that answers those. Everything
|
||||
// the mesh adds — the subjects from the membership, the held seats, the `tools` answer, a bundle
|
||||
// that failed named and the others serving — is the runtime's, outside this file.
|
||||
//
|
||||
// A tool the child lists as `<seat>.<verb>` is the module's implementation of that seat's verb;
|
||||
// any other name is the module's own tool. The same rule the in-process registration follows.
|
||||
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { accessSync, constants } from "node:fs";
|
||||
import type { ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { broker, type Envelope } from "@novox/mesh-sdk/messaging";
|
||||
import { atWork } from "./broker-nats.js";
|
||||
|
||||
/** The protocol version this speaks; a bundle says the same. */
|
||||
export const PROTOCOL = "2025-03-26";
|
||||
|
||||
/** How long a child has to answer `initialize` and `tools/list` before it is a failed bundle, and
|
||||
* how long a call may take before the caller is told the tool is slow rather than absent. */
|
||||
const HANDSHAKE_MS = 10_000;
|
||||
const CALL_MS = 30_000;
|
||||
|
||||
/** Whether an entrypoint is launched as a process rather than imported: anything that is not a
|
||||
* plain JavaScript file, and a JavaScript file marked executable — a bundle written against the
|
||||
* protocol in TypeScript, served the same way as any other language. */
|
||||
export function launches(entry: string): boolean {
|
||||
const javascript = /\.(m|c)?js$/.test(entry);
|
||||
let executable = false;
|
||||
try {
|
||||
accessSync(entry, constants.X_OK);
|
||||
executable = true;
|
||||
} catch {
|
||||
// not executable, or not there — importing will say which
|
||||
}
|
||||
return !javascript || executable;
|
||||
}
|
||||
|
||||
/** What a launched bundle registers: the groups the in-process path would have, by name. */
|
||||
export interface Launched {
|
||||
registrations: { module: string; tools: ToolDefinition[] }[];
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
interface Pending {
|
||||
resolve(v: any): void;
|
||||
reject(e: Error): void;
|
||||
timer: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch a bundle and learn its tools. Rejects when the child cannot be started or does not complete
|
||||
* the handshake, which the runtime records as the bundle having failed. A child that exits later is
|
||||
* started again on the next call, once; a call in flight when it died is told so.
|
||||
*/
|
||||
export async function launch(module: string, entry: string, env: NodeJS.ProcessEnv = process.env): Promise<Launched> {
|
||||
let child: ChildProcess | undefined;
|
||||
let nextId = 1;
|
||||
const pending = new Map<number, Pending>();
|
||||
let stopped = false;
|
||||
|
||||
const start = async (): Promise<void> => {
|
||||
const proc = spawn(entry, [], { stdio: ["pipe", "pipe", "pipe"], env });
|
||||
child = proc;
|
||||
let buffered = "";
|
||||
proc.stdout!.on("data", (chunk: Buffer) => {
|
||||
buffered += chunk.toString("utf8");
|
||||
let at: number;
|
||||
while ((at = buffered.indexOf("\n")) >= 0) {
|
||||
const line = buffered.slice(0, at).trim();
|
||||
buffered = buffered.slice(at + 1);
|
||||
if (!line) continue;
|
||||
let reply: { id?: number | string; method?: string; params?: unknown; result?: unknown; error?: { message?: string } };
|
||||
try {
|
||||
reply = JSON.parse(line);
|
||||
} catch {
|
||||
console.log(`[mesh-tools] ${module}'s bundle said something that is not a reply: ${line.slice(0, 120)}`);
|
||||
continue;
|
||||
}
|
||||
// **The bundle asks the runtime to emit** (novox/hq ADR 0193): published on the bus as this
|
||||
// module, and answered once the bus has accepted it, so the tool's emit means what it means
|
||||
// in-process. Nothing else a bundle may ask.
|
||||
if (typeof reply.method === "string") {
|
||||
const id = reply.id;
|
||||
const answer = (m: Record<string, unknown>) => proc.stdin!.write(JSON.stringify({ jsonrpc: "2.0", id, ...m }) + "\n");
|
||||
if (reply.method !== "mesh/publish") {
|
||||
if (id !== undefined) answer({ error: { code: -32601, message: `the runtime answers no ${reply.method} from a bundle` } });
|
||||
continue;
|
||||
}
|
||||
atWork.run({ module }, () => broker().publish(reply.params as Envelope<unknown>))
|
||||
.then(() => { if (id !== undefined) answer({ result: {} }); })
|
||||
.catch((err: unknown) => { if (id !== undefined) answer({ error: { code: -32000, message: err instanceof Error ? err.message : String(err) } }); });
|
||||
continue;
|
||||
}
|
||||
const waiting = typeof reply.id === "number" ? pending.get(reply.id) : undefined;
|
||||
if (!waiting) continue;
|
||||
pending.delete(reply.id as number);
|
||||
clearTimeout(waiting.timer);
|
||||
if (reply.error) waiting.reject(new Error(reply.error.message ?? "the bundle refused the request"));
|
||||
else waiting.resolve(reply.result);
|
||||
}
|
||||
});
|
||||
// stderr is the bundle's log; kept under the module's name so a fault reads where it belongs.
|
||||
// The last thing it said is kept, so a bundle that dies says why in its own words, not by code.
|
||||
let lastSaid = "";
|
||||
proc.stderr!.on("data", (chunk: Buffer) => {
|
||||
for (const line of chunk.toString("utf8").split("\n")) {
|
||||
if (!line.trim()) continue;
|
||||
console.log(`[${module}] ${line}`);
|
||||
if (/\S/.test(line) && !/^\s+at\s/.test(line) && !/^Node\.js v/.test(line)) lastSaid = line.trim();
|
||||
}
|
||||
});
|
||||
const exited = new Promise<never>((_, reject) => {
|
||||
proc.once("error", (err) => reject(err));
|
||||
proc.once("exit", (code, signal) => {
|
||||
const why = `${module}'s bundle exited (${signal ?? code})` + (lastSaid ? `: ${lastSaid}` : "");
|
||||
for (const [id, p] of pending) {
|
||||
pending.delete(id);
|
||||
clearTimeout(p.timer);
|
||||
p.reject(new Error(why));
|
||||
}
|
||||
if (child === proc) child = undefined;
|
||||
if (!stopped) console.log(`[mesh-tools] ${why}; started again on its next call`);
|
||||
reject(new Error(why));
|
||||
});
|
||||
});
|
||||
const ask = (method: string, params: unknown, ms: number): Promise<any> =>
|
||||
Promise.race([
|
||||
new Promise<any>((resolve, reject) => {
|
||||
const id = nextId++;
|
||||
const timer = setTimeout(() => {
|
||||
pending.delete(id);
|
||||
reject(new Error(`${module}'s bundle did not answer ${method} in ${ms / 1000}s`));
|
||||
}, ms);
|
||||
pending.set(id, { resolve, reject, timer });
|
||||
proc.stdin!.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
|
||||
}),
|
||||
exited,
|
||||
]);
|
||||
exited.catch(() => {}); // observed through the race; never unhandled
|
||||
(proc as ChildProcess & { ask?: typeof ask }).ask = ask;
|
||||
await ask("initialize", { protocolVersion: PROTOCOL, capabilities: {}, clientInfo: { name: "node-tools", version: "1" } }, HANDSHAKE_MS);
|
||||
proc.stdin!.write(JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) + "\n");
|
||||
};
|
||||
|
||||
const asking = async (method: string, params: unknown, ms: number): Promise<any> => {
|
||||
if (!child) await start();
|
||||
return (child as ChildProcess & { ask: (m: string, p: unknown, ms: number) => Promise<any> }).ask(method, params, ms);
|
||||
};
|
||||
|
||||
await start();
|
||||
const listed = (await asking("tools/list", {}, HANDSHAKE_MS)) as { tools?: { name: string; description?: string; inputSchema?: unknown }[] };
|
||||
const groups = new Map<string, ToolDefinition[]>();
|
||||
for (const t of listed.tools ?? []) {
|
||||
const dot = t.name.indexOf(".");
|
||||
const under = dot < 0 ? module : t.name.slice(0, dot);
|
||||
const name = dot < 0 ? t.name : t.name.slice(dot + 1);
|
||||
const tools = groups.get(under) ?? [];
|
||||
tools.push({
|
||||
name,
|
||||
description: t.description ?? "",
|
||||
input: (t.inputSchema as Record<string, unknown> | undefined) ?? {},
|
||||
run: async (args) => {
|
||||
const result = (await asking("tools/call", { name: t.name, arguments: args ?? {} }, CALL_MS)) as {
|
||||
content?: { type: string; text?: string }[];
|
||||
isError?: boolean;
|
||||
};
|
||||
const text = result?.content?.find((c) => c.type === "text")?.text ?? "";
|
||||
if (result?.isError) throw new Error(text || `${module}.${t.name} failed`);
|
||||
// The bundle's answer is JSON as text (that is what every MCP host renders); handed back as
|
||||
// the value it encodes so a caller on the bus sees what an in-process tool would return.
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
},
|
||||
});
|
||||
groups.set(under, tools);
|
||||
}
|
||||
return {
|
||||
registrations: [...groups].map(([under, tools]) => ({ module: under, tools })),
|
||||
stop: () => {
|
||||
stopped = true;
|
||||
child?.kill("SIGTERM");
|
||||
child = undefined;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,10 +1,14 @@
|
||||
// The runnable entrypoint. Three modes:
|
||||
//
|
||||
// mesh-tools serve — bind the broker and serve the assigned modules until
|
||||
// stopped. A module entrypoint that subscribes to events (on("#"))
|
||||
// stopped; as the node-tools module, also the console on loopback. A module entrypoint that subscribes to events (on("#"))
|
||||
// starts consuming as it is imported, so this also runs consumers.
|
||||
// mesh-tools emit TYPE [JSON] emit one event onto the mesh and exit — an operable primitive,
|
||||
// and what an events test uses to put a message on the wire.
|
||||
// mesh-tools prepare bring this module's state to the shape this version needs and exit
|
||||
// — the runtime's answer to the word the mesh asks every module
|
||||
// (novox/hq ADR 0135). The entrypoints come from MESH_PREPARE, which
|
||||
// the module's own image names beside MESH_TOOL_MODULES.
|
||||
// mesh-tools run ENTRYPOINT run one compiled module entrypoint to completion and exit — the
|
||||
// runtime side of a run-once step (novox/hq ADR 0052). It imports
|
||||
// the given entrypoint, whose top-level code does its work — seed a
|
||||
@@ -17,14 +21,19 @@
|
||||
// MESH_BROKER_FILE a sealed {url, fingerprint} the mesh delivered (novox/hq ADR 0043) — an
|
||||
// amqps account scoped to this module. Preferred: a module holds its own.
|
||||
// MESH_BROKER_URL a plain URL, for the bootstrap/admin case before a module has an account.
|
||||
// MESH_TOOL_MODULES /path/a,/path/b,… compiled module entrypoints (serve mode)
|
||||
// MESH_TOOL_MODULES <module>=/path/a,… the modules to serve and their compiled entrypoints (serve
|
||||
// mode; ADR 0175); a bare path is an entrypoint of the credential's own module
|
||||
// MESH_PREPARE /path/a,/path/b,… compiled entrypoints that prepare this module's state
|
||||
// MESH_MODULE / MESH_NODE the identity stamped onto emitted events (ADR 0042)
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { connectAmqp, fatalBrokerReason } from "./broker-amqp.js";
|
||||
import type { Credential } from "./broker-amqp.js";
|
||||
import { runTools } from "./runtime.js";
|
||||
import { connectNats, fatalBrokerReason as fatalNatsReason, type Credential } from "./broker-nats.js";
|
||||
import { takeToolEnvs, runTools, type ServedModule } from "./runtime.js";
|
||||
import { serveMcpHttp, type Listening } from "./http.js";
|
||||
|
||||
/** The credential this process connected with, for what it says beyond the connection (ADR 0159). */
|
||||
let lastCredential: Credential | undefined;
|
||||
import { invokeTool } from "@novox/mesh-sdk/tools";
|
||||
import { useBroker } from "@novox/mesh-sdk/messaging";
|
||||
import type { Broker } from "@novox/mesh-sdk/messaging";
|
||||
@@ -35,12 +44,19 @@ import { emit } from "@novox/mesh-sdk/events";
|
||||
* over the plain bootstrap URL otherwise. A scoped module assumes the foundation's exchanges exist —
|
||||
* its account may not declare them (ADR 0043).
|
||||
*/
|
||||
// fatalBrokerReasonFor is the reason a connection failure is final rather than "not yet", for
|
||||
// whichever bus this runtime is on — each transport knows its own refusals.
|
||||
function fatalBrokerReasonFor(err: unknown): string | null {
|
||||
return fatalNatsReason(err);
|
||||
}
|
||||
|
||||
async function connectBroker(): Promise<Broker> {
|
||||
const file = process.env.MESH_BROKER_FILE;
|
||||
if (file) {
|
||||
let credential: Credential;
|
||||
try {
|
||||
credential = JSON.parse(readFileSync(file, "utf8")) as Credential;
|
||||
lastCredential = credential;
|
||||
} catch (err) {
|
||||
console.error(`mesh-tools: cannot read the broker credential at ${file}: ${err}`);
|
||||
process.exit(1);
|
||||
@@ -54,7 +70,13 @@ async function connectBroker(): Promise<Broker> {
|
||||
// what the environment says.
|
||||
if (credential.node) process.env.MESH_NODE = credential.node;
|
||||
if (credential.module) process.env.MESH_MODULE = credential.module;
|
||||
return connectAmqp(credential, { assumeExchanges: true });
|
||||
// **The credential names the bus.** A module moved to the bus being built was handed a
|
||||
// credential for it — `nats://…` with user, password and fingerprint beside the address — and
|
||||
// nothing else in its environment changed (design 25; novox/hq design 28 task 5.2). The scheme
|
||||
// is enough to know which bus to speak; a runtime that always dialled the old one would keep
|
||||
// serving and answer nobody.
|
||||
// One bus (novox/hq ADR 0131, design 28 task 5.5): the credential names it, and it is this.
|
||||
return connectNats(credential);
|
||||
}
|
||||
const url = process.env.MESH_BROKER_URL;
|
||||
if (!url) {
|
||||
@@ -63,7 +85,7 @@ async function connectBroker(): Promise<Broker> {
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
return connectAmqp(url);
|
||||
return connectNats({ url });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -83,7 +105,7 @@ async function connectBrokerPatiently(): Promise<Broker> {
|
||||
try {
|
||||
return await connectBroker();
|
||||
} catch (err) {
|
||||
const fatal = fatalBrokerReason(err);
|
||||
const fatal = fatalBrokerReasonFor(err);
|
||||
if (fatal !== null) {
|
||||
console.error(`mesh-tools: ${fatal} — waiting will not fix this; giving up`);
|
||||
throw err;
|
||||
@@ -98,17 +120,68 @@ async function connectBrokerPatiently(): Promise<Broker> {
|
||||
}
|
||||
}
|
||||
|
||||
async function serve(): Promise<void> {
|
||||
const moduleEntrypoints = (process.env.MESH_TOOL_MODULES ?? "")
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
/**
|
||||
* What MESH_TOOL_MODULES names (novox/hq ADR 0175, to-be 38 WP1): `<module>=<entrypoint>` entries,
|
||||
* comma-separated, several per module allowed — the node's runtime serving every assigned module's
|
||||
* bundle. A bare path is the one-module form the per-module containers still set: an entrypoint of
|
||||
* the credential's own module. Both may appear; the result is one list of modules.
|
||||
*/
|
||||
export function servedModulesFrom(spec: string, own: string | undefined): { serves: ServedModule[]; moduleEntrypoints: string[] } {
|
||||
const serves = new Map<string, string[]>();
|
||||
const moduleEntrypoints: string[] = [];
|
||||
for (const raw of spec.split(",")) {
|
||||
const entry = raw.trim();
|
||||
if (!entry) continue;
|
||||
const eq = entry.indexOf("=");
|
||||
if (eq < 0) {
|
||||
moduleEntrypoints.push(entry);
|
||||
continue;
|
||||
}
|
||||
const module = entry.slice(0, eq).trim();
|
||||
const path = entry.slice(eq + 1).trim();
|
||||
if (!module || !path) {
|
||||
throw new Error(`MESH_TOOL_MODULES: "${entry}" is neither <module>=<entrypoint> nor an entrypoint of this module`);
|
||||
}
|
||||
if (module === own) {
|
||||
moduleEntrypoints.push(path);
|
||||
continue;
|
||||
}
|
||||
serves.set(module, [...(serves.get(module) ?? []), path]);
|
||||
}
|
||||
return { serves: [...serves].map(([module, entrypoints]) => ({ module, entrypoints })), moduleEntrypoints };
|
||||
}
|
||||
|
||||
/** The module that is the node's tool runtime (novox/hq ADR 0175, to-be 38 WP3): on its credential,
|
||||
* `serve` is also the console — MCP on the machine's loopback (design 34). */
|
||||
export const RUNTIME_MODULE = "node-tools";
|
||||
|
||||
/** Where the console listens when the runtime is node-tools and nothing says otherwise: the port
|
||||
* the module's manifest declares `from: machine`. MESH_CONSOLE_LISTEN overrides it either way. */
|
||||
const CONSOLE_LISTEN = "127.0.0.1:4270";
|
||||
|
||||
async function serve(): Promise<void> {
|
||||
const broker = await connectBrokerPatiently();
|
||||
const stop = await runTools({ broker, moduleEntrypoints });
|
||||
// Parsed after connecting: a bare entrypoint belongs to the module the credential names.
|
||||
const { serves, moduleEntrypoints } = servedModulesFrom(process.env.MESH_TOOL_MODULES ?? "", lastCredential?.module);
|
||||
// Each module's environment, composed by the mesh (ADR 0192): taken before any bundle is imported.
|
||||
const envs = takeToolEnvs();
|
||||
const stop = await runTools({ broker, serves, moduleEntrypoints, credential: lastCredential, envs });
|
||||
|
||||
// The console is this runtime's serving mode (ADR 0175 §6): as node-tools, or wherever the
|
||||
// listen address is given, the same process answers MCP on loopback for whoever is on the
|
||||
// machine. A module's own runtime in a container on the machine's network does not — two of
|
||||
// them on one port would be the fault, and the console is one per machine.
|
||||
const listen = process.env.MESH_CONSOLE_LISTEN ?? (lastCredential?.module === RUNTIME_MODULE ? CONSOLE_LISTEN : "");
|
||||
let consoleUp: Listening | undefined;
|
||||
if (listen) {
|
||||
const who = `${lastCredential?.node ?? "?"}.${lastCredential?.module ?? RUNTIME_MODULE}`;
|
||||
consoleUp = await serveMcpHttp(broker, who, listen);
|
||||
console.log(`mesh console listening on http://${consoleUp.address}/mcp as ${who}`);
|
||||
}
|
||||
|
||||
const shutdown = async (): Promise<void> => {
|
||||
stop();
|
||||
await consoleUp?.close();
|
||||
await broker.close();
|
||||
process.exit(0);
|
||||
};
|
||||
@@ -166,12 +239,49 @@ async function runEntry(entrypoint: string): Promise<void> {
|
||||
await import(pathToFileURL(entrypoint).href);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bring this module's state to the shape this version needs, and exit — the runtime's answer to the
|
||||
* one word the mesh asks every module (novox/hq ADR 0135).
|
||||
*
|
||||
* The entrypoints come from `MESH_PREPARE`, which a module's own image sets beside the entrypoints it
|
||||
* already lists there: the module knows which of its files prepares its state, and nothing else could.
|
||||
* Each is imported in the order given, to completion, with no broker — preparation runs before the
|
||||
* version that would use it, so there is nothing yet to talk to.
|
||||
*
|
||||
* **An empty list is a failure, not a no-op.** The mesh only asks this of a module whose manifest says
|
||||
* it prepares something; a module that says so and names nothing has been built wrong, and exiting 0
|
||||
* would let that version serve against a state nobody shaped.
|
||||
*/
|
||||
async function prepareState(): Promise<void> {
|
||||
const named = (process.env.MESH_PREPARE ?? "")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry) => entry !== "");
|
||||
if (named.length === 0) {
|
||||
console.error(
|
||||
"mesh-tools prepare: this module was asked to prepare its state and its image names nothing " +
|
||||
"to do it with — set MESH_PREPARE to the compiled entrypoint(s) that prepare it, the way " +
|
||||
"MESH_TOOL_MODULES names the ones it serves",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
for (const entrypoint of named) {
|
||||
console.log(`[mesh-tools] preparing with ${entrypoint}`);
|
||||
await runEntry(entrypoint);
|
||||
}
|
||||
console.log(`[mesh-tools] prepared: ${named.length} entrypoint(s) ran to completion`);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const [command, ...rest] = process.argv.slice(2);
|
||||
if (command === "run") {
|
||||
await runEntry(rest[0] ?? "");
|
||||
return;
|
||||
}
|
||||
if (command === "prepare") {
|
||||
await prepareState();
|
||||
return;
|
||||
}
|
||||
if (command === "invoke") {
|
||||
const [module, tool] = rest;
|
||||
if (!module || !tool) {
|
||||
@@ -193,4 +303,7 @@ async function main(): Promise<void> {
|
||||
await serve();
|
||||
}
|
||||
|
||||
void main();
|
||||
// Only when run, so a test can import the pieces.
|
||||
if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) {
|
||||
void main();
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* The mesh's tools as an MCP server (novox/hq design 25 §7, design 34).
|
||||
*
|
||||
* **A thin adapter and nothing more.** Every tool an agent sees is one a module answered for and one
|
||||
* this account may call; the schema is the module's own; the answer is the module's own. Nothing here
|
||||
* decides anything, which is why it is short — an MCP surface that reshaped arguments or summarised
|
||||
* answers would be a second definition of what a tool is, and the module's code is the first.
|
||||
*
|
||||
* Implemented against the protocol directly rather than through a library: the surface is three
|
||||
* methods and one framing, and a dependency here would be a dependency on every machine.
|
||||
*
|
||||
* One handler, two transports. Over stdio for a program a person starts (`mesh mcp`), over HTTP on a
|
||||
* machine's loopback for the console the mesh assigns there (`mesh serve`, http.ts). The handler does
|
||||
* not know which asked.
|
||||
*/
|
||||
import type { Broker } from "@novox/mesh-sdk/messaging";
|
||||
|
||||
import { callTool, seatsIn, toolKey, toolsOn, whyItFailed, type Listing, type Seats } from "./client.js";
|
||||
|
||||
/** The protocol version this speaks. Stated, because a host that wants another should be told so
|
||||
* rather than discovering it through a shape it did not expect. */
|
||||
export const PROTOCOL = "2025-03-26";
|
||||
|
||||
export interface Request {
|
||||
jsonrpc: string;
|
||||
id?: number | string | null;
|
||||
method: string;
|
||||
params?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface Reply {
|
||||
jsonrpc: "2.0";
|
||||
id: Request["id"];
|
||||
result?: unknown;
|
||||
error?: { code: number; message: string };
|
||||
}
|
||||
|
||||
/** How long a fetched tool list is kept before the modules are asked again. An agent asks on every
|
||||
* turn; the mesh changes on the order of minutes. */
|
||||
export const LISTING_KEPT_MS = 30_000;
|
||||
|
||||
export interface Surface {
|
||||
/** Answer one request; undefined for a notification, which expects none. */
|
||||
handle(request: Request): Promise<Reply | undefined>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The surface over one bus connection, as one account.
|
||||
*
|
||||
* The tool list is fetched when first asked and kept for a short while (design 34 §3): asking every
|
||||
* module on every `tools/list` would fan out on every agent turn for something nobody changed, and
|
||||
* never refreshing would hide a module assigned a moment ago.
|
||||
*/
|
||||
export function mcpSurface(bus: Broker, who: string): Surface {
|
||||
let known: { listing: Listing; at: number } | undefined;
|
||||
|
||||
const answer = (id: Request["id"], result: unknown): Reply => ({ jsonrpc: "2.0", id, result });
|
||||
const refuse = (id: Request["id"], code: number, message: string): Reply => ({
|
||||
jsonrpc: "2.0",
|
||||
id,
|
||||
error: { code, message },
|
||||
});
|
||||
|
||||
const listing = async (): Promise<Listing> => {
|
||||
if (!known || Date.now() - known.at > LISTING_KEPT_MS) {
|
||||
known = { listing: await toolsOn(bus), at: Date.now() };
|
||||
}
|
||||
return known.listing;
|
||||
};
|
||||
// The roles the last listing knew, so `<seat>.<verb>` resolves to the seat. Fetched once if a
|
||||
// call arrives before any list did; a listing that failed leaves no roles, and the name is then
|
||||
// a module's, which is the right fallback for a mesh whose control plane is away.
|
||||
const roles = async (): Promise<Seats | undefined> => {
|
||||
try {
|
||||
return seatsIn(await listing());
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
async handle(request) {
|
||||
// A notification has no id and expects no answer; `initialized` is the one every host sends.
|
||||
const notification = request.id === undefined || request.id === null;
|
||||
|
||||
switch (request.method) {
|
||||
case "initialize":
|
||||
return answer(request.id, {
|
||||
protocolVersion: PROTOCOL,
|
||||
capabilities: { tools: {} },
|
||||
serverInfo: { name: "mesh", version: "1" },
|
||||
// Said in the handshake, because an agent that knows whose authority it is acting under
|
||||
// can say so when a call is refused — and a refusal is the one thing here that is not
|
||||
// the mesh's fault or the tool's.
|
||||
instructions:
|
||||
`These are the tools of a Novox mesh, reached as ${who}. Every call goes to the module ` +
|
||||
`that serves it; what may be called was fixed when this account was issued, so a ` +
|
||||
`refusal means the account, not the tool. The list is what the running modules ` +
|
||||
`answered, plus every role's tools from the mesh's records — the mesh's own verbs ` +
|
||||
`(mesh-controller.status, .push, .assign …) among them; a module that did not answer ` +
|
||||
`is named in the list's _meta and can still be called by <module>.<tool>.`,
|
||||
});
|
||||
|
||||
case "notifications/initialized":
|
||||
return undefined;
|
||||
|
||||
case "ping":
|
||||
return notification ? undefined : answer(request.id, {});
|
||||
|
||||
case "tools/list": {
|
||||
let have: Listing;
|
||||
try {
|
||||
have = await listing();
|
||||
} catch (e) {
|
||||
return refuse(request.id, -32603, whyItFailed("mesh-catalog.catalog_modules", e));
|
||||
}
|
||||
return answer(request.id, {
|
||||
tools: have.tools.map((t) => ({
|
||||
name: `${t.module}.${t.name}`,
|
||||
description: t.description ?? `${t.name}, served by ${t.module}`,
|
||||
// The module's own schema, passed through — with `node`, the machine to ask when
|
||||
// the module runs on several (novox/hq ADR 0159); a seat's verb takes none, the
|
||||
// seat's scope decides. An empty object is a tool that takes nothing, which is a
|
||||
// real answer and not a missing one.
|
||||
inputSchema: t.seat && t.scope !== "node" ? asSchema(t.input)
|
||||
: t.seat ? withNode(asSchema(t.input), "the machine whose seat answers; required, the seat is held once per machine", true)
|
||||
: withNode(asSchema(t.input)),
|
||||
})),
|
||||
// Silence, named (design 34 §3): the modules the catalogue holds and nothing answered
|
||||
// for. Not a tool, so not in `tools`; not dropped either.
|
||||
_meta: { notAnswering: have.notAnswering },
|
||||
});
|
||||
}
|
||||
|
||||
case "tools/call": {
|
||||
const given = String(request.params?.name ?? "");
|
||||
const args = { ...((request.params?.arguments as Record<string, unknown> | undefined) ?? {}) };
|
||||
// The machine, when the caller names one, travels in the subject and never reaches the
|
||||
// module's arguments (novox/hq ADR 0159) — for a module's tool. A seat's verb takes no
|
||||
// machine from the console (the seat's scope decides), so a `node` among its arguments
|
||||
// is the verb's own, as `push` and `assign` take one, and is handed through untouched.
|
||||
const have = await listing().catch(() => undefined);
|
||||
const roles = have && seatsIn(have);
|
||||
const bare = given.split("@", 1)[0];
|
||||
const isSeatVerb = roles ? toolKey(bare, roles).startsWith("seat:") : false;
|
||||
// A node-scoped seat's verb is asked of one machine (design 33 §4, ADR 0170): `node`
|
||||
// names it and travels in the subject, as for a module's tool.
|
||||
const nodeScoped = isSeatVerb && (have?.tools.some((t) => t.seat && t.scope === "node" &&
|
||||
`${t.module}.${t.name}` === bare) ?? false);
|
||||
const takesNode = !isSeatVerb || nodeScoped;
|
||||
const node = takesNode && typeof args.node === "string" && args.node !== "" ? args.node : "";
|
||||
if (takesNode) delete args.node;
|
||||
if (nodeScoped && !node && !given.includes("@")) {
|
||||
return refuse(request.id, -32602, `${given} is a machine's seat's verb: name the machine with \`node\``);
|
||||
}
|
||||
const name = node && !given.includes("@") ? `${given}@${node}` : given;
|
||||
try {
|
||||
const { result, node: answeredBy } = await callTool(bus, name, args, roles, have);
|
||||
// Text, because that is what every host renders. The content is the module's answer
|
||||
// as JSON, unshaped: an adapter that flattened it would be deciding what matters in
|
||||
// somebody else's answer. Which machine answered follows it as its own line.
|
||||
const content: { type: string; text: string }[] = [
|
||||
{ type: "text", text: JSON.stringify(result, null, 2) },
|
||||
];
|
||||
if (answeredBy) content.push({ type: "text", text: `answered by ${answeredBy}` });
|
||||
return answer(request.id, { content });
|
||||
} catch (e) {
|
||||
// **An error the agent can act on, not a stack.** isError rather than a protocol
|
||||
// failure, because the call was well-formed and the mesh answered it — with a refusal,
|
||||
// an absence or a fault, and the words say which.
|
||||
return answer(request.id, {
|
||||
content: [{ type: "text", text: whyItFailed(name, e) }],
|
||||
isError: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
return notification ? undefined : refuse(request.id, -32601, `mesh's MCP surface has no ${request.method}`);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A module's declared input as a JSON schema an agent can read.
|
||||
*
|
||||
* The sdk keeps a tool's input opaque, and the catalogue's modules write it as a bare map of
|
||||
* property to description — `{ module: { type, description } }` — which is the `properties` of a
|
||||
* schema rather than a schema. Wrapped here when that is what arrived; passed through when a module
|
||||
* already wrote a schema; an empty object when it declared nothing. The module's words are kept
|
||||
* either way.
|
||||
*/
|
||||
export function asSchema(input: unknown): Record<string, unknown> {
|
||||
if (!input || typeof input !== "object" || Array.isArray(input)) {
|
||||
return { type: "object", properties: {} };
|
||||
}
|
||||
const given = input as Record<string, unknown>;
|
||||
if (given.type === "object" || "properties" in given) return given;
|
||||
if (Object.keys(given).length === 0) return { type: "object", properties: {} };
|
||||
return { type: "object", properties: given };
|
||||
}
|
||||
|
||||
/**
|
||||
* Serve over stdio until stdin closes, which is how a host ends a session.
|
||||
*/
|
||||
export async function serveMcp(bus: Broker, who: string): Promise<void> {
|
||||
const surface = mcpSurface(bus, who);
|
||||
const say = (message: unknown) => {
|
||||
process.stdout.write(`${JSON.stringify(message)}\n`);
|
||||
};
|
||||
for await (const line of lines()) {
|
||||
let request: Request;
|
||||
try {
|
||||
request = JSON.parse(line) as Request;
|
||||
} catch {
|
||||
// Unparseable, and with no id there is nobody to tell. Skipped rather than answered, because a
|
||||
// reply to a request that was never framed is noise on the same channel.
|
||||
continue;
|
||||
}
|
||||
const reply = await surface.handle(request);
|
||||
if (reply) say(reply);
|
||||
}
|
||||
}
|
||||
|
||||
/** stdin as newline-framed messages, which is what MCP over stdio is. */
|
||||
async function* lines(): AsyncGenerator<string> {
|
||||
let buffered = "";
|
||||
for await (const chunk of process.stdin) {
|
||||
buffered += (chunk as Buffer).toString("utf8");
|
||||
let at: number;
|
||||
while ((at = buffered.indexOf("\n")) >= 0) {
|
||||
const line = buffered.slice(0, at).trim();
|
||||
buffered = buffered.slice(at + 1);
|
||||
if (line !== "") yield line;
|
||||
}
|
||||
}
|
||||
if (buffered.trim() !== "") yield buffered.trim();
|
||||
}
|
||||
|
||||
/** Every module tool takes an optional `node`: the machine to ask when the module runs on several
|
||||
* (novox/hq ADR 0159). Added to the listing, stripped before the call, never seen by the module. */
|
||||
function withNode(schema: Record<string, unknown>, description?: string, required = false): Record<string, unknown> {
|
||||
const properties = { ...((schema.properties as Record<string, unknown> | undefined) ?? {}) };
|
||||
if (!("node" in properties)) {
|
||||
properties.node = {
|
||||
type: "string",
|
||||
description: description ??
|
||||
"the machine to ask, when this module runs on several; else whichever answers, and the answer says which",
|
||||
};
|
||||
}
|
||||
const out: Record<string, unknown> = { ...schema, type: "object", properties };
|
||||
if (required) {
|
||||
const have = Array.isArray(schema.required) ? (schema.required as string[]) : [];
|
||||
out.required = have.includes("node") ? have : [...have, "node"];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* `mesh` — the mesh's tools, for whoever is on a machine (novox/hq design 25 §7, design 34).
|
||||
*
|
||||
* Four verbs and nothing else. What tools are there, call one, serve the same two to an agent over
|
||||
* stdio, and serve them on a machine's loopback as the console the mesh assigns. Deliberately thin:
|
||||
* everything that could be a decision is one the mesh already made, and a client that grew opinions
|
||||
* would be a second place the mesh's behaviour is defined.
|
||||
*
|
||||
* mesh tools what the running modules answer
|
||||
* mesh call <module>.<tool> [json] call one, arguments as JSON on the command line or on stdin
|
||||
* mesh mcp the same two, as an MCP server over stdio
|
||||
* mesh serve [--listen host:port] the console: MCP over HTTP on this machine's loopback
|
||||
*
|
||||
* Who it speaks as, in order of preference:
|
||||
* MESH_BROKER_FILE the module credential the mesh delivered — the console's (ADR 0152)
|
||||
* MESH_CREDENTIAL / --credential <file> a person's, as `operator issue` printed it (design 25 §7)
|
||||
* MESH_CONSOLE / --console <url> no credential: `tools` and `call` go through a console
|
||||
* already running on this machine, over loopback HTTP
|
||||
*/
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
import type { Broker } from "@novox/mesh-sdk/messaging";
|
||||
|
||||
import {
|
||||
callTool,
|
||||
connectAs,
|
||||
connectAsTheConsole,
|
||||
credentialFrom,
|
||||
seatsIn,
|
||||
toolsOn,
|
||||
whyItFailed,
|
||||
type Listing,
|
||||
} from "./client.js";
|
||||
import { serveMcpHttp } from "./http.js";
|
||||
import { serveMcp } from "./mcp.js";
|
||||
|
||||
const usage = `mesh tools
|
||||
mesh call <module>.<tool> [json]
|
||||
mesh mcp
|
||||
mesh serve [--listen host:port]
|
||||
|
||||
--credential <file> a person's credential, the JSON \`operator issue\` printed; default $MESH_CREDENTIAL
|
||||
--console <url> a console on this machine to ask through instead; default $MESH_CONSOLE
|
||||
--listen <host:port> where \`serve\` listens; loopback only; default $MESH_CONSOLE_LISTEN or 127.0.0.1:4270
|
||||
MESH_BROKER_FILE the module credential the mesh delivered, which \`serve\` runs on`;
|
||||
|
||||
/** What the console listens on when nothing says otherwise. */
|
||||
const DEFAULT_LISTEN = "127.0.0.1:4270";
|
||||
|
||||
async function main(argv: string[]): Promise<number> {
|
||||
const args = [...argv];
|
||||
let credentialPath = process.env.MESH_CREDENTIAL ?? "";
|
||||
let consoleUrl = process.env.MESH_CONSOLE ?? "";
|
||||
let listen = process.env.MESH_CONSOLE_LISTEN ?? DEFAULT_LISTEN;
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const take = () => {
|
||||
const v = args[i + 1] ?? "";
|
||||
args.splice(i, 2);
|
||||
i--;
|
||||
return v;
|
||||
};
|
||||
if (args[i] === "--credential") credentialPath = take();
|
||||
else if (args[i] === "--console") consoleUrl = take();
|
||||
else if (args[i] === "--listen") listen = take();
|
||||
}
|
||||
const verb = args.shift();
|
||||
if (!verb || verb === "help" || verb === "--help") {
|
||||
console.log(usage);
|
||||
return verb ? 0 : 1;
|
||||
}
|
||||
|
||||
// Through a console already on this machine: no credential to hold, which is the point of one.
|
||||
if (consoleUrl && (verb === "tools" || verb === "call")) {
|
||||
return verb === "tools" ? listingVia(consoleUrl) : callingVia(consoleUrl, args);
|
||||
}
|
||||
|
||||
const { bus, who } = await connecting(credentialPath, verb);
|
||||
try {
|
||||
switch (verb) {
|
||||
case "tools":
|
||||
return await listing(bus, who);
|
||||
case "call":
|
||||
return await calling(bus, args);
|
||||
case "mcp":
|
||||
// Serves until stdin closes, which is how an MCP host ends a session.
|
||||
await serveMcp(bus, who);
|
||||
return 0;
|
||||
case "serve": {
|
||||
const up = await serveMcpHttp(bus, who, listen);
|
||||
console.log(`mesh console listening on http://${up.address}/mcp as ${who}`);
|
||||
await new Promise<void>((resolve) => {
|
||||
process.once("SIGTERM", () => resolve());
|
||||
process.once("SIGINT", () => resolve());
|
||||
});
|
||||
await up.close();
|
||||
return 0;
|
||||
}
|
||||
default:
|
||||
console.error(`mesh has no "${verb}".\n\n${usage}`);
|
||||
return 1;
|
||||
}
|
||||
} finally {
|
||||
await bus.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Who this process is on the bus. The module credential first: a console is started by the mesh with
|
||||
* MESH_BROKER_FILE and nothing else, and must not fall back to a person's file lying around.
|
||||
*/
|
||||
async function connecting(credentialPath: string, verb: string): Promise<{ bus: Broker; who: string }> {
|
||||
const delivered = process.env.MESH_BROKER_FILE;
|
||||
if (delivered) {
|
||||
return connectAsTheConsole(delivered);
|
||||
}
|
||||
if (!credentialPath) {
|
||||
throw new Error(
|
||||
verb === "serve"
|
||||
? "no credential: the console runs on MESH_BROKER_FILE, the module credential the mesh " +
|
||||
"delivered; to run it by hand, pass --credential <file> with a person's credential"
|
||||
: "no credential: set MESH_CREDENTIAL or pass --credential <file> (the JSON `operator issue` " +
|
||||
"printed, saved verbatim), or --console <url> to ask through a console on this machine",
|
||||
);
|
||||
}
|
||||
const held = await credentialFrom(credentialPath);
|
||||
return { bus: await connectAs(held), who: held.person ?? held.user ?? "somebody" };
|
||||
}
|
||||
|
||||
function printListing(have: Listing, who?: string): void {
|
||||
if (have.tools.length === 0) {
|
||||
console.log("no running module answered with any tool");
|
||||
}
|
||||
// **What the modules answered, not what this account may call.** The two differ and the
|
||||
// difference is the point: an account seeing only its own tools cannot tell "not installed" from
|
||||
// "not yours", and those need different people to fix them.
|
||||
for (const t of have.tools) {
|
||||
const name = `${t.module}.${t.name}`;
|
||||
const line = t.description ? `${name.padEnd(36)} ${t.description}` : name;
|
||||
console.log(t.seat ? `${line} (a role's tool: answered by whoever holds the ${t.module} seat)` : line);
|
||||
}
|
||||
if (have.notAnswering.length > 0) {
|
||||
console.log(
|
||||
`\nheld by the mesh and not answering: ${have.notAnswering.join(", ")} — not assigned, not up, ` +
|
||||
"or built before the runtime answered `tools`; each can still be called by name",
|
||||
);
|
||||
}
|
||||
if (who) {
|
||||
console.log(`\nasked as ${who}; what ${who} may call was fixed when the account was issued.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function listing(bus: Broker, who?: string): Promise<number> {
|
||||
let have: Listing;
|
||||
try {
|
||||
have = await toolsOn(bus);
|
||||
} catch (e) {
|
||||
console.error(whyItFailed("mesh-catalog.catalog_modules", e));
|
||||
return 1;
|
||||
}
|
||||
printListing(have, who);
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function calling(bus: Broker, args: string[]): Promise<number> {
|
||||
const key = args.shift();
|
||||
if (!key) {
|
||||
console.error("mesh call <module>.<tool> [json]");
|
||||
return 1;
|
||||
}
|
||||
const parsed = await argumentsFrom(args);
|
||||
if (parsed === undefined) return 1;
|
||||
try {
|
||||
// `<seat>.<verb>` reaches the role when the mesh lists that verb for the seat; `seat:` says so
|
||||
// outright and asks nothing first.
|
||||
const have = key.startsWith("seat:") ? undefined : await toolsOn(bus).catch(() => undefined);
|
||||
const { result, node } = await callTool(bus, key, parsed, have && seatsIn(have), have);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (node) console.error(`answered by ${node}`);
|
||||
return 0;
|
||||
} catch (e) {
|
||||
console.error(whyItFailed(key, e));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/** The console's answer to one MCP request, over loopback HTTP. */
|
||||
async function viaConsole(consoleUrl: string, method: string, params?: unknown): Promise<any> {
|
||||
const endpoint = consoleUrl.endsWith("/mcp") ? consoleUrl : `${consoleUrl.replace(/\/$/, "")}/mcp`;
|
||||
const res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", accept: "application/json" },
|
||||
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`the console at ${endpoint} answered ${res.status}`);
|
||||
}
|
||||
const reply = (await res.json()) as { result?: any; error?: { message: string } };
|
||||
if (reply.error) throw new Error(reply.error.message);
|
||||
return reply.result;
|
||||
}
|
||||
|
||||
async function listingVia(consoleUrl: string): Promise<number> {
|
||||
try {
|
||||
const result = await viaConsole(consoleUrl, "tools/list");
|
||||
const have: Listing = {
|
||||
tools: (result.tools ?? []).map((t: { name: string; description?: string; inputSchema?: unknown }) => {
|
||||
const at = t.name.indexOf(".");
|
||||
return { module: t.name.slice(0, at), name: t.name.slice(at + 1), description: t.description, input: t.inputSchema };
|
||||
}),
|
||||
notAnswering: result._meta?.notAnswering ?? [],
|
||||
};
|
||||
printListing(have);
|
||||
return 0;
|
||||
} catch (e) {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
async function callingVia(consoleUrl: string, args: string[]): Promise<number> {
|
||||
const key = args.shift();
|
||||
if (!key) {
|
||||
console.error("mesh call <module>.<tool> [json]");
|
||||
return 1;
|
||||
}
|
||||
const parsed = await argumentsFrom(args);
|
||||
if (parsed === undefined) return 1;
|
||||
try {
|
||||
const result = await viaConsole(consoleUrl, "tools/call", { name: key, arguments: parsed });
|
||||
const text = result?.content?.[0]?.text ?? JSON.stringify(result);
|
||||
if (result?.isError) {
|
||||
console.error(text);
|
||||
return 1;
|
||||
}
|
||||
console.log(text);
|
||||
return 0;
|
||||
} catch (e) {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/** A call's arguments: JSON on the command line, else on stdin, else nothing. Undefined when what
|
||||
* was given is not JSON, after saying so. */
|
||||
async function argumentsFrom(args: string[]): Promise<unknown> {
|
||||
const raw = args.length > 0 ? args.join(" ") : await maybeStdin();
|
||||
if (raw.trim() === "") return {};
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch (e) {
|
||||
console.error(`the arguments are not JSON: ${(e as Error).message}`);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Arguments on stdin, for a call whose JSON is too long or too quoted to type. Empty when stdin is a
|
||||
* terminal, so `mesh call x.y` with no arguments does not hang waiting for something nobody is
|
||||
* typing. */
|
||||
async function maybeStdin(): Promise<string> {
|
||||
if (process.stdin.isTTY) return "";
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of process.stdin) chunks.push(chunk as Buffer);
|
||||
return Buffer.concat(chunks).toString("utf8");
|
||||
}
|
||||
|
||||
// Only when run, so a test can import the pieces.
|
||||
if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) {
|
||||
main(process.argv.slice(2))
|
||||
.then((code) => process.exit(code))
|
||||
.catch((e) => {
|
||||
console.error(e instanceof Error ? e.message : String(e));
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { main, usage };
|
||||
export const _readFile = readFile;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user