Compare commits
157
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6c05e42626 | ||
|
|
91a4f1b0ad | ||
|
|
b4affb3efd | ||
|
|
fb31cc4e99 | ||
|
|
bd555e6014 | ||
|
|
c3754e48c8 | ||
|
|
a00c780352 | ||
|
|
5bbe669cfb | ||
|
|
c55b1bcf38 | ||
|
|
6965119bce | ||
|
|
387f547346 | ||
|
|
fce9189579 | ||
|
|
a2402c4873 | ||
|
|
f917fc60c1 | ||
|
|
0a6fd0cb5c | ||
|
|
a918352ff1 | ||
|
|
d5e4e4e8ab | ||
|
|
07f6078050 | ||
|
|
e40cb291f8 | ||
|
|
e0d7122567 | ||
|
|
c637da2eb7 | ||
|
|
7390652f33 | ||
|
|
db3b5fe63d | ||
|
|
76e025dbbc | ||
|
|
f87c9720ed | ||
|
|
d71cd5cef6 | ||
|
|
31247dff6b | ||
|
|
8b1ab467a9 | ||
|
|
84ede95c4a | ||
|
|
7fd9d893c0 | ||
|
|
0fb4591c9d | ||
|
|
7da4c55752 | ||
|
|
b4819dc1fb | ||
|
|
40e0e8282b | ||
|
|
01ee2ef394 | ||
|
|
8b8a760047 | ||
|
|
db3dd04be7 | ||
|
|
d13890b944 | ||
|
|
b3903446f9 | ||
|
|
e550c87a29 | ||
|
|
ff7661a136 | ||
|
|
84533edf2e | ||
|
|
ea516ece5a | ||
|
|
8c1e04dd37 | ||
|
|
43d3fba501 | ||
|
|
df929b48b5 | ||
|
|
961ec0ef96 | ||
|
|
5a9e997349 | ||
|
|
fadc73f73b | ||
|
|
d206b0dde1 | ||
|
|
2bb0a6461d | ||
|
|
38a634723b | ||
|
|
8fe7ef3b82 | ||
|
|
63c1e3faa2 | ||
|
|
ad22c5e9fe | ||
|
|
d6c516ab44 | ||
|
|
b8e42cf54c | ||
|
|
24c0ec0d32 | ||
|
|
bd24ceafa9 | ||
|
|
3f538027e9 | ||
|
|
8114b45884 | ||
|
|
d7d908cf67 | ||
|
|
e475d9f849 | ||
|
|
e6f27a5f74 | ||
|
|
c678511871 | ||
|
|
4f636b7080 | ||
|
|
49f9269376 | ||
|
|
29fd452628 | ||
|
|
331fa06b54 | ||
|
|
5b2ff5e321 | ||
|
|
4fbfdfad13 | ||
|
|
75943ca7b0 | ||
|
|
fb6631aee1 | ||
|
|
d41bee158c | ||
|
|
110e55cb48 | ||
|
|
c494018074 | ||
|
|
4cdf63443d | ||
|
|
1e9995faba | ||
|
|
f6c401ecd2 | ||
|
|
c2f01bf10b | ||
|
|
be2fc2e9b6 | ||
|
|
aa05629375 | ||
|
|
35a7ff70fe | ||
|
|
747bfa71aa | ||
|
|
316a5a2fc3 | ||
|
|
0edcd2501d | ||
|
|
ac645e2cae | ||
|
|
a479a90fb2 | ||
|
|
3c9068f99a | ||
|
|
250aa73a56 | ||
|
|
e32eef2379 | ||
|
|
070ae53128 | ||
|
|
8c0618cfc5 | ||
|
|
54680238a0 | ||
|
|
200249a84f | ||
|
|
67850457ef | ||
|
|
9d67b0995f | ||
|
|
e9009b7f04 | ||
|
|
9fc97e45c9 | ||
|
|
1583fa85d9 | ||
|
|
95a12105cf | ||
|
|
2966c1e32c | ||
|
|
1a0424117a | ||
|
|
23f5babe33 | ||
|
|
27266f99a3 | ||
|
|
a9b02bc676 | ||
|
|
f016c67a72 | ||
|
|
6cc3398880 | ||
|
|
8cdb7c3032 | ||
|
|
3baeaece61 | ||
|
|
a5097dc69a | ||
|
|
9f25e5c9f6 | ||
|
|
bb6a0e3192 | ||
|
|
3331719407 | ||
|
|
5d6cee9d84 | ||
|
|
2cd5550dc7 | ||
|
|
d8ac69904d | ||
|
|
b0ded57294 | ||
|
|
77b05089c8 | ||
|
|
0211844e6a | ||
|
|
0e985f9bf1 | ||
|
|
860043fd7f | ||
|
|
589bfc0bc5 | ||
|
|
3b5a38e4a4 | ||
|
|
755f77ca45 | ||
|
|
9153a280f8 | ||
|
|
8ee6db7d9a | ||
|
|
5dffb920ff | ||
|
|
e0fadc43d9 | ||
|
|
d295fd4250 | ||
|
|
18260ce9be | ||
|
|
4165ad59b5 | ||
|
|
f183b86fa0 | ||
|
|
06e6721ed6 | ||
|
|
7412c5b6c9 | ||
|
|
33d136e702 | ||
|
|
6b3e15042f | ||
|
|
21d038d339 | ||
|
|
5d70eb3e35 | ||
|
|
d3b0330ed1 | ||
|
|
9921486fd5 | ||
|
|
d0dcae53a8 | ||
|
|
52e275ad2f | ||
|
|
954e1a53ee | ||
|
|
424a7eec82 | ||
|
|
09c5f809cb | ||
|
|
847055c758 | ||
|
|
aee834aeb6 | ||
|
|
494d0787e2 | ||
|
|
172b5af589 | ||
|
|
3f0c1a6cec | ||
|
|
4348b42844 | ||
|
|
82aefe1988 | ||
|
|
d535dd5322 | ||
|
|
336a2b1328 | ||
|
|
541de48e04 | ||
|
|
ab0f888d67 |
-80
@@ -1,80 +0,0 @@
|
||||
ARG NODE_BASE=node:22-bookworm-slim
|
||||
# The mesh-tools module: the two images every TypeScript module is COMPILED in and may RUN in. The
|
||||
# runtime itself ships as the node-tools module's bundle (node-tools/, novox/hq ADR 0175, to-be 38
|
||||
# WP3); these images are the toolchain for TypeScript bundles and the base a module's own service
|
||||
# may still be built on. They are no longer how tools reach a node.
|
||||
#
|
||||
# **They were the same image, and that was a mistake.** A module's recipe starts from this and
|
||||
# invokes the compiler out of it, so the compiler had to be here — and because the same image was
|
||||
# also what every module ran in, every running container on every machine carried a TypeScript
|
||||
# compiler it would never invoke. The answer is separate stages rather than one image bad at both
|
||||
# jobs. `build.artifacts` in module.json names the published stage each — `toolchain` and `runtime`.
|
||||
#
|
||||
# The `deps` stage is neither published nor named there: it is where the mesh's package registry is
|
||||
# reached, so it is where — and only where — the credential to reach it exists. The toolchain copies
|
||||
# resolved node_modules out of it, so the credential is in no image any machine ever holds
|
||||
# (novox/hq ADR 0076). This is the buildkit-secret's job done without buildkit, because a machine's
|
||||
# docker may carry no buildx.
|
||||
|
||||
# ---- deps: node_modules resolved from the mesh's registry, credential and all ----------------
|
||||
FROM ${NODE_BASE} AS deps
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY node-tools/package.json ./
|
||||
# The builder writes .npmrc into the build context; it authenticates to the mesh's package registry
|
||||
# for the @novox scope, which is where @novox/mesh-sdk resolves. This stage is not published, so the
|
||||
# credential travels no further than here. Development dependencies included: the compiler is one.
|
||||
COPY .npmrc ./.npmrc
|
||||
RUN npm install --no-audit --no-fund
|
||||
# **The SDK this image carries is the one the mesh last published** (novox/hq issue 212). The range in
|
||||
# package.json is resolved once and the layer above is cached, so a release reached no toolchain until
|
||||
# that file changed. The exact version arrives as a build argument from the SDK module's published
|
||||
# package (module.json `build.on`), so a new release is a new argument, this layer runs again — and
|
||||
# the planner orders this module after the SDK, so a release rebuilds the toolchain and, after it,
|
||||
# every bundle compiled in it (issue 211).
|
||||
ARG MESH_SDK=@novox/mesh-sdk@latest
|
||||
RUN npm install --no-audit --no-fund "${MESH_SDK}"
|
||||
|
||||
# ---- compiling: the runtime's own code built, WITHOUT the credential -------------------------
|
||||
FROM ${NODE_BASE} AS compiling
|
||||
WORKDIR /app
|
||||
COPY node-tools/package.json ./
|
||||
# The resolved libraries, but not the .npmrc that resolved them.
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
COPY node-tools/tsconfig.json ./
|
||||
COPY node-tools/src ./src
|
||||
RUN npm run build
|
||||
|
||||
# ---- what a running bundle needs, and nothing else -------------------------------------------
|
||||
# Its own stage so the toolchain image keeps its build tools while the runtime image does not.
|
||||
FROM compiling AS lean
|
||||
RUN npm prune --omit=dev
|
||||
|
||||
# ---- toolchain: what a TypeScript bundle is compiled in ---------------------------------------
|
||||
# The compiler, and esbuild (a development dependency) at /app/node_modules/esbuild: the builder
|
||||
# bundles every entrypoint and launcher into one file with it (novox/hq ADR 0193), so a bundle
|
||||
# carries what it imports and not this image's node_modules.
|
||||
# Beside the compiler, at /app/runtime, what every TypeScript bundle runs with: the production
|
||||
# dependencies the SDK and the runtime need, and a package.json saying the compiled files are ES
|
||||
# modules. The builder copies this directory whole into a compiled bundle (novox/hq ADR 0188 §5),
|
||||
# so a bundle unpacked on a machine starts — a `.js` without that package.json is read as
|
||||
# CommonJS, and an import of `nats` without node_modules beside it resolves to nothing.
|
||||
FROM compiling AS toolchain
|
||||
COPY --from=lean /app/node_modules /app/runtime/node_modules
|
||||
RUN printf '{"type":"module","private":true}\n' > /app/runtime/package.json
|
||||
# **git, because a pull request's merge check runs in this image too** (novox/hq ADR 0237): a repository's
|
||||
# own merge-check.sh that declares this toolchain runs here, and a suite that reads a repository's history
|
||||
# (the lab's last-run record) ran into "spawnSync git ENOENT". Nothing compiled here calls it.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ---- runtime: what a module's own service may run in ------------------------------------------
|
||||
FROM ${NODE_BASE} AS runtime
|
||||
WORKDIR /app
|
||||
COPY node-tools/package.json ./
|
||||
COPY --from=lean /app/node_modules ./node_modules
|
||||
COPY --from=compiling /app/dist ./dist
|
||||
ENTRYPOINT ["node", "dist/main.js"]
|
||||
@@ -1,13 +1,11 @@
|
||||
# mesh-tools
|
||||
|
||||
Two modules in one repository (novox/hq ADR 0069), one piece of software:
|
||||
One module, `node-tools`; the toolchain image module that stood beside it is retired (novox/hq ADR 0524 §5):
|
||||
|
||||
- **`node-tools`** (`node-tools/`) — the node's **tool runtime** as a module (ADR 0175, to-be 38
|
||||
WP3): one process per machine the host runs from this bundle, serving every assigned module's tools
|
||||
and every held seat's verbs on the bus, and answering MCP on the machine's loopback — the console
|
||||
(design 34). The code, its tests and the `mesh` client all live there.
|
||||
- **`mesh-tools`** (this directory) — the two images TypeScript bundles are compiled in and a module's
|
||||
own *service* may still run in. Built from the same code; no longer how tools reach a node.
|
||||
|
||||
The runtime:
|
||||
|
||||
@@ -25,8 +23,7 @@ marked executable — is **launched** rather than imported (novox/hq ADR 0188):
|
||||
as a child with its own environment and speaks MCP over stdio to it, `tools/list` once and
|
||||
`tools/call` per call. A tool it lists as `<seat>.<verb>` is the seat's implementation. A child that
|
||||
exits is named in the log and started again on its next call. So a tools bundle may be written in
|
||||
any language; the mesh's SDK for each is the stdio loop and nothing more (`node-tools/src/launch.ts`
|
||||
is the runtime's side of it).
|
||||
any language; the mesh's SDK for each is the stdio loop and nothing more (`node-tools/internal/launch` is the runtime's side of it).
|
||||
|
||||
Everything hard — dispatch, collection, duplicate-name safety — is the sdk's. This is the thin
|
||||
wrapper that binds the bus and loads the modules. Keeping the bus client here, out of the sdk, is
|
||||
@@ -47,10 +44,9 @@ MESH_OPERATOR_ACCOUNT whose machine this is, and MESH_OPERATOR_HOME where thei
|
||||
MESH_BROKER_URL a plain URL instead of the credential, for the bootstrap case
|
||||
```
|
||||
|
||||
`node dist/main.js`. On a node the controller composes the variables and the host supervises the
|
||||
`node dist/main.js`. On a node the mesh composes the variables and the node-engine supervises the
|
||||
process like any other host-side workload (novox/hq to-be 38). As `node-tools` the same process is
|
||||
the console: MCP on `127.0.0.1:4270` (or `MESH_CONSOLE_LISTEN`). The container (`Dockerfile`) is how
|
||||
a module's own *service* may still be built; it is no longer how tools reach a node.
|
||||
the console: MCP on `127.0.0.1:4270` (or `MESH_CONSOLE_LISTEN`).
|
||||
|
||||
## `mesh` — the tools for whoever is on a machine
|
||||
|
||||
@@ -69,14 +65,14 @@ answer took, its size in bytes, how many modules and tools it announced, whether
|
||||
fit the bus, when it was last heard — and who was expected and not heard).
|
||||
|
||||
`mesh_search` also finds a tool by the shell command it replaces (novox/hq ADR 0245): each seat verb and
|
||||
module tool says what it replaces in the controller's records (`replaces`, read from the controller's
|
||||
module tool says what it replaces in the indexer's records (`replaces`, read from its
|
||||
`tools` and `modules` answers), so `journalctl -u x`, `systemctl status x` or `docker ps` puts the verb
|
||||
for it first, and says the command it matched as `instead_of`. A word also matches through a short table
|
||||
of the words agents use for the mesh's (`logs` finds the journal). Best first: a replaced command, then
|
||||
words in a name or a replaced command, then words in a description; seats before modules on a tie.
|
||||
|
||||
Discovery asks the bus (ADR 0197): every runtime answers the NATS services protocol's `$SRV.PING` and
|
||||
`$SRV.INFO` with what it serves at that moment, and the console reads where the controller's records
|
||||
`$SRV.INFO` with what it serves at that moment, and the console reads where the assigner seat's records
|
||||
place each module. The console waits at least 750 ms, and up to 5 s for every runtime that answered
|
||||
PING to send what it serves — a large runtime's answer crossing the bus to a broker on another machine
|
||||
can arrive well after the rest. A runtime that said it is there and did not say what it serves, or
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
#!/bin/sh
|
||||
# mesh-check-toolchain: go
|
||||
#
|
||||
# The tool runtime's own check (novox/hq ADR 0237 as amended): the second layer of a pull request's merge
|
||||
# check, `mesh/repo-check`, run by the build seat in the mesh's Go toolchain — and by hand.
|
||||
#
|
||||
# The gate — every machine of the facts snapshot composed with this change — is the build seat's first
|
||||
# layer (`mesh/merge-gate`), run because the mesh's module graph builds node-tools and mesh-tools from
|
||||
# here. This is the code's own: node-tools (Go) formatted, vetted and its suite, every test on a bus of its
|
||||
# own at the release the mesh runs (internal/meshtest), packages in parallel, under the race detector when
|
||||
# the toolchain has a C compiler.
|
||||
#
|
||||
# **Said, never passed silently**: the runtime's and the console's tests launch TypeScript and Python
|
||||
# bundles that import the mesh's own package (@novox/mesh-sdk), which comes from the package registry — and
|
||||
# a check of code nobody has approved is given no credential for it. Where node and that package are not
|
||||
# present those two packages are not tested here, and the TypeScript in node-tools/src neither.
|
||||
set -eu
|
||||
cd node-tools
|
||||
|
||||
unformatted=$(gofmt -l cmd internal)
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "not gofmt'd:"
|
||||
echo "$unformatted"
|
||||
exit 1
|
||||
fi
|
||||
CGO_ENABLED=0 go vet ./...
|
||||
|
||||
packages=$(go list ./...)
|
||||
bundleless=""
|
||||
if ! command -v node >/dev/null 2>&1 || [ ! -d node_modules/@novox/mesh-sdk ]; then
|
||||
echo "NOT TESTED HERE: internal/runtime and internal/console launch bundles that need node and @novox/mesh-sdk"
|
||||
packages=$(printf '%s\n' "$packages" | grep -v -e '/internal/runtime$' -e '/internal/console$')
|
||||
# What of the console launches no bundle runs anyway: how an address resolves (novox/hq issue 287),
|
||||
# and how a search ranks (novox/hq ADR 0245).
|
||||
bundleless='^(TestASeatAndAModuleOfOneNameAreEachReached|TestSearch.*|TestASearchByCommandSaysWhatItReplaces)$'
|
||||
# And what of the runtime launches no bundle: how a bundle's seat traffic reaches the bus, what its
|
||||
# environment holds, what a runtime on a module's own account serves (novox/hq ADR 0259), and that a write
|
||||
# by compare-and-set loses no update under contention (novox/hq research 042).
|
||||
runtimeless='^(TestARuntimeOnAModulesOwnAccountServesThatModuleAlone|TestNoBusWordReachesABundle|TestAToolCallNeverReachesASeatsEventAcceptOrProof|TestABundlesSeatTrafficGoesThroughItsModulesBus|TestALostCompareAndSetCarriesItsCode|TestTwoWritersIncrementingOneCounterOnTheBusLoseNoUpdate|TestAStaleWriteIsRefusedAndWritesNothing)$'
|
||||
fi
|
||||
if command -v gcc >/dev/null 2>&1; then
|
||||
CGO_ENABLED=1 go test -race -count=1 $packages
|
||||
else
|
||||
echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector"
|
||||
CGO_ENABLED=0 go test -count=1 $packages
|
||||
fi
|
||||
if [ -n "$bundleless" ]; then
|
||||
CGO_ENABLED=0 go test -count=1 -run "$bundleless" ./internal/console
|
||||
fi
|
||||
if [ -n "${runtimeless:-}" ]; then
|
||||
CGO_ENABLED=0 go test -count=1 -run "$runtimeless" ./internal/runtime
|
||||
fi
|
||||
echo "NOT TESTED HERE: node-tools/src (TypeScript) needs @novox/mesh-sdk from the package registry"
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"module": "mesh-tools",
|
||||
"version": "1",
|
||||
"slug": "tools",
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "build",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile",
|
||||
"target": "toolchain"
|
||||
},
|
||||
{
|
||||
"name": "runtime",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile",
|
||||
"target": "runtime"
|
||||
}
|
||||
],
|
||||
"on": [
|
||||
{
|
||||
"arg": "NODE_BASE",
|
||||
"image": "node@sha256:48e4b67d85f87bd551df43704e24d252f56cc5f8e9718841aace50f19948f0f9"
|
||||
},
|
||||
{
|
||||
"arg": "MESH_SDK",
|
||||
"module": "mesh-sdk",
|
||||
"artifact": "lib"
|
||||
}
|
||||
]
|
||||
},
|
||||
"resources": []
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
The node's tool runtime as a module (novox/hq ADR 0175, to-be 38 WP3). Assigned to a machine, it is
|
||||
one process the host runs from this bundle, as the operator's account: it serves every assigned
|
||||
module's tools and every held seat's verbs on the bus, and answers MCP on the machine's loopback —
|
||||
the console (design 34). The controller composes the process (which bundles to load, where the
|
||||
the console (design 34). The mesh composes the process (which bundles to load, where the
|
||||
credential is, whose machine it is); this manifest says only what the machine must have for it: the
|
||||
interpreter, a place for the credential, the loopback port, and leave to call every tool.
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
// wherever MESH_CONSOLE_LISTEN says — it is also the console, MCP over HTTP on loopback.
|
||||
//
|
||||
// MESH_BROKER_FILE the credential the mesh sealed to this machine for the runtime
|
||||
// MESH_LEDGERS_FILE its own secret ledgers, each module's ledger credential; beside the broker file when unset
|
||||
// MESH_TOOL_MODULES <module>=<entrypoint>,… the bundles to launch
|
||||
// MESH_TOOL_ENV {"<module>": {"<word>": "<value>"}} what each is given (ADR 0192)
|
||||
// MESH_OPERATOR_ACCOUNT whose machine this is, and MESH_OPERATOR_HOME their home
|
||||
@@ -12,7 +13,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand/v2"
|
||||
@@ -24,6 +28,9 @@ import (
|
||||
"github.com/novox/mesh-tools/node-tools/internal/alive"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/console"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/firstview"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/provide"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
@@ -40,8 +47,11 @@ func main() {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %q is not a mode; this runtime serves (novox/hq ADR 0193)\n", os.Args[1])
|
||||
os.Exit(2)
|
||||
}
|
||||
cred := credential()
|
||||
conn := connectPatiently(cred)
|
||||
until := time.Now().Add(startWait)
|
||||
cred := credential(until)
|
||||
conn := connectPatiently(cred, until)
|
||||
// And the seat list it describes the seats by, read through the seat mesh-catalog (novox/hq ADR 0636 §4).
|
||||
_ = startSeatList(conn)
|
||||
|
||||
served, err := runtime.ServedModulesFrom(os.Getenv(runtime.ToolModules), cred.Module)
|
||||
if err != nil {
|
||||
@@ -51,7 +61,15 @@ func main() {
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
}
|
||||
stop, err := runtime.Run(conn, served, envs, log.Printf)
|
||||
// Its own secret ledgers: each module's ledger connection, the only one its history is published on (novox/hq ADR
|
||||
// 0597 §9). Followed, so a credential minted or retired is taken with no restart.
|
||||
var opts []runtime.Option
|
||||
if file := runtime.LedgersFileOf(os.Getenv("MESH_BROKER_FILE")); file != "" {
|
||||
ledgers := runtime.FollowLedgerCredentials(file, bus.Dial, log.Printf)
|
||||
defer ledgers.Stop()
|
||||
opts = append(opts, runtime.WithLedgerWriters(ledgers.Writer))
|
||||
}
|
||||
stop, err := runtime.Run(conn, served, envs, log.Printf, opts...)
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
}
|
||||
@@ -62,18 +80,26 @@ func main() {
|
||||
stopSaying = alive.Keep(conn, cred.Node, alive.Every, log.Printf)
|
||||
}
|
||||
|
||||
// As the machine's runtime it is also the provider of `mcp-endpoint`: it answers its consumers through the
|
||||
// SDK's helper wherever it runs, whether or not the module is handed (novox/hq ADR 0349 clause 12, issue 791).
|
||||
stopProviding := provide.Start(conn, cred.Module, cred.Node, time.Minute, log.Printf)
|
||||
|
||||
listen := os.Getenv("MESH_CONSOLE_LISTEN")
|
||||
if listen == "" && cred.Module == runtimeModule {
|
||||
listen = consoleListen
|
||||
}
|
||||
var up *console.Listening
|
||||
if listen != "" {
|
||||
// The indexer's and the assigner's states the console indexes the mesh by come to it as first views, asked of the owners'
|
||||
// own tools and followed on their views families, never read from their state (novox/hq ADR 0615 §1).
|
||||
source, stopViews := firstview.Start(context.Background(), conn, log.Printf)
|
||||
defer stopViews()
|
||||
node := cred.Node
|
||||
if node == "" {
|
||||
node = "?"
|
||||
}
|
||||
who := node + "." + cred.Module
|
||||
up, err = console.Serve(console.NewSurface(conn, who), listen)
|
||||
up, err = console.Serve(console.NewSurface(conn, source, who), listen)
|
||||
if err != nil {
|
||||
log.Fatalf("node-tools: %v", err)
|
||||
}
|
||||
@@ -83,7 +109,11 @@ func main() {
|
||||
signals := make(chan os.Signal, 1)
|
||||
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
|
||||
<-signals
|
||||
// First of all: the service manager signalled every bundle with this process, and none is started again
|
||||
// (issue 1126).
|
||||
launch.ShutDown()
|
||||
stopSaying()
|
||||
stopProviding()
|
||||
stop()
|
||||
if up != nil {
|
||||
_ = up.Close()
|
||||
@@ -91,48 +121,149 @@ func main() {
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
// credential reads the credential the mesh delivered; a missing or unreadable one is a fault of
|
||||
// configuration, said and final.
|
||||
func credential() bus.Credential {
|
||||
// startWait is how long a runtime waits, from its start, for its broker credential to be on the machine and for the bus
|
||||
// to accept the account it names (issue 1496): the node-engine starts a module's runtime before the provisioner's client
|
||||
// fills the credential, and the bus carries a new account only after it reloads its users. Past it the runtime gives up,
|
||||
// loudly; inside it the wait is said once.
|
||||
var startWait = 5 * time.Minute
|
||||
|
||||
// say is a line in the runtime's journal.
|
||||
func say(format string, args ...any) { fmt.Fprintf(os.Stderr, "node-tools: "+format+"\n", args...) }
|
||||
|
||||
// credential reads the credential the mesh delivered, waiting for it within the start's bound (issue 1496); one that can
|
||||
// never be a credential (no url) is a fault of configuration, said and final.
|
||||
func credential(until time.Time) bus.Credential {
|
||||
file := os.Getenv("MESH_BROKER_FILE")
|
||||
if file == "" {
|
||||
if url := os.Getenv("MESH_BROKER_URL"); url != "" {
|
||||
return bus.Credential{URL: url, Module: runtimeModule}
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "node-tools: set MESH_BROKER_FILE (a sealed credential) or MESH_BROKER_URL — there is no broker to reach")
|
||||
say("set MESH_BROKER_FILE (a sealed credential) or MESH_BROKER_URL — there is no broker to reach")
|
||||
os.Exit(1)
|
||||
}
|
||||
raw, err := os.ReadFile(file)
|
||||
cred, err := awaitCredential(file, until, say)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: cannot read the broker credential at %s: %v\n", file, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
var cred bus.Credential
|
||||
if err := json.Unmarshal(raw, &cred); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: cannot read the broker credential at %s: %v\n", file, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if cred.URL == "" {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %s carries no url — it is not a broker credential\n", file)
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
// connectPatiently retries while the bus is merely not reachable yet — the normal case at startup —
|
||||
// and gives up at once on what waiting cannot fix (issue 058).
|
||||
func connectPatiently(cred bus.Credential) *bus.Conn {
|
||||
for delay := 2 * time.Second; ; delay = min(delay*2, 30*time.Second) {
|
||||
conn, err := bus.Connect(cred)
|
||||
if err == nil {
|
||||
return conn
|
||||
// errNotCredential is a file that is there and readable and still no broker credential: waiting will not fix it.
|
||||
var errNotCredential = errors.New("it carries no url — it is not a broker credential")
|
||||
|
||||
// readCredential reads the credential at file. A file not there yet, empty, or not yet whole JSON is one the node-engine
|
||||
// has still to write: the error says so and waiting may fix it. errNotCredential is final.
|
||||
func readCredential(file string) (bus.Credential, error) {
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return bus.Credential{}, err
|
||||
}
|
||||
if len(bytes.TrimSpace(raw)) == 0 {
|
||||
return bus.Credential{}, errors.New("the file is empty")
|
||||
}
|
||||
var cred bus.Credential
|
||||
if err := json.Unmarshal(raw, &cred); err != nil {
|
||||
return bus.Credential{}, err
|
||||
}
|
||||
if cred.URL == "" {
|
||||
return bus.Credential{}, errNotCredential
|
||||
}
|
||||
return cred, nil
|
||||
}
|
||||
|
||||
// awaitCredential is the credential at file, read at once and, while it is not there or not whole, again each time its
|
||||
// directory changes (runtime.WatchFile, inotify), until the time until. The wait is said once; at until it ends with an
|
||||
// error naming the file, how long it waited and what the last read found. A file that is not a credential ends it at once.
|
||||
func awaitCredential(file string, until time.Time, say func(string, ...any)) (bus.Credential, error) {
|
||||
cred, err := readCredential(file)
|
||||
if err == nil {
|
||||
return cred, nil
|
||||
}
|
||||
if errors.Is(err, errNotCredential) {
|
||||
return bus.Credential{}, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
began := time.Now()
|
||||
say("waits for its broker credential at %s (%v): the node-engine writes it once the provisioner's client fills it; "+
|
||||
"it gives up at %s", file, err, until.Format("15:04:05"))
|
||||
changed := make(chan struct{}, 1)
|
||||
stop, werr := runtime.WatchFile(file, func() {
|
||||
select {
|
||||
case changed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
if fatal := bus.Fatal(err); fatal != "" {
|
||||
fmt.Fprintf(os.Stderr, "node-tools: %s — waiting will not fix this; giving up\n", fatal)
|
||||
os.Exit(1)
|
||||
}, func(format string, args ...any) { say(format, args...) },
|
||||
fmt.Sprintf("the credential is read again every %s instead", rereadEvery))
|
||||
if werr != nil {
|
||||
say("cannot watch %s (%v): it is read again every %s instead", file, werr, rereadEvery)
|
||||
} else {
|
||||
defer stop()
|
||||
}
|
||||
for {
|
||||
left := time.Until(until)
|
||||
if left <= 0 {
|
||||
return bus.Credential{}, fmt.Errorf("cannot read the broker credential at %s: waited %s for it, and the last read "+
|
||||
"found: %v", file, time.Since(began).Round(time.Second), err)
|
||||
}
|
||||
// The watch wakes the read; the clock is only the backstop for a watch that could not be made or missed a change.
|
||||
select {
|
||||
case <-changed:
|
||||
case <-time.After(min(left, rereadEvery)):
|
||||
}
|
||||
if cred, err = readCredential(file); err == nil {
|
||||
say("read its broker credential at %s after %s", file, time.Since(began).Round(time.Second))
|
||||
return cred, nil
|
||||
}
|
||||
if errors.Is(err, errNotCredential) {
|
||||
return bus.Credential{}, fmt.Errorf("%s: %w", file, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rereadEvery is the backstop of the wait for the credential, beside the watch of its directory.
|
||||
var rereadEvery = 30 * time.Second
|
||||
|
||||
// connectPatiently retries while the bus is merely not reachable yet — the normal case at startup — and, until the start's
|
||||
// bound, while the bus refuses the account: the bus carries a new account only after it reloads its users, which may come
|
||||
// after the credential is on the machine (issue 1496, issue 1423). What waiting cannot fix (a pin that does not match, an
|
||||
// address that is no URL) gives up at once (issue 058), and so does a refusal past the bound.
|
||||
func connectPatiently(cred bus.Credential, until time.Time) *bus.Conn {
|
||||
conn, err := connectWithin(cred, until, bus.Connect, time.Sleep, say)
|
||||
if err != nil {
|
||||
say("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
// connectWithin is connectPatiently with its dial, its sleep and its words given, as a test gives them.
|
||||
func connectWithin(cred bus.Credential, until time.Time, dial func(bus.Credential) (*bus.Conn, error), sleep func(time.Duration),
|
||||
say func(string, ...any)) (*bus.Conn, error) {
|
||||
refusedSaid := false
|
||||
for delay := 2 * time.Second; ; delay = min(delay*2, 30*time.Second) {
|
||||
conn, err := dial(cred)
|
||||
if err == nil {
|
||||
if refusedSaid {
|
||||
say("the bus accepts the account now")
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
wait := delay + time.Duration(rand.IntN(1000))*time.Millisecond
|
||||
fmt.Fprintf(os.Stderr, "node-tools: the broker is not reachable yet (%v); retrying in %ds\n", err, int(wait.Round(time.Second)/time.Second))
|
||||
time.Sleep(wait)
|
||||
switch fatal := bus.Fatal(err); {
|
||||
case bus.Refused(err) && time.Now().Add(wait).Before(until):
|
||||
if !refusedSaid {
|
||||
say("the bus refuses this account (%v): it may not carry it yet, since it loads a new account by a reload; "+
|
||||
"retrying until %s", err, until.Format("15:04:05"))
|
||||
refusedSaid = true
|
||||
}
|
||||
case bus.Refused(err) && refusedSaid:
|
||||
return nil, fmt.Errorf("the bus still refuses this account (%v) at the end of the start's bound, %s; giving up", err,
|
||||
until.Format("15:04:05"))
|
||||
case fatal != "":
|
||||
return nil, fmt.Errorf("%s — waiting will not fix this; giving up", fatal)
|
||||
default:
|
||||
say("the broker is not reachable yet (%v); retrying in %ds", err, int(wait.Round(time.Second)/time.Second))
|
||||
}
|
||||
sleep(wait)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/runtime"
|
||||
)
|
||||
|
||||
// A runtime started before its credential is on the machine waits for it, woken by the watch of its directory, and gives
|
||||
// up loudly at its bound; a file that is no credential ends the wait at once (issue 1496: on ace mesh-build-node's and
|
||||
// mesh-build-image's runtimes exited twice on "cannot read the broker credential ... no such file").
|
||||
func TestAwaitCredential(t *testing.T) {
|
||||
const good = `{"url":"tls://bus.example:4222","module":"m"}`
|
||||
rereadEvery = time.Hour // only the watch can wake the read in time
|
||||
settle := runtime.Settle
|
||||
runtime.Settle = 10 * time.Millisecond // a unit test stays well under a second
|
||||
t.Cleanup(func() { rereadEvery, runtime.Settle = 30*time.Second, settle })
|
||||
place := func(path, body string) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
before string // written before the wait, "" none
|
||||
later string // written 20ms into the wait, "" none
|
||||
deeper bool // the file's directory is not there at the start
|
||||
bound time.Duration
|
||||
want string // "" a credential; else a word of the error
|
||||
said int
|
||||
}{
|
||||
{name: "there at the start", before: good, bound: time.Second, said: 0},
|
||||
{name: "written during the wait", later: good, bound: 5 * time.Second, said: 2},
|
||||
{name: "empty, then filled", before: " ", later: good, bound: 5 * time.Second, said: 2},
|
||||
{name: "its directory made during the wait", later: good, deeper: true, bound: 5 * time.Second, said: 2},
|
||||
{name: "never written", bound: 100 * time.Millisecond, want: "waited", said: 1},
|
||||
{name: "no url", before: `{"module":"m"}`, bound: 5 * time.Second, want: "not a broker credential", said: 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
file := filepath.Join(dir, "broker")
|
||||
if c.deeper {
|
||||
file = filepath.Join(dir, "var", "lib", "m", "broker")
|
||||
}
|
||||
if c.before != "" {
|
||||
place(file, c.before)
|
||||
}
|
||||
if c.later != "" {
|
||||
go func() { time.Sleep(20 * time.Millisecond); place(file, c.later) }()
|
||||
}
|
||||
var said []string
|
||||
began := time.Now()
|
||||
cred, err := awaitCredential(file, time.Now().Add(c.bound), func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
switch {
|
||||
case c.want == "" && (err != nil || cred.URL == ""):
|
||||
t.Fatalf("no credential: %v", err)
|
||||
case c.want != "" && (err == nil || !strings.Contains(err.Error(), c.want)):
|
||||
t.Fatalf("error %v, want one saying %q", err, c.want)
|
||||
case c.want == "" && c.later != "" && time.Since(began) > 3*time.Second:
|
||||
t.Fatalf("read after %s: the watch did not wake it", time.Since(began))
|
||||
}
|
||||
if len(said) != c.said {
|
||||
t.Fatalf("said %d lines, want %d: %q", len(said), c.said, said)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A bus that refuses the account at a start is asked again until the start's bound, said once; a refusal past it, a pin
|
||||
// that does not match and an address that is no URL give up (issue 1496, issue 058).
|
||||
func TestConnectWithin(t *testing.T) {
|
||||
refused := errors.New("nats: Authorization Violation")
|
||||
unreachable := errors.New("nats: no servers available for connection")
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
errs []error // what each dial answers, then a connection
|
||||
bound time.Duration
|
||||
fails string
|
||||
saidAt int // lines said
|
||||
}{
|
||||
{name: "refused twice, then accepted", errs: []error{refused, refused}, bound: time.Hour, saidAt: 2},
|
||||
{name: "unreachable, then accepted", errs: []error{unreachable}, bound: time.Hour, saidAt: 1},
|
||||
{name: "refused past the bound", errs: []error{refused, refused}, bound: 0, fails: "refused this account"},
|
||||
{name: "refused, then the bound ends", errs: []error{refused, refused, refused, refused, refused, refused}, bound: 5 * time.Second,
|
||||
fails: "still refuses"},
|
||||
{name: "pin", errs: []error{fmt.Errorf("tls: %w", bus.ErrPin)}, bound: time.Hour, fails: "pin"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
dials := 0
|
||||
dial := func(bus.Credential) (*bus.Conn, error) {
|
||||
dials++
|
||||
if dials <= len(c.errs) {
|
||||
return nil, c.errs[dials-1]
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
until := time.Now().Add(c.bound)
|
||||
var said []string
|
||||
_, err := connectWithin(bus.Credential{URL: "tls://bus.example:4222"}, until, dial, func(time.Duration) {},
|
||||
func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
switch {
|
||||
case c.fails == "" && err != nil:
|
||||
t.Fatalf("gave up: %v", err)
|
||||
case c.fails != "" && (err == nil || !strings.Contains(err.Error(), c.fails)):
|
||||
t.Fatalf("error %v, want one saying %q", err, c.fails)
|
||||
case c.fails == "" && len(said) != c.saidAt:
|
||||
t.Fatalf("said %d lines, want %d: %q", len(said), c.saidAt, said)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// The tool runner reads the seat list as every part does (novox/hq ADR 0635 §1, ADR 0636): through the seat mesh-catalog,
|
||||
// from the bootstrapped definitions until it has been read, and following the seat's events. What it holds is given to
|
||||
// catalogue at every change; the console describes the seats from it, so the descriptions are the mesh's own and not a
|
||||
// list compiled into the runner.
|
||||
|
||||
// busOfTheRunner is the reader's bus over the runner's connection: a verb of the seat asked on its subject, an event heard
|
||||
// on the seat's event subject.
|
||||
type busOfTheRunner struct{ conn *bus.Conn }
|
||||
|
||||
func (b busOfTheRunner) Ask(key string, body any) (json.RawMessage, error) {
|
||||
seat, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".")
|
||||
answered, err := b.conn.Ask(key, body, "mesh.seat."+seat+".tool."+verb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return answered.Result, nil
|
||||
}
|
||||
|
||||
func (b busOfTheRunner) Subscribe(pattern string, handle func(string, json.RawMessage) error) error {
|
||||
seat, event, _ := strings.Cut(pattern, ".")
|
||||
_, err := b.conn.Raw(seatdef.EventSubject(seat, event), func(_ string, data []byte) []byte {
|
||||
_ = handle(pattern, data)
|
||||
return nil // an event is answered by nobody
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// seatListWithin is how long the runner waits for the first read of the seat list before it serves; the read goes on
|
||||
// after it, and the descriptions are the bootstrapped ones until it ends (ADR 0636 §1).
|
||||
const seatListWithin = 5 * time.Second
|
||||
|
||||
// startSeatList starts the runner's reader and gives catalogue its bootstrapped seats at once, then waits for the first
|
||||
// read to end (the reader's Ready) for the bound, so the runner's first descriptions are of the list in the common case.
|
||||
func startSeatList(conn *bus.Conn) *seatlist.Reader {
|
||||
timer := time.NewTimer(seatListWithin)
|
||||
defer timer.Stop()
|
||||
return startSeatListOn(busOfTheRunner{conn}, timer.C, log.Printf)
|
||||
}
|
||||
|
||||
// startSeatListOn is startSeatList over any bus, with the bound as a channel the caller gives.
|
||||
func startSeatListOn(b seatlist.Bus, bound <-chan time.Time, logf func(string, ...any)) *seatlist.Reader {
|
||||
reader, err := seatlist.Start(seatlist.Config{Bus: b, Bootstrap: seatlist.Bootstrapped(),
|
||||
OnChange: func(seats []catalogue.Seat) { catalogue.UseSeats(seats) }})
|
||||
if err != nil {
|
||||
logf("node-tools: %v", fmt.Errorf("the tool runner cannot follow the seat list: %w", err))
|
||||
}
|
||||
if reader == nil {
|
||||
return nil
|
||||
}
|
||||
catalogue.UseSeats(reader.Seats())
|
||||
select {
|
||||
case <-reader.Ready():
|
||||
catalogue.UseSeats(reader.Seats())
|
||||
case <-bound:
|
||||
logf("node-tools: the seat list is not read yet; the runner serves with the bootstrapped seats and reads on")
|
||||
}
|
||||
return reader
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// busOfAStandIn is a seat mesh-catalog that answers after release, or never (a refusal).
|
||||
type busOfAStandIn struct {
|
||||
release chan struct{}
|
||||
asked atomic.Int32
|
||||
}
|
||||
|
||||
func (b *busOfAStandIn) Ask(string, any) (json.RawMessage, error) {
|
||||
b.asked.Add(1)
|
||||
<-b.release
|
||||
return nil, errors.New("the seat does not answer")
|
||||
}
|
||||
|
||||
func (b *busOfAStandIn) Subscribe(string, func(string, json.RawMessage) error) error { return nil }
|
||||
|
||||
// ADR 0636 §1: the runner waits for the first read of the seat list on the reader's Ready, but only to a bound that the
|
||||
// test gives as a channel; past it, it serves with the bootstrapped seats and says so, and nothing sleeps.
|
||||
func TestTheRunnerWaitsForTheFirstReadOfTheSeatListOnlyToItsBound(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
release bool // the seat ends its first read (here by refusing) before the bound
|
||||
wantSaid string
|
||||
wantReady bool
|
||||
}{
|
||||
{"the first read ends before the bound", true, "", true},
|
||||
{"the bound passes with the seat silent", false, "the seat list is not read yet", false},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
b := &busOfAStandIn{release: make(chan struct{})}
|
||||
bound := make(chan time.Time, 1)
|
||||
var said []string
|
||||
if row.release {
|
||||
close(b.release)
|
||||
} else {
|
||||
bound <- time.Time{}
|
||||
}
|
||||
r := startSeatListOn(b, bound, func(f string, a ...any) { said = append(said, f) })
|
||||
if r == nil {
|
||||
t.Fatal("no reader")
|
||||
}
|
||||
ended := false
|
||||
select {
|
||||
case <-r.Ready():
|
||||
ended = true
|
||||
default:
|
||||
}
|
||||
if ended != row.wantReady {
|
||||
t.Errorf("first read ended = %v, want %v", ended, row.wantReady)
|
||||
}
|
||||
if got := strings.Join(said, ";"); !strings.Contains(got, row.wantSaid) || (row.wantSaid == "" && got != "") {
|
||||
t.Errorf("said %q, want %q", got, row.wantSaid)
|
||||
}
|
||||
if !row.release {
|
||||
close(b.release)
|
||||
<-r.Ready()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+13
-6
@@ -3,20 +3,27 @@ module github.com/novox/mesh-tools/node-tools
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/sys v0.48.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/klauspost/compress v1.20.1 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.2 // indirect
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
go.uber.org/automaxprocs v1.6.0 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
golang.org/x/time v0.16.0 // indirect
|
||||
)
|
||||
|
||||
+8
-10
@@ -1,13 +1,13 @@
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
@@ -16,12 +16,10 @@ github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
|
||||
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// **A machine heard and its runtime gone is a machine nobody can ask anything.** The node-engine says
|
||||
// every minute that the machine is there; nothing said whether the runtime serving every module's
|
||||
// tools and every held seat's verbs was. Now it says so too, on its own subject, every interval, with
|
||||
// the interval — the controller's watchdog raises `tools-silent` after three missed — and its build.
|
||||
// the interval — the mesh's watchdog raises `tools-silent` after three missed — and its build.
|
||||
// Core NATS, kept by nobody, as the host's heartbeat: a lost one is the next one.
|
||||
package alive
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -35,6 +37,17 @@ var Window = 750 * time.Millisecond
|
||||
// serves is waited for until it arrives or this passes, and a service that never sends it is named.
|
||||
var Patience = 5 * time.Second
|
||||
|
||||
// Digest is the PING answer's metadata key for a digest of what the instance serves now: a console
|
||||
// that holds the answer of that digest does not ask for it again (novox/hq issue 554). Every round
|
||||
// pulled every runtime's whole announcement — some 880 kB, every five seconds while a session works —
|
||||
// to the laptop, and the hundred-byte PING answers queued behind it.
|
||||
const Digest = "digest"
|
||||
|
||||
// Capabilities is the service metadata key a runtime states what it can do under, a comma-separated list of capabilities
|
||||
// (runtime.CallCapabilities): its machine's node-engine asks the runtime's PING and takes them into the node's record, so a
|
||||
// module needing one is placed only where the runtime states it.
|
||||
const Capabilities = "capabilities"
|
||||
|
||||
// Shortened is the service metadata key a runtime sets when its announcement was cut to fit the bus.
|
||||
const Shortened = "shortened"
|
||||
|
||||
@@ -157,7 +170,14 @@ func Serve(conn *bus.Conn, s Service, current func() []Endpoint) (func(), error)
|
||||
var v any
|
||||
switch parts[1] {
|
||||
case "PING":
|
||||
v = micro.Ping{ServiceIdentity: s.identity(), Type: micro.PingResponseType}
|
||||
id := s.identity()
|
||||
md := map[string]string{}
|
||||
for k, v := range id.Metadata {
|
||||
md[k] = v
|
||||
}
|
||||
md[Digest] = DigestOf(Info(s, current()))
|
||||
id.Metadata = md
|
||||
v = micro.Ping{ServiceIdentity: id, Type: micro.PingResponseType}
|
||||
case "INFO":
|
||||
v = Info(s, current())
|
||||
case "STATS":
|
||||
@@ -205,6 +225,16 @@ func Serve(conn *bus.Conn, s Service, current func() []Endpoint) (func(), error)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DigestOf is a digest of an info_response: equal for equal announcements, whatever the moment.
|
||||
func DigestOf(i micro.Info) string {
|
||||
body, err := json.Marshal(i)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return hex.EncodeToString(sum[:12])
|
||||
}
|
||||
|
||||
// shorten is an info_response cut to fit the bus: descriptions to their first line, then none, the
|
||||
// service's metadata saying which (so discovery can tell a person what it was given).
|
||||
func shorten(logf func(string, ...any), info micro.Info, body []byte, limit int64) []byte {
|
||||
@@ -244,6 +274,11 @@ type Heard struct {
|
||||
Took time.Duration
|
||||
Bytes int
|
||||
Shortened string // why its descriptions are not whole, or ""
|
||||
// Digest is the digest its PING gave for this answer, or "" when it gave none.
|
||||
Digest string
|
||||
// Reused is true when the instance's PING said it serves what the console already held, so its
|
||||
// announcement was not asked for again; Took is then how long its PING took.
|
||||
Reused bool
|
||||
}
|
||||
|
||||
// Machine is the machine an instance runs on: its metadata's, or the one its endpoints name.
|
||||
@@ -273,12 +308,24 @@ type Discovery struct {
|
||||
Waited time.Duration
|
||||
Heard []Heard
|
||||
Silent []Instance
|
||||
// Pinged is how many PING answers arrived.
|
||||
Pinged int
|
||||
}
|
||||
|
||||
// Gather asks every service on the bus who it is and what it serves, and waits at least the window
|
||||
// and at most the patience: until every instance that answered PING has answered INFO. An answer
|
||||
// Empty is a round in which nothing arrived at all, not even a PING: no answer about the mesh, but a
|
||||
// round that failed (novox/hq issue 554).
|
||||
func (d Discovery) Empty() bool { return d.Pinged == 0 && len(d.Heard) == 0 }
|
||||
|
||||
// Known is the announcement a console already holds for an instance at this digest, if any.
|
||||
type Known func(name, id, digest string) (Heard, bool)
|
||||
|
||||
// Gather asks every service on the bus who it is (`$SRV.PING`), and each instance that answers what it
|
||||
// serves (`$SRV.INFO.<name>.<id>`) unless its PING's digest is one `known` holds; it waits at least the
|
||||
// window and at most the patience: until at least one instance answered PING and every instance that
|
||||
// did has said what it serves. A round in which no PING arrives within the window is not taken as the
|
||||
// mesh's answer that nothing is there (novox/hq issue 554): it waits on, up to the patience. An answer
|
||||
// that is not the protocol's is skipped.
|
||||
func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
func Gather(conn *bus.Conn, known Known) (Discovery, error) {
|
||||
start := time.Now()
|
||||
d := Discovery{At: start.UTC()}
|
||||
pings, stopPings, err := conn.Collect("$SRV.PING", nil)
|
||||
@@ -286,7 +333,7 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
return d, err
|
||||
}
|
||||
defer stopPings()
|
||||
infos, stopInfos, err := conn.Collect("$SRV.INFO", nil)
|
||||
infos, askInfo, stopInfos, err := conn.Collector("$SRV.INFO")
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
@@ -296,8 +343,14 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
pinged := map[key]int{}
|
||||
answered := map[key]int{}
|
||||
where := map[key]string{}
|
||||
digest := map[key]string{}
|
||||
asked := map[key]bool{}
|
||||
reused := map[key]int{} // index in d.Heard of an answer held already
|
||||
var order []key
|
||||
complete := func() bool {
|
||||
if len(pinged) == 0 {
|
||||
return false // nobody said they are there: not an answer, a round that has not arrived
|
||||
}
|
||||
for k, n := range pinged {
|
||||
if answered[k] < n {
|
||||
return false
|
||||
@@ -305,6 +358,15 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
}
|
||||
return true
|
||||
}
|
||||
ask := func(k key) {
|
||||
if asked[k] {
|
||||
return
|
||||
}
|
||||
asked[k] = true
|
||||
if err := askInfo("$SRV.INFO."+k.name+"."+k.id, nil); err != nil {
|
||||
conn.Logf("[mesh-tools] discovery could not ask %s/%s what it serves: %v", k.name, k.id, err)
|
||||
}
|
||||
}
|
||||
least := time.NewTimer(Window)
|
||||
defer least.Stop()
|
||||
most := time.NewTimer(Patience)
|
||||
@@ -317,21 +379,49 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
if json.Unmarshal(a.Data, &p) != nil || p.Type != micro.PingResponseType {
|
||||
continue
|
||||
}
|
||||
d.Pinged++
|
||||
k := key{p.Name, p.ID}
|
||||
if _, seen := pinged[k]; !seen {
|
||||
_, seen := pinged[k]
|
||||
if !seen {
|
||||
order = append(order, k)
|
||||
}
|
||||
pinged[k]++
|
||||
if where[k] == "" {
|
||||
where[k] = p.Metadata["node"]
|
||||
}
|
||||
if seen {
|
||||
// Two instances under one name and id: what each serves is asked of both, never assumed.
|
||||
if i, ok := reused[k]; ok {
|
||||
d.Heard = append(d.Heard[:i], d.Heard[i+1:]...)
|
||||
for kk, j := range reused {
|
||||
if j > i {
|
||||
reused[kk] = j - 1
|
||||
}
|
||||
}
|
||||
delete(reused, k)
|
||||
answered[k]--
|
||||
}
|
||||
ask(k)
|
||||
continue
|
||||
}
|
||||
digest[k] = p.Metadata[Digest]
|
||||
if h, ok := heldAt(known, k.name, k.id, digest[k]); ok {
|
||||
h.Took, h.Reused, h.Digest = a.At.Sub(start), true, digest[k]
|
||||
reused[k] = len(d.Heard)
|
||||
d.Heard = append(d.Heard, h)
|
||||
answered[k]++
|
||||
break
|
||||
}
|
||||
ask(k)
|
||||
case a := <-infos:
|
||||
var i micro.Info
|
||||
if json.Unmarshal(a.Data, &i) != nil || i.Type != micro.InfoResponseType {
|
||||
continue
|
||||
}
|
||||
answered[key{i.Name, i.ID}]++
|
||||
d.Heard = append(d.Heard, Heard{Info: i, Took: a.At.Sub(start), Bytes: len(a.Data), Shortened: i.Metadata[Shortened]})
|
||||
k := key{i.Name, i.ID}
|
||||
answered[k]++
|
||||
d.Heard = append(d.Heard, Heard{Info: i, Took: a.At.Sub(start), Bytes: len(a.Data), Shortened: i.Metadata[Shortened],
|
||||
Digest: digest[k]})
|
||||
case <-least.C:
|
||||
windowOver = true
|
||||
case <-most.C:
|
||||
@@ -350,6 +440,14 @@ func Gather(conn *bus.Conn) (Discovery, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// heldAt is the answer `known` holds for an instance at this digest; never for a PING without one.
|
||||
func heldAt(known Known, name, id, digest string) (Heard, bool) {
|
||||
if known == nil || digest == "" {
|
||||
return Heard{}, false
|
||||
}
|
||||
return known(name, id, digest)
|
||||
}
|
||||
|
||||
// Endpoints reads an info_response's endpoints back into what they announce.
|
||||
func Endpoints(i micro.Info) []Endpoint {
|
||||
var out []Endpoint
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
)
|
||||
|
||||
// novox/hq issue 554: the laptop's console took a round in which not even a PING arrived within the
|
||||
// 750 ms window as the mesh's answer that nothing is there, and refused every seat and module call. A
|
||||
// PING that arrives after the window is waited for, up to the patience.
|
||||
func TestARoundWaitsForAPingThatArrivesAfterTheWindow(t *testing.T) {
|
||||
was := Patience
|
||||
Patience = 2500 * time.Millisecond
|
||||
t.Cleanup(func() { Patience = was })
|
||||
|
||||
late := connect(t, "node-tools", "laptop")
|
||||
s := Service{Name: "node-tools", ID: "laptop", Metadata: map[string]string{"node": "laptop"}}
|
||||
ping, _ := json.Marshal(micro.Ping{ServiceIdentity: s.identity(), Type: micro.PingResponseType})
|
||||
stop, err := late.Raw("$SRV.PING", func(string, []byte) []byte { time.Sleep(Window + 300*time.Millisecond); return ping })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
body, _ := json.Marshal(Info(s, []Endpoint{{Kind: KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Subject: "mesh.mod.slack.tool.slack_check.laptop"}}))
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO.node-tools.laptop"} {
|
||||
stop, err := late.Raw(subject, func(string, []byte) []byte { return body })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
late.Flush()
|
||||
|
||||
d, err := Gather(connect(t, "console", "laptop"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.Empty() || len(d.Heard) != 1 || d.Heard[0].Machine() != "laptop" {
|
||||
t.Fatalf("a PING after the window was not waited for: waited %s, pinged %d, heard %+v", d.Waited, d.Pinged, d.Heard)
|
||||
}
|
||||
if d.Waited < Window || d.Waited >= Patience {
|
||||
t.Errorf("waited %s; the PING came after %s and patience is %s", d.Waited, Window+300*time.Millisecond, Patience)
|
||||
}
|
||||
}
|
||||
|
||||
// A round in which nothing at all arrives is said to be empty, after the patience — not answered as an
|
||||
// empty mesh at the window.
|
||||
func TestARoundThatHearsNothingIsEmptyAfterThePatience(t *testing.T) {
|
||||
was := Patience
|
||||
Patience = 1500 * time.Millisecond
|
||||
t.Cleanup(func() { Patience = was })
|
||||
d, err := Gather(connect(t, "console", "laptop"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !d.Empty() || d.Waited < Patience {
|
||||
t.Errorf("empty %v after %s (patience %s)", d.Empty(), d.Waited, Patience)
|
||||
}
|
||||
}
|
||||
|
||||
// What a runtime announces is fetched again only when its PING's digest changed: every round pulled
|
||||
// some 880 kB of announcements to the laptop, and the PING answers queued behind them (issue 554).
|
||||
func TestAnUnchangedAnnouncementIsNotFetchedAgain(t *testing.T) {
|
||||
runtime := connect(t, "node-tools", "desk")
|
||||
var endpoints atomic.Value
|
||||
endpoints.Store([]Endpoint{{Kind: KindTool, Module: "alpha", Tool: "one", Node: "desk", Subject: "mesh.mod.alpha.tool.one.desk"}})
|
||||
stop, err := Serve(runtime, Service{Name: "node-tools", ID: "desk", Metadata: map[string]string{"node": "desk"}},
|
||||
func() []Endpoint { return endpoints.Load().([]Endpoint) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
runtime.Flush()
|
||||
// Every request for what desk serves, counted where it is answered.
|
||||
var asked atomic.Int32
|
||||
spy := connect(t, "node-tools", "spy")
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO.node-tools", "$SRV.INFO.node-tools.desk"} {
|
||||
stop, err := spy.Raw(subject, func(string, []byte) []byte { asked.Add(1); return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
spy.Flush()
|
||||
|
||||
console := connect(t, "console", "laptop")
|
||||
first, err := Gather(console, nil)
|
||||
if err != nil || len(first.Heard) != 1 || first.Heard[0].Digest == "" || first.Heard[0].Reused {
|
||||
t.Fatalf("first round: %v %+v", err, first.Heard)
|
||||
}
|
||||
if asked.Load() != 1 {
|
||||
t.Errorf("the first round asked what desk serves %d time(s)", asked.Load())
|
||||
}
|
||||
held := first.Heard[0]
|
||||
known := func(name, id, digest string) (Heard, bool) {
|
||||
if name == held.Info.Name && id == held.Info.ID && digest == held.Digest {
|
||||
return held, true
|
||||
}
|
||||
return Heard{}, false
|
||||
}
|
||||
asked.Store(0)
|
||||
second, err := Gather(console, known)
|
||||
if err != nil || len(second.Heard) != 1 || !second.Heard[0].Reused || len(Endpoints(second.Heard[0].Info)) != 1 {
|
||||
t.Fatalf("an unchanged announcement: %v %+v", err, second.Heard)
|
||||
}
|
||||
if asked.Load() != 0 {
|
||||
t.Errorf("an unchanged announcement was asked for %d time(s)", asked.Load())
|
||||
}
|
||||
|
||||
endpoints.Store([]Endpoint{
|
||||
{Kind: KindTool, Module: "alpha", Tool: "one", Node: "desk", Subject: "mesh.mod.alpha.tool.one.desk"},
|
||||
{Kind: KindTool, Module: "alpha", Tool: "two", Node: "desk", Subject: "mesh.mod.alpha.tool.two.desk"},
|
||||
})
|
||||
third, err := Gather(console, known)
|
||||
if err != nil || len(third.Heard) != 1 || third.Heard[0].Reused || len(Endpoints(third.Heard[0].Info)) != 2 ||
|
||||
third.Heard[0].Digest == held.Digest {
|
||||
t.Fatalf("a changed announcement was not fetched: %v %+v", err, third.Heard)
|
||||
}
|
||||
if asked.Load() != 1 {
|
||||
t.Errorf("a changed announcement was asked for %d time(s)", asked.Load())
|
||||
}
|
||||
}
|
||||
@@ -33,18 +33,22 @@ func lagging(t *testing.T, conn *bus.Conn, s Service, lag time.Duration, endpoin
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
stop, err = conn.Raw("$SRV.INFO", func(string, []byte) []byte {
|
||||
// Asked of every service, and of this instance by name and id — as discovery asks it (issue 554).
|
||||
info := func(string, []byte) []byte {
|
||||
if lag < 0 {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(lag)
|
||||
body, _ := json.Marshal(Info(s, endpoints))
|
||||
return body
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO." + s.Name + "." + s.ID} {
|
||||
stop, err := conn.Raw(subject, info)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
conn.Flush()
|
||||
}
|
||||
|
||||
@@ -73,7 +77,7 @@ func TestDiscoveryWaitsForEveryRuntimeThatSaidItIsThere(t *testing.T) {
|
||||
|
||||
asker := connect(t, "console", "desk")
|
||||
start := time.Now()
|
||||
d, err := Gather(asker)
|
||||
d, err := Gather(asker, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -109,7 +113,7 @@ func TestDiscoveryDoesNotWaitWhenEveryoneAnswered(t *testing.T) {
|
||||
t.Cleanup(stop)
|
||||
fast.Flush()
|
||||
start := time.Now()
|
||||
d, err := Gather(connect(t, "console", "desk"))
|
||||
d, err := Gather(connect(t, "console", "desk"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's bus state in the new form (novox/hq ADR 0595, design 25 § 12.5; issue 1966 T1): an object store, a stream,
|
||||
// a work queue, a ledger, each asked for in `requires` by a local name and bound in the module's membership by the bus
|
||||
// names the bus module created. **A bundle reaches only its own module's bindings**: one account per machine carries
|
||||
// every module on it, so what a module may reach is decided here, from its membership, and refused with the reason
|
||||
// before anything is sent, as its state is (state.go).
|
||||
|
||||
// Binding is the module's binding of a local name, of the storage type storage, or the reason there is none.
|
||||
func (c *Conn) Binding(module, local, storage string) (membership.BusBinding, error) {
|
||||
m := c.Membership(module)
|
||||
if m == nil {
|
||||
return membership.BusBinding{}, fmt.Errorf("%s has no membership issued on %s yet, so none of its bus state is "+
|
||||
"reachable until the mesh issues one (novox/hq ADR 0595)", module, c.node)
|
||||
}
|
||||
var names []string
|
||||
for _, b := range m.Bindings {
|
||||
if b.Local != local {
|
||||
names = append(names, b.Local+" ("+b.Storage+")")
|
||||
continue
|
||||
}
|
||||
if b.Storage != storage {
|
||||
return membership.BusBinding{}, fmt.Errorf("%s's %q is bound with the storage type %s, not %s (novox/hq ADR 0595)",
|
||||
module, local, b.Storage, storage)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
sort.Strings(names)
|
||||
bound := "none"
|
||||
if len(names) > 0 {
|
||||
bound = strings.Join(names, ", ")
|
||||
}
|
||||
return membership.BusBinding{}, fmt.Errorf("%s was bound no %s called %q: a module reaches only the bus state its "+
|
||||
"manifest asks for in `requires`, by its local name, once the mesh created it; it was bound: %s (novox/hq ADR 0595)",
|
||||
module, storage, local, bound)
|
||||
}
|
||||
|
||||
// Bindings is every binding a module's membership carries now.
|
||||
func (c *Conn) Bindings(module string) []membership.BusBinding {
|
||||
if m := c.Membership(module); m != nil {
|
||||
return append([]membership.BusBinding(nil), m.Bindings...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// jetStream is a JetStream handle on this connection.
|
||||
func (c *Conn) jetStream() (jetstream.JetStream, error) { return jetstream.New(c.nc) }
|
||||
|
||||
func (c *Conn) objectStore(ctx context.Context, module, local string) (jetstream.ObjectStore, membership.BusBinding, error) {
|
||||
b, err := c.Binding(module, local, catalogue.StorageObjectStore)
|
||||
if err != nil {
|
||||
return nil, b, err
|
||||
}
|
||||
js, err := c.jetStream()
|
||||
if err != nil {
|
||||
return nil, b, err
|
||||
}
|
||||
obs, err := js.ObjectStore(ctx, b.ObjectStore)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil, b, fmt.Errorf("%s's objects %q are bound and their store %s is not on the bus", module, local, b.ObjectStore)
|
||||
}
|
||||
return obs, b, err
|
||||
}
|
||||
|
||||
func objectInfoOf(i *jetstream.ObjectInfo) buscalls.ObjectInfo {
|
||||
return buscalls.ObjectInfo{Name: i.Name, Size: i.Size, Chunks: i.Chunks, Digest: i.Digest, Description: i.Description,
|
||||
Modified: i.ModTime, Deleted: i.Deleted}
|
||||
}
|
||||
|
||||
// ObjectsPut stores one object whole under its name in the module's object store.
|
||||
func (c *Conn) ObjectsPut(module string, r buscalls.ObjectsPutRequest) (buscalls.ObjectInfo, error) {
|
||||
if r.Name == "" {
|
||||
return buscalls.ObjectInfo{}, errors.New("an object is put under a name")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
obs, _, err := c.objectStore(ctx, module, r.Objects)
|
||||
if err != nil {
|
||||
return buscalls.ObjectInfo{}, err
|
||||
}
|
||||
info, err := obs.PutBytes(ctx, r.Name, r.Data)
|
||||
if err != nil {
|
||||
return buscalls.ObjectInfo{}, err
|
||||
}
|
||||
if r.Description != "" {
|
||||
if err := obs.UpdateMeta(ctx, r.Name, jetstream.ObjectMeta{Name: r.Name, Description: r.Description}); err != nil {
|
||||
return buscalls.ObjectInfo{}, err
|
||||
}
|
||||
if info, err = obs.GetInfo(ctx, r.Name); err != nil {
|
||||
return buscalls.ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
return objectInfoOf(info), nil
|
||||
}
|
||||
|
||||
// ObjectsGet is one object of the module's object store, its info and its bytes; nil when there is none.
|
||||
func (c *Conn) ObjectsGet(module string, r buscalls.ObjectsNameRequest) (*buscalls.ObjectData, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
obs, _, err := c.objectStore(ctx, module, r.Objects)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := obs.Get(ctx, r.Name)
|
||||
if errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Close()
|
||||
info, err := res.Info()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := obs.GetBytes(ctx, r.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &buscalls.ObjectData{Info: objectInfoOf(info), Data: data}, nil
|
||||
}
|
||||
|
||||
// ObjectsDelete deletes one object of the module's object store; one that is not there is not an error.
|
||||
func (c *Conn) ObjectsDelete(module string, r buscalls.ObjectsNameRequest) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
obs, _, err := c.objectStore(ctx, module, r.Objects)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := obs.Delete(ctx, r.Name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ObjectsList is every object the module's object store holds.
|
||||
func (c *Conn) ObjectsList(module string, r buscalls.ObjectsRequest) (buscalls.ObjectsListed, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
obs, _, err := c.objectStore(ctx, module, r.Objects)
|
||||
if err != nil {
|
||||
return buscalls.ObjectsListed{}, err
|
||||
}
|
||||
out := buscalls.ObjectsListed{Objects: []buscalls.ObjectInfo{}}
|
||||
infos, err := obs.List(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoObjectsFound) {
|
||||
return out, nil
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, i := range infos {
|
||||
out.Objects = append(out.Objects, objectInfoOf(i))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ObjectsInfo is how full the module's object store is: its objects, its bytes and its cap.
|
||||
func (c *Conn) ObjectsInfo(module string, r buscalls.ObjectsRequest) (buscalls.Fill, error) {
|
||||
listed, err := c.ObjectsList(module, r)
|
||||
if err != nil {
|
||||
return buscalls.Fill{}, err
|
||||
}
|
||||
b, _ := c.Binding(module, r.Objects, catalogue.StorageObjectStore)
|
||||
fill, err := c.streamFill("OBJ_" + b.ObjectStore)
|
||||
if err != nil {
|
||||
return buscalls.Fill{}, err
|
||||
}
|
||||
fill.Count = uint64(len(listed.Objects))
|
||||
return fill.Fill, nil
|
||||
}
|
||||
|
||||
// ObjectsWatch hands deliver every object of the module's object store as it is, then each stored or deleted.
|
||||
func (c *Conn) ObjectsWatch(module string, r buscalls.ObjectsWatchRequest, deliver func(buscalls.ObjectsChange) error) (func(), error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
obs, _, err := c.objectStore(ctx, module, r.Objects)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
watchCtx, stop := context.WithCancel(context.Background())
|
||||
w, err := obs.Watch(watchCtx, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
stop()
|
||||
return nil, err
|
||||
}
|
||||
current := true
|
||||
for current {
|
||||
select {
|
||||
case i, ok := <-w.Updates():
|
||||
if !ok {
|
||||
stop()
|
||||
return nil, errors.New("the object store's watch ended before its current objects were handed over")
|
||||
}
|
||||
if i == nil {
|
||||
current = false
|
||||
continue
|
||||
}
|
||||
if err := deliver(buscalls.ObjectsChange{Watch: r.Watch, Object: objectInfoOf(i), Current: true}); err != nil {
|
||||
c.Logf("[mesh-tools] %s did not take an object of %s: %v", module, r.Objects, err)
|
||||
}
|
||||
case <-time.After(StateTimeout):
|
||||
_ = w.Stop()
|
||||
stop()
|
||||
return nil, fmt.Errorf("the object store %s did not hand its current objects in %s", r.Objects, StateTimeout)
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
for i := range w.Updates() {
|
||||
if i == nil {
|
||||
continue
|
||||
}
|
||||
if err := deliver(buscalls.ObjectsChange{Watch: r.Watch, Object: objectInfoOf(i)}); err != nil {
|
||||
c.Logf("[mesh-tools] %s did not take an object of %s: %v", module, r.Objects, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { _ = w.Stop(); stop() }, nil
|
||||
}
|
||||
|
||||
// streamFill is a stream's fill and its first and last sequence, by its name on the bus.
|
||||
func (c *Conn) streamFill(name string) (buscalls.StreamFilled, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
js, err := c.jetStream()
|
||||
if err != nil {
|
||||
return buscalls.StreamFilled{}, err
|
||||
}
|
||||
s, err := js.Stream(ctx, name)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return buscalls.StreamFilled{}, fmt.Errorf("the stream %s is bound and not on the bus", name)
|
||||
}
|
||||
if err != nil {
|
||||
return buscalls.StreamFilled{}, err
|
||||
}
|
||||
info, err := s.Info(ctx)
|
||||
if err != nil {
|
||||
return buscalls.StreamFilled{}, err
|
||||
}
|
||||
max := uint64(0)
|
||||
if info.Config.MaxBytes > 0 {
|
||||
max = uint64(info.Config.MaxBytes)
|
||||
}
|
||||
return buscalls.StreamFilled{Fill: buscalls.Fill{Count: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: max},
|
||||
First: info.State.FirstSeq, Last: info.State.LastSeq}, nil
|
||||
}
|
||||
|
||||
// streamBinding is the module's binding of a stream by its local name, a work queue among them.
|
||||
func (c *Conn) streamBinding(module, local string) (membership.BusBinding, error) {
|
||||
return c.Binding(module, local, catalogue.StorageStream)
|
||||
}
|
||||
|
||||
// StreamFill is how full the module's stream is.
|
||||
func (c *Conn) StreamFill(module string, r buscalls.StreamRequest) (buscalls.StreamFilled, error) {
|
||||
b, err := c.streamBinding(module, r.Stream)
|
||||
if err != nil {
|
||||
return buscalls.StreamFilled{}, err
|
||||
}
|
||||
return c.streamFill(b.Stream)
|
||||
}
|
||||
|
||||
// StreamReadMost is the most messages one read answers.
|
||||
const StreamReadMost = 1000
|
||||
|
||||
// StreamRead is the module's stream's messages from a sequence on, Subjects narrowing them, at most Limit of them.
|
||||
func (c *Conn) StreamRead(module string, r buscalls.StreamReadRequest) (buscalls.StreamPage, error) {
|
||||
b, err := c.streamBinding(module, r.Stream)
|
||||
if err != nil {
|
||||
return buscalls.StreamPage{}, err
|
||||
}
|
||||
return c.readStream(b.Stream, r.Subjects, r.From, r.Limit)
|
||||
}
|
||||
|
||||
// readStream reads a stream by its bus name from a sequence on, the messages on the subjects filter names (every
|
||||
// subject when empty), at most limit of them.
|
||||
func (c *Conn) readStream(name, filter string, from uint64, limit int) (buscalls.StreamPage, error) {
|
||||
if limit <= 0 || limit > StreamReadMost {
|
||||
limit = StreamReadMost
|
||||
}
|
||||
if filter == "" {
|
||||
filter = ">"
|
||||
}
|
||||
if from == 0 {
|
||||
from = 1
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
js, err := c.jetStream()
|
||||
if err != nil {
|
||||
return buscalls.StreamPage{}, err
|
||||
}
|
||||
s, err := js.Stream(ctx, name)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return buscalls.StreamPage{}, fmt.Errorf("the stream %s is bound and not on the bus", name)
|
||||
}
|
||||
if err != nil {
|
||||
return buscalls.StreamPage{}, err
|
||||
}
|
||||
info, err := s.Info(ctx)
|
||||
if err != nil {
|
||||
return buscalls.StreamPage{}, err
|
||||
}
|
||||
page := buscalls.StreamPage{Messages: []buscalls.StreamMessage{}, Last: info.State.LastSeq}
|
||||
for seq := from; seq <= info.State.LastSeq && len(page.Messages) < limit; {
|
||||
msg, err := s.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(filter))
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return page, err
|
||||
}
|
||||
body := json.RawMessage(msg.Data)
|
||||
if !json.Valid(msg.Data) {
|
||||
body, _ = json.Marshal(string(msg.Data))
|
||||
}
|
||||
page.Messages = append(page.Messages, buscalls.StreamMessage{Sequence: msg.Sequence, Subject: msg.Subject,
|
||||
Body: body, At: msg.Time})
|
||||
seq = msg.Sequence + 1
|
||||
if len(page.Messages) == limit && seq <= info.State.LastSeq {
|
||||
page.Next = seq
|
||||
}
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// WorkQueueSize is how many messages wait in the module's work queue, against its cap.
|
||||
func (c *Conn) WorkQueueSize(module string, r buscalls.WorkQueueRequest) (buscalls.WorkQueueSized, error) {
|
||||
b, err := c.streamBinding(module, r.Queue)
|
||||
if err != nil {
|
||||
return buscalls.WorkQueueSized{}, err
|
||||
}
|
||||
fill, err := c.streamFill(b.Stream)
|
||||
if err != nil {
|
||||
return buscalls.WorkQueueSized{}, err
|
||||
}
|
||||
return buscalls.WorkQueueSized{Fill: fill.Fill}, nil
|
||||
}
|
||||
|
||||
// WorkQueueDead is the module's work queue's dead letters: in a work queue a message is gone once settled, so one still
|
||||
// kept below a consumer's acknowledgement floor was delivered as often as the consumer allows and never settled.
|
||||
func (c *Conn) WorkQueueDead(module string, r buscalls.WorkQueueDeadRequest) (buscalls.WorkQueueDeadLetters, error) {
|
||||
b, err := c.streamBinding(module, r.Queue)
|
||||
if err != nil {
|
||||
return buscalls.WorkQueueDeadLetters{}, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
js, err := c.jetStream()
|
||||
if err != nil {
|
||||
return buscalls.WorkQueueDeadLetters{}, err
|
||||
}
|
||||
s, err := js.Stream(ctx, b.Stream)
|
||||
if err != nil {
|
||||
return buscalls.WorkQueueDeadLetters{}, fmt.Errorf("the work queue %s: %w", b.Stream, err)
|
||||
}
|
||||
out := buscalls.WorkQueueDeadLetters{Dead: []buscalls.DeadLetter{}}
|
||||
consumers := s.ListConsumers(ctx)
|
||||
for ci := range consumers.Info() {
|
||||
floor := ci.AckFloor.Stream
|
||||
if ci.Config.MaxDeliver <= 0 || floor == 0 {
|
||||
continue
|
||||
}
|
||||
page, err := c.readStream(b.Stream, ci.Config.FilterSubject, 1, r.Limit)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, m := range page.Messages {
|
||||
if m.Sequence > floor {
|
||||
break
|
||||
}
|
||||
out.Dead = append(out.Dead, buscalls.DeadLetter{Sequence: m.Sequence, Subject: m.Subject,
|
||||
Deliveries: ci.Config.MaxDeliver, Body: m.Body, At: m.At})
|
||||
}
|
||||
}
|
||||
if err := consumers.Err(); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if r.Limit > 0 && len(out.Dead) > r.Limit {
|
||||
out.Dead = out.Dead[:r.Limit]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// 1966/AC12: given a bundle whose membership binds objects `files`, when it puts and gets `a`, then it reads back what it
|
||||
// put; a local name it was not bound is refused with the reason, and so is another module's binding and a binding of
|
||||
// another storage type. A stream and a work queue bound alike answer their read-back, fill and size.
|
||||
func TestABundleReachesItsOwnBoundObjectsAndStreamsOnly(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx := context.Background()
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: "m_files"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "m_events", Subjects: []string{"mesh.state.m.events.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{"mesh.state.m.events.one", "mesh.state.m.events.two"} {
|
||||
if _, err := js.Publish(ctx, s, []byte(`{"n":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
m := mt.MembershipOf("m", "anchor", false, nil)
|
||||
m.Bindings = []membership.BusBinding{
|
||||
{Local: "files", Storage: catalogue.StorageObjectStore, Template: catalogue.TemplateObjects, ObjectStore: "m_files"},
|
||||
{Local: "events", Storage: catalogue.StorageStream, Template: catalogue.TemplateEvents, Stream: "m_events",
|
||||
Subject: "mesh.state.m.events"},
|
||||
}
|
||||
other := mt.MembershipOf("other", "anchor", false, nil)
|
||||
mesh.Issue(t, m)
|
||||
mesh.Issue(t, other)
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("m")
|
||||
conn.Follow("other")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("m") == nil || conn.Membership("other") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if _, err := conn.ObjectsPut("m", buscalls.ObjectsPutRequest{Objects: "files", Name: "a", Data: []byte("what was put")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := conn.ObjectsGet("m", buscalls.ObjectsNameRequest{Objects: "files", Name: "a"})
|
||||
if err != nil || got == nil || string(got.Data) != "what was put" || got.Info.Name != "a" {
|
||||
t.Fatalf("read back %+v, %v", got, err)
|
||||
}
|
||||
if listed, err := conn.ObjectsList("m", buscalls.ObjectsRequest{Objects: "files"}); err != nil || len(listed.Objects) != 1 {
|
||||
t.Fatalf("listed %+v, %v", listed, err)
|
||||
}
|
||||
page, err := conn.StreamRead("m", buscalls.StreamReadRequest{Stream: "events", Limit: 1})
|
||||
if err != nil || len(page.Messages) != 1 || page.Next != 2 || page.Last != 2 {
|
||||
t.Fatalf("read %+v, %v", page, err)
|
||||
}
|
||||
if fill, err := conn.StreamFill("m", buscalls.StreamRequest{Stream: "events"}); err != nil || fill.Count != 2 {
|
||||
t.Fatalf("fill %+v, %v", fill, err)
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
name, module, refusal string
|
||||
call func(module string) error
|
||||
}{
|
||||
{"a local name not bound", "m", `bound no object-store called "photos"`, func(module string) error {
|
||||
_, err := conn.ObjectsGet(module, buscalls.ObjectsNameRequest{Objects: "photos", Name: "a"})
|
||||
return err
|
||||
}},
|
||||
{"another module's binding", "other", `other was bound no object-store called "files"`, func(module string) error {
|
||||
_, err := conn.ObjectsGet(module, buscalls.ObjectsNameRequest{Objects: "files", Name: "a"})
|
||||
return err
|
||||
}},
|
||||
{"a binding of another storage type", "m", `"events" is bound with the storage type stream, not object-store`, func(module string) error {
|
||||
_, err := conn.ObjectsList(module, buscalls.ObjectsRequest{Objects: "events"})
|
||||
return err
|
||||
}},
|
||||
{"a work queue not bound", "m", `bound no stream called "jobs"`, func(module string) error {
|
||||
_, err := conn.WorkQueueSize(module, buscalls.WorkQueueRequest{Queue: "jobs"})
|
||||
return err
|
||||
}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
err := c.call(c.module)
|
||||
if err == nil || !strings.Contains(err.Error(), c.refusal) {
|
||||
t.Fatalf("refused with %v, want the reason %q", err, c.refusal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+408
-133
@@ -5,6 +5,8 @@
|
||||
// mesh.mod.<module>.event.<type> an event a module emits
|
||||
// mesh.mod.<module>.tool.<tool> a tool a module serves
|
||||
// mesh.seat.<seat>.tool.<verb> a role's verb, answered by whoever holds the seat
|
||||
// mesh.mod.<module>.call.<tool>.<caller>, mesh.seat.<seat>.call.<verb>.<caller>
|
||||
// the same, naming the bus user that called (caller.go, hq issue 365)
|
||||
package bus
|
||||
|
||||
import (
|
||||
@@ -16,10 +18,12 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -28,13 +32,9 @@ import (
|
||||
"github.com/novox/mesh-tools/node-tools/internal/wire"
|
||||
)
|
||||
|
||||
// RequestTimeout is how long a call waits for its answer — what modules already expect. The
|
||||
// controller answers its verbs well inside it (its AnswerWithin, ten seconds), with a call id while a
|
||||
// verb still runs, so a caller is never left guessing past this (novox/hq issue 265).
|
||||
// RequestTimeout is how long a call waits for its answer — what modules already expect.
|
||||
const RequestTimeout = 30 * time.Second
|
||||
|
||||
const assignmentsStream = "ASSIGNMENTS"
|
||||
|
||||
// Credential is a broker credential as the mesh delivers it (novox/hq ADR 0120).
|
||||
type Credential struct {
|
||||
URL string `json:"url"`
|
||||
@@ -64,9 +64,23 @@ type Membership struct {
|
||||
Tools string `json:"tools"`
|
||||
// State is every bucket the module's code may reach, by the name it uses (novox/hq ADR 0201).
|
||||
State []StateIssued `json:"state,omitempty"`
|
||||
// Logs is every log the module keeps, by its local name (novox/hq ADR 0297): its own alone.
|
||||
Logs []LogIssued `json:"logs,omitempty"`
|
||||
// SeatTraffic is what the module's code may submit, say, take, ask, answer and read on seats that name
|
||||
// their caller or their kind (novox/hq ADR 0259): the runtime does for it only what is listed here.
|
||||
SeatTraffic *SeatTraffic `json:"seat-traffic,omitempty"`
|
||||
// Streams are the module's own streams (novox/hq ADR 0363 §2): an event one of them carries is published
|
||||
// on that stream's subject, never on EVENTS (streams.go).
|
||||
Streams []StreamIssued `json:"streams,omitempty"`
|
||||
// Hears are the module streams the module hears, each through a durable consumer on that stream the
|
||||
// runtime binds beside the one on EVENTS (ADR 0363 §8).
|
||||
Hears []HeardIssued `json:"hears,omitempty"`
|
||||
// Waits are the event families a session may wait on through the mesh MCP server (novox/hq ADR 0367 §2):
|
||||
// the tool runner's alone, composed from its manifest with the grant that lets it subscribe them.
|
||||
Waits []WaitIssued `json:"waits,omitempty"`
|
||||
// Bindings are the module's bus state in the new form, by local name (novox/hq ADR 0595 §1, mesh-sdk
|
||||
// membership.BusBinding): its object stores, streams, work queues and ledgers, its own alone (bindings.go).
|
||||
Bindings []membership.BusBinding `json:"bindings,omitempty"`
|
||||
}
|
||||
|
||||
// Served is an address a tool is answered on; `{tool}` stands for the tool's name.
|
||||
@@ -117,12 +131,19 @@ func Fatal(err error) string {
|
||||
strings.Contains(msg, "nats: invalid url") {
|
||||
return "the bus address is not a usable URL"
|
||||
}
|
||||
if regexp.MustCompile(`(?i)authorization violation|user authentication expired|permissions violation`).MatchString(msg) {
|
||||
if Refused(err) || regexp.MustCompile(`(?i)user authentication expired|permissions violation`).MatchString(msg) {
|
||||
return "the bus refused this account"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Refused says whether the bus refused the account a connection named (an authorization violation): final for a running
|
||||
// runtime, but at a start possibly only "not yet", since the bus carries a new account after it reloads its users (issue
|
||||
// 1496). Fatal still names it; the caller decides how long a start waits.
|
||||
func Refused(err error) bool {
|
||||
return err != nil && regexp.MustCompile(`(?i)authorization violation`).MatchString(err.Error())
|
||||
}
|
||||
|
||||
func normalizeFingerprint(f string) string {
|
||||
f = strings.TrimSpace(f)
|
||||
if len(f) > 7 && strings.EqualFold(f[:7], "sha256:") {
|
||||
@@ -161,7 +182,7 @@ type Conn struct {
|
||||
cred Credential
|
||||
mu sync.Mutex
|
||||
issued map[string]*Membership // module → membership; present with nil = followed, none issued
|
||||
onNew []func(Membership)
|
||||
onNew []*func(Membership)
|
||||
subs []*nats.Subscription
|
||||
Logf func(format string, args ...any)
|
||||
// answering is every subject this runtime answers on, so a subscription the bus refused can be
|
||||
@@ -170,6 +191,10 @@ type Conn struct {
|
||||
// RetryAfter is how long after a refusal a subscription is asked for again, by attempt; past the
|
||||
// last, it is given up and said. A variable so a test need not wait minutes.
|
||||
RetryAfter []time.Duration
|
||||
// calls are the caller-named calls in flight (caller.go, novox/hq issue 365).
|
||||
calls callers
|
||||
// waits are the waits in flight by subject, each told when the bus refuses its subscription (wait.go).
|
||||
waits map[string]map[chan string]bool
|
||||
}
|
||||
|
||||
// answered is one subject the runtime answers on: what to subscribe again with, and how often it was
|
||||
@@ -204,6 +229,12 @@ func (c *Conn) refused(err error) {
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
return
|
||||
}
|
||||
if c.publishRefused(err) {
|
||||
return
|
||||
}
|
||||
if subject := subjectRefused(err); subject != "" && c.waitRefused(subject) {
|
||||
return
|
||||
}
|
||||
m := refusedSubject.FindStringSubmatch(err.Error())
|
||||
if len(m) < 2 {
|
||||
return
|
||||
@@ -257,9 +288,20 @@ func (c *Conn) refused(err error) {
|
||||
})
|
||||
}
|
||||
|
||||
// Connect dials the bus as the credential's module. A module's subjects come from its credential,
|
||||
// never from its calls (ADR 0074).
|
||||
// Connect dials the bus as the credential's module and follows its membership. A module's subjects come from its
|
||||
// credential, never from its calls (ADR 0074).
|
||||
func Connect(cred Credential) (*Conn, error) {
|
||||
c, err := Dial(cred)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Follow(cred.Module)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Dial dials the bus as the credential's module and follows nothing: a connection that only publishes for its module,
|
||||
// as a module's ledger connection does (novox/hq ADR 0597 §9).
|
||||
func Dial(cred Credential) (*Conn, error) {
|
||||
if cred.Module == "" {
|
||||
return nil, errors.New("a broker credential with no module: the runtime derives its subjects " +
|
||||
"from the account the mesh issued, and cannot guess which module it is")
|
||||
@@ -298,7 +340,6 @@ func Connect(cred Credential) (*Conn, error) {
|
||||
}
|
||||
c = &Conn{nc: nc, js: js, self: cred.Module, node: cred.Node, cred: cred,
|
||||
issued: map[string]*Membership{}, Logf: log.Printf, answering: map[string]*answered{}}
|
||||
c.Follow(cred.Module)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
@@ -311,13 +352,10 @@ func (c *Conn) Node() string { return c.node }
|
||||
// Credential is what this connection was opened with.
|
||||
func (c *Conn) Credential() Credential { return c.cred }
|
||||
|
||||
// MembershipSubject is the one address a runtime derives for an assignment (ADR 0160).
|
||||
func MembershipSubject(node, module string) string {
|
||||
return "mesh.assignment." + node + "." + module
|
||||
}
|
||||
|
||||
// Follow reads a module's membership on this machine once and follows it live, so its tools are
|
||||
// served where the mesh issued them (ADR 0175).
|
||||
// served where the mesh issued them (ADR 0175). It is read from the grants part's state memberships, its key
|
||||
// `<node>.<module>` (novox/hq ADR 0571 clause 7 and 10, mesh-sdk membership.GrantsBucket), and nowhere else:
|
||||
// the old assignments stream is not read and its subject is not heard.
|
||||
func (c *Conn) Follow(module string) {
|
||||
c.mu.Lock()
|
||||
if c.node == "" {
|
||||
@@ -330,41 +368,150 @@ func (c *Conn) Follow(module string) {
|
||||
}
|
||||
c.issued[module] = nil
|
||||
c.mu.Unlock()
|
||||
subject := MembershipSubject(c.node, module)
|
||||
// The subject-addressed direct get: the one address the mesh grants this account on the
|
||||
// stream's API.
|
||||
if got, err := c.nc.Request("$JS.API.DIRECT.GET."+assignmentsStream+"."+subject, nil, 5*time.Second); err == nil {
|
||||
if got.Header.Get("Status") == "" && len(got.Data) > 0 {
|
||||
var m Membership
|
||||
if json.Unmarshal(got.Data, &m) == nil {
|
||||
c.mu.Lock()
|
||||
c.issued[module] = &m
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
c.followGrants(module)
|
||||
if c.Membership(module) == nil {
|
||||
c.Logf("[mesh-tools] no membership issued for %s on %s yet; serving the derived shape until one arrives", module, c.node)
|
||||
}
|
||||
sub, err := c.nc.Subscribe(subject, func(msg *nats.Msg) {
|
||||
}
|
||||
|
||||
// GrantsPatience bounds the first read of a membership from the grants part's state.
|
||||
var GrantsPatience = 2 * time.Second
|
||||
|
||||
// GrantsRetry is the first wait before the memberships state is read again after it could not be; it doubles up to
|
||||
// GrantsRetryMost.
|
||||
var (
|
||||
GrantsRetry = time.Second
|
||||
GrantsRetryMost = 30 * time.Second
|
||||
)
|
||||
|
||||
// issue applies a membership that arrived for module, live, and re-serves on it.
|
||||
func (c *Conn) issue(module string, m Membership) {
|
||||
c.mu.Lock()
|
||||
c.issued[module] = &m
|
||||
handlers := make([]func(Membership), 0, len(c.onNew))
|
||||
for _, h := range c.onNew {
|
||||
handlers = append(handlers, *h)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
c.Logf("[mesh-tools] %s on %s was issued a new membership; re-serving on it", module, c.node)
|
||||
for _, h := range handlers {
|
||||
h(m)
|
||||
}
|
||||
_ = c.nc.Flush()
|
||||
}
|
||||
|
||||
// followGrants reads module's key of the grants part's state memberships and watches it: the value there now is applied
|
||||
// before Follow returns, each change after it live. A state that is not there yet, or not readable (a bus answering 503
|
||||
// right after its boot, a timeout), is said once and read again, with a growing wait, until it can be or the
|
||||
// connection closes: Follow runs once per module, so nothing else would ever read it again.
|
||||
func (c *Conn) followGrants(module string) {
|
||||
if c.watchGrants(module, true) {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
wait := GrantsRetry
|
||||
for !c.nc.IsClosed() {
|
||||
time.Sleep(wait)
|
||||
if c.nc.IsClosed() {
|
||||
return
|
||||
}
|
||||
if c.watchGrants(module, false) {
|
||||
c.Logf("[mesh-tools] the grants part's memberships are read again; %s on %s follows its key", module, c.node)
|
||||
return
|
||||
}
|
||||
if wait *= 2; wait > GrantsRetryMost {
|
||||
wait = GrantsRetryMost
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// watchGrants is one attempt of followGrants: true once the key is watched. A failure is said when say is true.
|
||||
func (c *Conn) watchGrants(module string, say bool) bool {
|
||||
logf := func(format string, a ...any) {
|
||||
if say {
|
||||
c.Logf(format, a...)
|
||||
}
|
||||
}
|
||||
key := membership.GrantsKey(c.node, module)
|
||||
// First the key's direct get, which the account is granted alone: a bus that refuses it, or a state not there yet,
|
||||
// answers at once, where binding the bucket would wait out its bound on a refusal.
|
||||
bucket := membership.GrantsBucket
|
||||
probe, err := c.nc.Request("$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+key, nil, GrantsPatience)
|
||||
if err != nil || probe.Header.Get("Status") == "503" {
|
||||
why := "it did not answer"
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%s); %s has no membership "+
|
||||
"until it can be", bucket, why, module)
|
||||
return false
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
logf("[mesh-tools] the grants part's memberships (%s) cannot be read (%v); %s has no membership "+
|
||||
"until it can be", membership.GrantsBucket, err, module)
|
||||
return false
|
||||
}
|
||||
w, err := kv.Watch(context.Background(), key)
|
||||
if err != nil {
|
||||
logf("[mesh-tools] %s's key %s of the grants part's memberships cannot be watched (%v); %s has no membership "+
|
||||
"until it can be", module, key, err, module)
|
||||
return false
|
||||
}
|
||||
apply := func(e jetstream.KeyValueEntry) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return // a membership withdrawn is the stream's, or the module's unassignment's, to say
|
||||
}
|
||||
// Read through mesh-sdk's one codec of this state, the grants part's writer's (issue 774): it takes the bare
|
||||
// membership and a wrapper the part wrote before of one published or unchanged, and refuses a held-back one's
|
||||
// wrapper or a value naming no node or module, which then leaves the module with no membership.
|
||||
// A refused value arriving after a good one keeps the last good membership: nothing here clears it.
|
||||
var m Membership
|
||||
if err := json.Unmarshal(msg.Data, &m); err != nil {
|
||||
c.Logf("[mesh-tools] a membership arrived that is not one: %v", err)
|
||||
body, err := membership.BareGrantsMembership(e.Value())
|
||||
if err == nil {
|
||||
err = json.Unmarshal(body, &m)
|
||||
}
|
||||
if err != nil {
|
||||
c.Logf("[mesh-tools] the membership at %s is not one: %v", key, err)
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.issued[module] = &m
|
||||
handlers := append([]func(Membership){}, c.onNew...)
|
||||
c.mu.Unlock()
|
||||
c.Logf("[mesh-tools] %s on %s was issued a new membership; re-serving on it", module, c.node)
|
||||
for _, h := range handlers {
|
||||
h(m)
|
||||
}
|
||||
_ = c.nc.Flush()
|
||||
})
|
||||
if err == nil {
|
||||
c.track(sub)
|
||||
c.issue(module, m)
|
||||
}
|
||||
timer := time.NewTimer(StateTimeout)
|
||||
defer timer.Stop()
|
||||
for current := true; current; {
|
||||
select {
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if e == nil {
|
||||
current = false
|
||||
continue
|
||||
}
|
||||
apply(e)
|
||||
case <-timer.C:
|
||||
_ = w.Stop()
|
||||
logf("[mesh-tools] %s's key %s of the grants part's memberships did not answer in %s; %s has no membership "+
|
||||
"until it can be", module, key, StateTimeout, module)
|
||||
return false
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
for e := range w.Updates() {
|
||||
if e != nil {
|
||||
apply(e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// Membership is what the mesh issued a module here, or nil when nothing has been issued.
|
||||
@@ -396,10 +543,28 @@ func (c *Conn) Serving() []string {
|
||||
// OnMembership is called with every new membership any followed module is issued.
|
||||
func (c *Conn) OnMembership(h func(Membership)) {
|
||||
c.mu.Lock()
|
||||
c.onNew = append(c.onNew, h)
|
||||
c.onNew = append(c.onNew, &h)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// onMembership is OnMembership until the answered function is called, which drops the handler.
|
||||
func (c *Conn) onMembership(h func(Membership)) func() {
|
||||
mine := &h
|
||||
c.mu.Lock()
|
||||
c.onNew = append(c.onNew, mine)
|
||||
c.mu.Unlock()
|
||||
return func() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for i, had := range c.onNew {
|
||||
if had == mine {
|
||||
c.onNew = append(c.onNew[:i:i], c.onNew[i+1:]...)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Conn) track(s *nats.Subscription) {
|
||||
c.mu.Lock()
|
||||
c.subs = append(c.subs, s)
|
||||
@@ -441,10 +606,17 @@ type reply struct {
|
||||
|
||||
// answerOn answers one subject with one handler, and says which machine answered (ADR 0159).
|
||||
func (c *Conn) answerOn(subject, queue string, h Handler) (func(), error) {
|
||||
return c.answerCalledOn(subject, queue, func(_ string, body json.RawMessage) (any, error) { return h(body) })
|
||||
}
|
||||
|
||||
// answerCalledOn answers one subject, handing the handler the caller the subject a call arrived on names:
|
||||
// under a pattern calledPattern made, its last token; on any other subject, "" — none was proven (novox/hq
|
||||
// issue 365). Never from the body: a body is the caller's to write.
|
||||
func (c *Conn) answerCalledOn(subject, queue string, h CalledHandler) (func(), error) {
|
||||
cb := func(msg *nats.Msg) {
|
||||
go func() {
|
||||
var r reply
|
||||
result, err := h(json.RawMessage(msg.Data))
|
||||
result, err := h(callerOn(subject, msg.Subject), json.RawMessage(msg.Data))
|
||||
if err != nil {
|
||||
r.Error = err.Error()
|
||||
} else {
|
||||
@@ -497,18 +669,50 @@ func nullable(v any) any {
|
||||
|
||||
// HandleSubject answers a subject outright: a seat's verb where the mesh issued it.
|
||||
func (c *Conn) HandleSubject(subject string, h Handler) (func(), error) {
|
||||
return c.answerOn(subject, "", h)
|
||||
return c.HandleSubjectCalled(subject, func(_ string, body json.RawMessage) (any, error) { return h(body) })
|
||||
}
|
||||
|
||||
// HandleSubjectCalled answers a tool subject and its caller-named counterpart (novox/hq issue 365): a call on
|
||||
// the counterpart is handed the caller its subject names, one on the subject itself "" — none proven. The
|
||||
// subject that names no caller is served beside the other for one release, so callers move without a gap
|
||||
// (its retirement: hq issue 464).
|
||||
func (c *Conn) HandleSubjectCalled(subject string, h CalledHandler) (func(), error) {
|
||||
return c.answerBoth(subject, "", h)
|
||||
}
|
||||
|
||||
// answerBoth answers a subject and, when it is a tool's, the caller-named pattern beside it, in one queue.
|
||||
func (c *Conn) answerBoth(subject, queue string, h CalledHandler) (func(), error) {
|
||||
stop, err := c.answerCalledOn(subject, queue, h)
|
||||
if err != nil {
|
||||
return func() {}, err
|
||||
}
|
||||
pattern := calledPattern(subject)
|
||||
if pattern == "" {
|
||||
return stop, nil
|
||||
}
|
||||
stopCalled, err := c.answerCalledOn(pattern, queue, h)
|
||||
if err != nil {
|
||||
c.Logf("[mesh-tools] cannot serve the caller-named %s: %v; its callers are asked on %s alone", pattern, err, subject)
|
||||
return stop, nil
|
||||
}
|
||||
return func() { stop(); stopCalled() }, nil
|
||||
}
|
||||
|
||||
// Handle serves `<module>.<tool>` where the mesh issued that module, and follows its membership:
|
||||
// when a new one arrives, it serves where it now says and stops where it no longer does.
|
||||
func (c *Conn) Handle(key string, h Handler) (func(), error) {
|
||||
return c.HandleCalled(key, func(_ string, body json.RawMessage) (any, error) { return h(body) })
|
||||
}
|
||||
|
||||
// HandleCalled is Handle telling the handler who called: the bus user a caller-named subject names, or "" for
|
||||
// a call on the subject that names none (novox/hq issue 365).
|
||||
func (c *Conn) HandleCalled(key string, h CalledHandler) (func(), error) {
|
||||
if strings.HasPrefix(key, "seat:") {
|
||||
subject, err := ToolSubject(key, c.self)
|
||||
if err != nil {
|
||||
return func() {}, err
|
||||
}
|
||||
return c.answerOn(subject, "", h)
|
||||
return c.answerBoth(subject, "", h)
|
||||
}
|
||||
module, tool := c.self, key
|
||||
if dot := strings.Index(key, "."); dot >= 0 {
|
||||
@@ -520,15 +724,19 @@ func (c *Conn) Handle(key string, h Handler) (func(), error) {
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var stops []func()
|
||||
stopped := false
|
||||
serve := func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if stopped {
|
||||
return
|
||||
}
|
||||
for _, s := range stops {
|
||||
s()
|
||||
}
|
||||
stops = nil
|
||||
for _, s := range c.servedOn(module, tool) {
|
||||
if stop, err := c.answerOn(s.Subject, s.Queue, h); err == nil {
|
||||
if stop, err := c.answerBoth(s.Subject, s.Queue, h); err == nil {
|
||||
stops = append(stops, stop)
|
||||
} else {
|
||||
c.Logf("[mesh-tools] cannot serve %s on %s: %v", key, s.Subject, err)
|
||||
@@ -536,14 +744,18 @@ func (c *Conn) Handle(key string, h Handler) (func(), error) {
|
||||
}
|
||||
}
|
||||
serve()
|
||||
c.OnMembership(func(m Membership) {
|
||||
// Its stop drops the membership handler too: a handler left behind would bind the stopped tool again
|
||||
// on the module's next membership, and call a handler whose bundle was replaced (issue 1945).
|
||||
unfollow := c.onMembership(func(m Membership) {
|
||||
if m.Module == module {
|
||||
serve()
|
||||
}
|
||||
})
|
||||
return func() {
|
||||
unfollow()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
stopped = true
|
||||
for _, s := range stops {
|
||||
s()
|
||||
}
|
||||
@@ -585,66 +797,59 @@ func (c *Conn) Ask(key string, body any, on string) (Answered, error) {
|
||||
if err != nil {
|
||||
return Answered{}, err
|
||||
}
|
||||
answered, handingOver, err := c.askOnce(subject, data)
|
||||
if !handingOver {
|
||||
return answered, err
|
||||
// **Named by its caller first** (novox/hq issue 365): on the caller-named subject, which the bus lets this
|
||||
// account publish with its own name alone; asked again on the subject that names no caller only when the
|
||||
// first reached nobody — nothing serves it yet, or the bus does not grant it yet — so no call is done twice.
|
||||
if called := CalledSubject(subject, c.cred.User); called != "" {
|
||||
answered, fallBack, err := c.askCalled(called, data)
|
||||
if !fallBack {
|
||||
return answered, err
|
||||
}
|
||||
}
|
||||
// **Refused because the controller is handing over: asked once more** (novox/hq issue 289). The
|
||||
// controller being replaced did nothing and said so; the one after it answers the same call. Once:
|
||||
// a second refusal is a controller that is not coming back, and the caller is told.
|
||||
time.Sleep(HandoverPause)
|
||||
answered, again, err := c.askOnce(subject, data)
|
||||
if err != nil && again {
|
||||
return Answered{}, fmt.Errorf("asked twice, %s apart, and both times refused as a handover: %w", HandoverPause, err)
|
||||
}
|
||||
return answered, err
|
||||
return c.askOnce(subject, data)
|
||||
}
|
||||
|
||||
// HandoverPause is how long a call refused as a handover waits before it is asked once more: the
|
||||
// supervisor's restart of the controller, which stops the one being replaced and starts the next. A
|
||||
// variable so a test need not wait.
|
||||
var HandoverPause = 3 * time.Second
|
||||
// ErrNoResponders is the bus's own refusal: nothing subscribes to the subject asked. It is told apart
|
||||
// from a tool's own error by errors.Is, never by its text (a tool may say "503" itself, issue 1529).
|
||||
var ErrNoResponders = errors.New("503 no responders")
|
||||
|
||||
// RetryHandingOver is the mark the controller puts on a call it refused because it is being replaced
|
||||
// (mesh-controller internal/link, `"retry": "handing-over"`).
|
||||
const RetryHandingOver = "handing-over"
|
||||
|
||||
// askOnce asks once, and says whether the answer was a refusal marked to be asked again.
|
||||
func (c *Conn) askOnce(subject string, data []byte) (Answered, bool, error) {
|
||||
// askOnce asks once.
|
||||
func (c *Conn) askOnce(subject string, data []byte) (Answered, error) {
|
||||
msg, err := c.nc.Request(subject, data, RequestTimeout)
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return Answered{}, false, errors.New("503 no responders")
|
||||
return Answered{}, ErrNoResponders
|
||||
}
|
||||
if errors.Is(err, nats.ErrTimeout) {
|
||||
return Answered{}, false, errors.New("timeout")
|
||||
return Answered{}, errors.New("timeout")
|
||||
}
|
||||
return Answered{}, false, err
|
||||
}
|
||||
// An answer larger than one message comes in parts (pages.go, novox/hq issue 314).
|
||||
body, err := c.whole(subject, msg.Data)
|
||||
if err != nil {
|
||||
return Answered{}, false, err
|
||||
return Answered{}, err
|
||||
}
|
||||
return answerOf(msg.Data)
|
||||
}
|
||||
|
||||
// answerOf reads one answer.
|
||||
func answerOf(data []byte) (Answered, error) {
|
||||
var r struct {
|
||||
Result json.RawMessage `json:"result"`
|
||||
Error string `json:"error"`
|
||||
Node string `json:"node"`
|
||||
Retry string `json:"retry"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &r); err != nil {
|
||||
return Answered{}, false, err
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return Answered{}, err
|
||||
}
|
||||
if r.Error != "" {
|
||||
return Answered{}, r.Retry == RetryHandingOver, errors.New(r.Error)
|
||||
return Answered{}, errors.New(r.Error)
|
||||
}
|
||||
return Answered{Result: r.Result, Node: r.Node}, false, nil
|
||||
return Answered{Result: r.Result, Node: r.Node}, nil
|
||||
}
|
||||
|
||||
// PublishAs emits an event as a module: published into JetStream and awaited, de-duplicated by its
|
||||
// own id (ADR 0042). The body is the payload; the metadata rides as headers.
|
||||
func (c *Conn) PublishAs(module string, env Envelope) error {
|
||||
msg := nats.NewMsg("mesh.mod." + module + ".event." + env.Key)
|
||||
// On the module's own stream when its membership names one carrying the event (ADR 0363 §2), on EVENTS
|
||||
// otherwise — and only there, as before, for a membership that names no stream.
|
||||
msg := nats.NewMsg(eventSubject(c.Membership(module), module, env.Key))
|
||||
for k, v := range env.Headers {
|
||||
msg.Header.Set(k, v)
|
||||
}
|
||||
@@ -704,6 +909,21 @@ type Arrival struct {
|
||||
// that said it is there, not for a fixed moment (2026-10-05). Answers beyond what the caller has
|
||||
// taken are kept up to the channel's room; an empty answer is no answer.
|
||||
func (c *Conn) Collect(subject string, body []byte) (<-chan Arrival, func(), error) {
|
||||
out, ask, stop, err := c.Collector(subject)
|
||||
if err != nil {
|
||||
return nil, func() {}, err
|
||||
}
|
||||
if err := ask(subject, body); err != nil {
|
||||
stop()
|
||||
return nil, func() {}, err
|
||||
}
|
||||
return out, stop, nil
|
||||
}
|
||||
|
||||
// Collector is one inbox for many requests: every answer to every request `ask` publishes arrives on
|
||||
// the one channel, until stopped — for discovery, which asks each service that said it is there for
|
||||
// what it serves, as each says so (novox/hq issue 554). `what` names the requests in a log line.
|
||||
func (c *Conn) Collector(what string) (<-chan Arrival, func(subject string, body []byte) error, func(), error) {
|
||||
inbox := c.nc.NewRespInbox()
|
||||
out := make(chan Arrival, 1024)
|
||||
var mu sync.Mutex
|
||||
@@ -721,11 +941,11 @@ func (c *Conn) Collect(subject string, body []byte) (<-chan Arrival, func(), err
|
||||
select {
|
||||
case out <- a:
|
||||
default:
|
||||
c.Logf("[mesh-tools] more answers to %s than are kept; one (%d bytes) dropped", subject, len(msg.Data))
|
||||
c.Logf("[mesh-tools] more answers to %s than are kept; one (%d bytes) dropped", what, len(msg.Data))
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, func() {}, err
|
||||
return nil, nil, func() {}, err
|
||||
}
|
||||
stop := func() {
|
||||
_ = sub.Unsubscribe()
|
||||
@@ -733,11 +953,8 @@ func (c *Conn) Collect(subject string, body []byte) (<-chan Arrival, func(), err
|
||||
stopped = true
|
||||
mu.Unlock()
|
||||
}
|
||||
if err := c.nc.PublishRequest(subject, inbox, body); err != nil {
|
||||
stop()
|
||||
return nil, func() {}, err
|
||||
}
|
||||
return out, stop, nil
|
||||
ask := func(subject string, body []byte) error { return c.nc.PublishRequest(subject, inbox, body) }
|
||||
return out, ask, stop, nil
|
||||
}
|
||||
|
||||
// MaxPayload is the largest message the bus carries, as the server told this connection.
|
||||
@@ -828,8 +1045,8 @@ func (c *Conn) AskAs(module, key string, body any) (Answered, error) {
|
||||
}
|
||||
|
||||
// EventsStream is where every module's events land, and ConsumerOf the durable consumer the
|
||||
// controller makes for a module on a machine: the same names the module's own runtime bound
|
||||
// (node-tools/src/broker-nats.ts), so moving the module into the node's runtime neither loses an
|
||||
// mesh makes for a module on a machine: the same names the module's own runtime bound
|
||||
// (the retired TypeScript runtime's broker-nats.ts), so moving the module into the node's runtime neither loses an
|
||||
// event nor sees one twice.
|
||||
const EventsStream = "EVENTS"
|
||||
|
||||
@@ -843,7 +1060,7 @@ func ConsumerOf(node, module string) string {
|
||||
|
||||
// NakDelay is how long an event a handler failed waits before it is offered again: a transient cause
|
||||
// gets another attempt, a permanent one exhausts the consumer's max-deliver rather than spinning. The
|
||||
// consumer then gives the event up, and the controller keeps it in DEAD_LETTERS until a person delivers
|
||||
// consumer then gives the event up, and the mesh keeps it in DEAD_LETTERS until a person delivers
|
||||
// it again — under `mesh.again.<consumer>.…`, from which keyOf reads the same key — or drops it
|
||||
// (novox/hq issue 330).
|
||||
var NakDelay = 5 * time.Second
|
||||
@@ -854,64 +1071,122 @@ var NakDelay = 5 * time.Second
|
||||
// handed over, and a burst would end in max-deliver with nothing failed (novox/hq issue 387).
|
||||
var EventHeartbeat = 10 * time.Second
|
||||
|
||||
// ConsumeAs reads a module's durable consumer and hands each event to deliver (novox/hq ADR 0198):
|
||||
// acknowledged when deliver returns nil, negatively acknowledged after NakDelay when it returns an
|
||||
// error, terminated when it is not an event at all. The consumer is the controller's to create; this
|
||||
// binds to it and never makes one. Runs until stopped.
|
||||
// ConsumeAs reads a module's durable consumer on EVENTS, and each one on a module's stream its
|
||||
// membership names (novox/hq ADR 0363 §8), and hands each event to deliver one at a time (novox/hq
|
||||
// ADR 0198): acknowledged when deliver returns nil, negatively acknowledged after NakDelay when it
|
||||
// returns an error, terminated when it is not an event at all. The consumers are the mesh's to
|
||||
// create; this binds to them and never makes one. Runs until stopped.
|
||||
func (c *Conn) ConsumeAs(module string, deliver func(Envelope) error) (func(), error) {
|
||||
return c.ConsumeAsBeating(module, deliver, nil)
|
||||
}
|
||||
|
||||
// IdleHeartbeat is how often the runtime tells a bundle that its module's consumer is bound and the bus answers,
|
||||
// while no event arrives (stdio.OnHeartbeat), so a subscription's silence is told from a quiet source.
|
||||
var IdleHeartbeat = 15 * time.Second
|
||||
|
||||
// Beat is the runtime's word on a module's consumer between events: it is bound and the bus answers (At), or the
|
||||
// bus's heartbeat stopped arriving or the bus ended the loop (Missed, with Why). Its JSON is the stdio Heartbeat.
|
||||
type Beat struct {
|
||||
At time.Time `json:"at"`
|
||||
Missed bool `json:"missed,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// ConsumeAsBeating is ConsumeAs that also says, to beat, how the module's consumer on EVENTS stands: a Beat at each
|
||||
// IdleHeartbeat in which no event arrived and the bus answered about the consumer, and one Missed (said once) when
|
||||
// the bus's own heartbeat to the loop stopped arriving, the bus ended the loop or no longer answers. A Beat that is
|
||||
// not Missed follows the first answer after one that was. A nil beat says nothing.
|
||||
func (c *Conn) ConsumeAsBeating(module string, deliver func(Envelope) error, beat func(Beat)) (func(), error) {
|
||||
return c.ConsumeAsBeatingWhen(module, deliver, beat, nil)
|
||||
}
|
||||
|
||||
// tickHook, when set before a consume starts, is called after each idle heartbeat interval was handled (tests only,
|
||||
// through SetTickHook), so a test counts beats instead of sleeping. A consume reads it once, when it starts.
|
||||
var tickHook func()
|
||||
|
||||
// ConsumeAsBeatingWhen is ConsumeAsBeating that asks the bus about the consumer, and says anything, only while wanted
|
||||
// answers true (nil is always): a module whose children asked for no heartbeat costs the bus no consumer-info request.
|
||||
func (c *Conn) ConsumeAsBeatingWhen(module string, deliver func(Envelope) error, beat func(Beat), wanted func() bool) (func(), error) {
|
||||
asked := func() bool { return beat != nil && (wanted == nil || wanted()) }
|
||||
durable := ConsumerOf(c.node, module)
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
var lastEvent atomic.Int64
|
||||
var missed atomic.Bool
|
||||
say := func(why string) {
|
||||
if asked() && missed.CompareAndSwap(false, true) {
|
||||
beat(Beat{At: time.Now().UTC(), Missed: true, Why: why})
|
||||
}
|
||||
}
|
||||
h := newHearing(c, js, module, func(env Envelope) error {
|
||||
lastEvent.Store(time.Now().UnixNano())
|
||||
missed.Store(false) // an event arrived: the consumer is heard, whatever was said before
|
||||
return deliver(env)
|
||||
})
|
||||
// Bound by name, never created and never matched against a subject: the consumer and its
|
||||
// filters are the controller's (design 29 §3), exactly as the module's own runtime bound it.
|
||||
consumer, err := js.Consumer(ctx, EventsStream, durable)
|
||||
// filters are the mesh's (design 29 §3), exactly as the module's own runtime bound it.
|
||||
events, _, doing, err := h.bind(EventsStream, durable, func(err error) {
|
||||
if errors.Is(err, jetstream.ErrNoHeartbeat) {
|
||||
say("the bus's heartbeat to the consumer " + durable + " stopped arriving")
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("binding %s's consumer %s: %w", module, durable, err)
|
||||
return nil, fmt.Errorf("%s %s's consumer %s: %w", doing, module, durable, err)
|
||||
}
|
||||
reading, err := consumer.Consume(func(msg jetstream.Msg) {
|
||||
env, ok := envelopeOf(msg.Subject(), msg.Headers(), msg.Data())
|
||||
if !ok {
|
||||
// Unparseable: redelivering bytes no version of a handler can read is a loop.
|
||||
_ = msg.Term()
|
||||
return
|
||||
// And every consumer on a module's stream its membership names (novox/hq ADR 0363 §8), followed as the
|
||||
// membership changes. A membership naming none binds nothing more: the module hears EVENTS alone.
|
||||
h.start()
|
||||
if beat != nil {
|
||||
every := IdleHeartbeat
|
||||
hook := tickHook
|
||||
tickOnce := func() {
|
||||
select {
|
||||
case <-events.Closed():
|
||||
say("the bus ended reading the consumer " + durable)
|
||||
return
|
||||
default:
|
||||
}
|
||||
if !asked() {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), every)
|
||||
_, err := js.Consumer(ctx, EventsStream, durable)
|
||||
cancel()
|
||||
if err != nil {
|
||||
say("the bus does not answer about the consumer " + durable + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
if time.Since(time.Unix(0, lastEvent.Load())) < every {
|
||||
return // an event arrived in this interval: it was the word
|
||||
}
|
||||
missed.Store(false)
|
||||
beat(Beat{At: time.Now().UTC()})
|
||||
}
|
||||
// Kept alive while the bundle works on it, as a worker's work is (seat.go): the consumer's ack wait
|
||||
// (30 seconds) is for a runtime that died, not a handler that is slow — the runtime gives a handler
|
||||
// launch.EventTimeout. Without it an event handled longer than the ack wait was offered again while
|
||||
// it was handled, and counted toward max-deliver (novox/hq issue 387).
|
||||
done, stopped := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(stopped)
|
||||
tick := time.NewTicker(EventHeartbeat)
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
case <-h.done:
|
||||
return
|
||||
case <-tick.C:
|
||||
_ = msg.InProgress()
|
||||
}
|
||||
tickOnce()
|
||||
if hook != nil {
|
||||
hook()
|
||||
}
|
||||
}
|
||||
}()
|
||||
err := deliver(env)
|
||||
close(done)
|
||||
// No in-progress after the answer: one after a Nak would stretch its delay to the whole ack wait.
|
||||
<-stopped
|
||||
if err != nil {
|
||||
_ = msg.NakWithDelay(NakDelay)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
}, jetstream.PullMaxMessages(1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s's consumer %s: %w", module, durable, err)
|
||||
}
|
||||
var once sync.Once
|
||||
return func() { once.Do(reading.Stop) }, nil
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
h.stop()
|
||||
events.Stop()
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
|
||||
// envelopeOf rebuilds the envelope a module sees from a delivered event: its key is the emitter and
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A tool call names its caller (novox/hq issue 365, by ADR 0259 §3's precedent for asks).
|
||||
//
|
||||
// mesh.mod.<module>.call.<tool>.<caller> to whichever instance answers
|
||||
// mesh.mod.<module>.call.<tool>.<node>.<caller> to the instance on one machine
|
||||
// mesh.seat.<seat>.call.<verb>.<caller> a mesh seat's verb
|
||||
// mesh.seat.<seat>.call.<verb>.<node>.<caller> a node seat's verb on one machine
|
||||
//
|
||||
// The last token is the bus user that published the call, and the mesh grants each user the publish of
|
||||
// these subjects with its own name there and no other — so the caller is a fact the bus server enforces, and
|
||||
// the runtime hands it to the module from the subject a call arrived on, never from the payload.
|
||||
//
|
||||
// **A kind of its own, `call`, not the `tool` subject with a token appended.** Every grant to call a tool is a
|
||||
// wildcard over the `tool` kind — `mesh.mod.<m>.tool.<t>.*` for the instance on any machine, `mesh.mod.*.tool.>`
|
||||
// for every tool — and either matches a `tool` subject with any caller appended, so a caller could name
|
||||
// another. Under `call` no grant but the caller-named ones reaches the caller token.
|
||||
//
|
||||
// The `tool` subjects are served beside these for one release, and say Unproven, so callers move without a
|
||||
// gap; their retirement is hq issue 464.
|
||||
const CallKind = "call"
|
||||
|
||||
// A bus user's name carries dots (`<node>.<module>`, `person.<name>`); a subject token cannot. In the token
|
||||
// each dot is written `~`, which no part of a bus user's name may hold (the mesh refuses any part that
|
||||
// is not letters, digits, `_` or `-`), so the token is the user's name and nothing else.
|
||||
const userDot = "~"
|
||||
|
||||
var userName = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`)
|
||||
|
||||
// CallerToken is a bus user's name as the last token of a call it publishes, or "" for a name that is not a
|
||||
// bus user's.
|
||||
func CallerToken(user string) string {
|
||||
if !userName.MatchString(user) {
|
||||
return ""
|
||||
}
|
||||
return strings.ReplaceAll(user, ".", userDot)
|
||||
}
|
||||
|
||||
// UserOfToken is the bus user a caller token names, or "" when it names none.
|
||||
func UserOfToken(token string) string {
|
||||
user := strings.ReplaceAll(token, userDot, ".")
|
||||
if !userName.MatchString(user) {
|
||||
return ""
|
||||
}
|
||||
return user
|
||||
}
|
||||
|
||||
// toolShape splits a tool subject — `mesh.mod.<m>.tool.<rest>` or `mesh.seat.<s>.tool.<rest>` — into what
|
||||
// comes before the kind and what after; ok is false for any other subject.
|
||||
func toolShape(subject string) (head, rest string, ok bool) {
|
||||
parts := strings.SplitN(subject, ".", 5)
|
||||
if len(parts) != 5 || parts[0] != "mesh" || (parts[1] != "mod" && parts[1] != "seat") || parts[3] != "tool" ||
|
||||
parts[2] == "" || parts[4] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0] + "." + parts[1] + "." + parts[2], parts[4], true
|
||||
}
|
||||
|
||||
// CalledSubject is where a call to a tool subject goes naming its caller: the same address under the `call`
|
||||
// kind, the caller's token last. "" for a subject that is not a tool's, or a caller that is not a bus user.
|
||||
func CalledSubject(subject, user string) string {
|
||||
head, rest, ok := toolShape(subject)
|
||||
token := CallerToken(user)
|
||||
if !ok || token == "" || strings.ContainsAny(rest, "*>") {
|
||||
return ""
|
||||
}
|
||||
return head + "." + CallKind + "." + rest + "." + token
|
||||
}
|
||||
|
||||
// calledPattern is what a runtime answering a tool subject also subscribes, to hear the calls that name their
|
||||
// caller: the same address under the `call` kind, with any caller. "" for a subject that is not a tool's.
|
||||
func calledPattern(subject string) string {
|
||||
head, rest, ok := toolShape(subject)
|
||||
if !ok || strings.ContainsAny(rest, "*>") {
|
||||
return ""
|
||||
}
|
||||
return head + "." + CallKind + "." + rest + ".*"
|
||||
}
|
||||
|
||||
// callerOn is the bus user a call names, from the subject it arrived on: its last token, under a pattern
|
||||
// calledPattern made. "" — said by the module as Unproven — for a call on the subject that names no caller.
|
||||
func callerOn(pattern, subject string) string {
|
||||
// Only the `call` kind names a caller: a `tool` pattern ending in a wildcard (a node seat's verb on any
|
||||
// machine) is never read as one.
|
||||
parts := strings.SplitN(pattern, ".", 5)
|
||||
if len(parts) != 5 || parts[0] != "mesh" || (parts[1] != "mod" && parts[1] != "seat") || parts[3] != CallKind ||
|
||||
!strings.HasSuffix(pattern, ".*") {
|
||||
return ""
|
||||
}
|
||||
dot := strings.LastIndex(subject, ".")
|
||||
if dot < 0 || subject[:dot] != strings.TrimSuffix(pattern, ".*") {
|
||||
return ""
|
||||
}
|
||||
return UserOfToken(subject[dot+1:])
|
||||
}
|
||||
|
||||
// CalledHandler answers one call, told who called: the bus user the subject named, or "" when it named none.
|
||||
type CalledHandler func(caller string, body json.RawMessage) (any, error)
|
||||
|
||||
// callers is what a connection keeps of its caller-named calls: those in flight, by subject, so a refused
|
||||
// publish ends its call at once instead of after RequestTimeout.
|
||||
type callers struct {
|
||||
mu sync.Mutex
|
||||
// fellBack are the caller-named subjects already said to have reached nobody, so each is said once.
|
||||
fellBack map[string]bool
|
||||
next uint64
|
||||
pending map[string]map[uint64]context.CancelCauseFunc
|
||||
}
|
||||
|
||||
// errCallRefused is the bus refusing a caller-named publish: this user is not granted it yet (the mesh's
|
||||
// grant not delivered to the bus), so the call is asked again on the subject that names no caller.
|
||||
var errCallRefused = errors.New("the bus refused the caller-named subject")
|
||||
|
||||
var refusedPublish = regexp.MustCompile(`(?i)Publish to "(\S+)"`)
|
||||
|
||||
// publishRefused ends every call in flight on a subject the bus refused this connection the publish of.
|
||||
func (c *Conn) publishRefused(err error) bool {
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
return false
|
||||
}
|
||||
m := refusedPublish.FindStringSubmatch(err.Error())
|
||||
if len(m) < 2 {
|
||||
return false
|
||||
}
|
||||
c.calls.mu.Lock()
|
||||
cancels := c.calls.pending[m[1]]
|
||||
delete(c.calls.pending, m[1])
|
||||
c.calls.mu.Unlock()
|
||||
if len(cancels) > 0 {
|
||||
c.Logf("[mesh-tools] the bus refused this account the caller-named %s; asked again on the subject that "+
|
||||
"names no caller, until the mesh grants it (novox/hq issue 365)", m[1])
|
||||
}
|
||||
for _, cancel := range cancels {
|
||||
cancel(errCallRefused)
|
||||
}
|
||||
return len(cancels) > 0
|
||||
}
|
||||
|
||||
// askCalled asks once on a caller-named subject. fallBack says the call reached nobody — nothing serves that
|
||||
// subject yet, or the bus refused this user its publish — so it may be asked on the subject that names no
|
||||
// caller without being done twice.
|
||||
func (c *Conn) askCalled(subject string, data []byte) (answered Answered, fallBack bool, err error) {
|
||||
ctx, cancel := context.WithCancelCause(context.Background())
|
||||
ctx, stop := context.WithTimeout(ctx, RequestTimeout)
|
||||
defer stop()
|
||||
c.calls.mu.Lock()
|
||||
if c.calls.pending == nil {
|
||||
c.calls.pending = map[string]map[uint64]context.CancelCauseFunc{}
|
||||
}
|
||||
if c.calls.pending[subject] == nil {
|
||||
c.calls.pending[subject] = map[uint64]context.CancelCauseFunc{}
|
||||
}
|
||||
c.calls.next++
|
||||
id := c.calls.next
|
||||
c.calls.pending[subject][id] = cancel
|
||||
c.calls.mu.Unlock()
|
||||
defer func() {
|
||||
c.calls.mu.Lock()
|
||||
if list := c.calls.pending[subject]; list != nil {
|
||||
delete(list, id)
|
||||
if len(list) == 0 {
|
||||
delete(c.calls.pending, subject)
|
||||
}
|
||||
}
|
||||
c.calls.mu.Unlock()
|
||||
cancel(nil)
|
||||
}()
|
||||
msg, err := c.nc.RequestWithContext(ctx, subject, data)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(context.Cause(ctx), errCallRefused):
|
||||
return Answered{}, true, nil
|
||||
case errors.Is(err, nats.ErrNoResponders):
|
||||
c.saidFallBack(subject)
|
||||
return Answered{}, true, nil
|
||||
case errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout):
|
||||
return Answered{}, false, errors.New("timeout")
|
||||
}
|
||||
return Answered{}, false, err
|
||||
}
|
||||
answered, err = answerOf(msg.Data)
|
||||
return answered, false, err
|
||||
}
|
||||
|
||||
// saidFallBack says, once per subject, that a caller-named call reached nobody and is asked on the subject that
|
||||
// names no caller: a runtime or holder not yet serving it, or a grant that drifted, is seen in the log rather
|
||||
// than read later as a quiet "unproven" (novox/hq issue 365; expected until hq issue
|
||||
// 464).
|
||||
func (c *Conn) saidFallBack(subject string) {
|
||||
c.calls.mu.Lock()
|
||||
if c.calls.fellBack == nil {
|
||||
c.calls.fellBack = map[string]bool{}
|
||||
}
|
||||
first := !c.calls.fellBack[subject]
|
||||
c.calls.fellBack[subject] = true
|
||||
c.calls.mu.Unlock()
|
||||
if first {
|
||||
c.Logf("[mesh-tools] nothing answers the caller-named %s; asked on the subject that names no caller, where "+
|
||||
"the module is told no caller (said once per subject; novox/hq issue 365)", subject)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A bus whose users are granted as the controller grants them (mesh-module-assigner internal/broker): each named
|
||||
// user publishes the subjects listed for it.
|
||||
func grantedBus(t *testing.T, publish map[string][]string) string {
|
||||
t.Helper()
|
||||
var users []*server.User
|
||||
for user, allow := range publish {
|
||||
// And the membership's direct get every connection asks once, answered "no responders" here.
|
||||
allow = append(allow, "$JS.API.DIRECT.GET.>")
|
||||
users = append(users, &server.User{Username: user, Password: "pw", Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: allow},
|
||||
Subscribe: &server.SubjectPermission{Allow: []string{">"}},
|
||||
}})
|
||||
}
|
||||
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, NoLog: true, NoSigs: true,
|
||||
Users: users})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
return s.ClientURL()
|
||||
}
|
||||
|
||||
func dial(t *testing.T, url, module, node, user string) *bus.Conn {
|
||||
t.Helper()
|
||||
c, err := bus.Connect(bus.Credential{URL: url, Module: module, Node: node, User: user, Password: "pw"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Logf = func(string, ...any) {}
|
||||
t.Cleanup(c.Close)
|
||||
return c
|
||||
}
|
||||
|
||||
type heard struct {
|
||||
mu sync.Mutex
|
||||
callers []string
|
||||
}
|
||||
|
||||
func (h *heard) handler(caller string, _ json.RawMessage) (any, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.callers = append(h.callers, caller)
|
||||
return map[string]any{"caller": caller}, nil
|
||||
}
|
||||
|
||||
func (h *heard) all() []string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]string(nil), h.callers...)
|
||||
}
|
||||
|
||||
// A caller asks on the caller-named subject first; where nothing serves it yet — a runtime from before issue
|
||||
// 365 — the same call is asked once on the subject that names no caller, at once, and done once.
|
||||
func TestACallerFallsBackAtOnceWhereNothingServesTheCallerNamedSubject(t *testing.T) {
|
||||
url := mt.URL(t)
|
||||
server := dial(t, url, "ledger", "anchor", "")
|
||||
h := &heard{}
|
||||
// An older runtime: the subject that names no caller, alone.
|
||||
stop, err := server.HandleSubject("mesh.mod.ledger.tool.who", func(body json.RawMessage) (any, error) {
|
||||
return h.handler("", body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
server.Flush()
|
||||
asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools")
|
||||
began := time.Now()
|
||||
answered, err := asker.Ask("ledger.who", map[string]any{}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if took := time.Since(began); took > 2*time.Second {
|
||||
t.Errorf("the fall-back took %s", took)
|
||||
}
|
||||
if got := h.all(); len(got) != 1 || got[0] != "" || !strings.Contains(string(answered.Result), `"caller":""`) {
|
||||
t.Errorf("the call was answered %v, heard %v; want once, on the old subject", string(answered.Result), got)
|
||||
}
|
||||
}
|
||||
|
||||
// Where the bus does not grant the caller-named subject yet — the controller's grants not yet delivered — the
|
||||
// refusal ends the call at once and it is asked on the subject that names no caller: no caller waits out
|
||||
// RequestTimeout for a grant that is on its way.
|
||||
func TestACallerTheBusRefusesTheCallerNamedSubjectFallsBackAtOnce(t *testing.T) {
|
||||
url := grantedBus(t, map[string][]string{
|
||||
"anchor.node-tools": {"_INBOX.>"},
|
||||
"shanks.node-tools": {"mesh.mod.*.tool.>"},
|
||||
})
|
||||
server := dial(t, url, "node-tools", "anchor", "anchor.node-tools")
|
||||
h := &heard{}
|
||||
stop, err := server.HandleSubjectCalled("mesh.mod.ledger.tool.who", h.handler)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
server.Flush()
|
||||
asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools")
|
||||
var said []string
|
||||
var mu sync.Mutex
|
||||
asker.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
began := time.Now()
|
||||
if _, err := asker.Ask("ledger.who", map[string]any{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
if !strings.Contains(strings.Join(said, "\n"), "the bus refused this account the caller-named mesh.mod.ledger.call.who.shanks~node-tools") {
|
||||
t.Errorf("the refusal was not said: %v", said)
|
||||
}
|
||||
mu.Unlock()
|
||||
if took := time.Since(began); took > 3*time.Second {
|
||||
t.Errorf("the refused caller-named call held the caller %s", took)
|
||||
}
|
||||
if got := h.all(); len(got) != 1 || got[0] != "" {
|
||||
t.Errorf("heard %v; want one call, on the subject that names no caller", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With the controller's grant — the caller-named subjects with its own name last — a caller is heard as itself;
|
||||
// and a caller cannot publish on a subject naming another, however it asks.
|
||||
func TestAGrantedCallerIsHeardAsItselfAndCannotNameAnother(t *testing.T) {
|
||||
url := grantedBus(t, map[string][]string{
|
||||
"anchor.node-tools": {"_INBOX.>"},
|
||||
"shanks.node-tools": {"mesh.mod.*.call.*.shanks~node-tools", "mesh.mod.*.call.*.*.shanks~node-tools"},
|
||||
})
|
||||
server := dial(t, url, "node-tools", "anchor", "anchor.node-tools")
|
||||
h := &heard{}
|
||||
stop, err := server.HandleSubjectCalled("mesh.mod.ledger.tool.who", h.handler)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
server.Flush()
|
||||
asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools")
|
||||
if _, err := asker.Ask("ledger.who", map[string]any{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Naming another on the subject itself: refused by the bus, so nothing is heard.
|
||||
refusals := make(chan error, 4)
|
||||
forger, err := nats.Connect(url, nats.UserInfo("shanks.node-tools", "pw"),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer forger.Close()
|
||||
if err := forger.Publish("mesh.mod.ledger.call.who.novox~node-tools", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = forger.Flush()
|
||||
select {
|
||||
case err := <-refusals:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("the bus said %v, want a permissions violation", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("the bus did not refuse a call naming another caller")
|
||||
}
|
||||
if got := h.all(); len(got) != 1 || got[0] != "shanks.node-tools" {
|
||||
t.Errorf("heard %v; want the asker alone, as itself", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A fall-back for want of a responder on the caller-named subject is said, once per subject, so a runtime or
|
||||
// grant that drifted behind is seen rather than read as a quiet "unproven".
|
||||
func TestAFallBackForWantOfAResponderIsSaidOncePerSubject(t *testing.T) {
|
||||
url := mt.URL(t)
|
||||
// A runtime from before issue 365: the subject that names no caller, alone.
|
||||
older, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer older.Close()
|
||||
if _, err := older.Subscribe("mesh.mod.ledger.tool.who", func(m *nats.Msg) { _ = m.Respond([]byte(`{"result":{}}`)) }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = older.Flush()
|
||||
asker := dial(t, url, "node-tools", "shanks", "shanks.node-tools")
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
asker.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := asker.Ask("ledger.who", map[string]any{}, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, "nothing answers the caller-named mesh.mod.ledger.call.who.shanks~node-tools") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("the fall-back was said %d times, want once: %v", n, said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package bus
|
||||
|
||||
import "testing"
|
||||
|
||||
// A call names its caller under the `call` kind, the caller's bus user its last token (novox/hq issue 365).
|
||||
func TestACallerNamedSubjectIsTheToolsAddressUnderCallWithTheCallerLast(t *testing.T) {
|
||||
for subject, want := range map[string]string{
|
||||
"mesh.mod.ledger.tool.who": "mesh.mod.ledger.call.who.shanks~node-tools",
|
||||
"mesh.mod.ledger.tool.who.anchor": "mesh.mod.ledger.call.who.anchor.shanks~node-tools",
|
||||
"mesh.seat.issue-tracker.tool.open": "mesh.seat.issue-tracker.call.open.shanks~node-tools",
|
||||
"mesh.seat.node-desk.tool.who.anchor": "mesh.seat.node-desk.call.who.anchor.shanks~node-tools",
|
||||
"mesh.mod.ledger.event.opened": "",
|
||||
"mesh.seat.intake.accept.ask.x": "",
|
||||
"mesh.mod.ledger.tool.>": "",
|
||||
"mesh.mod.*.tool.who": "mesh.mod.*.call.who.shanks~node-tools",
|
||||
"mesh.node.anchor.ask.report": "",
|
||||
"mesh.seat.operator-channel.proof.x.y": "",
|
||||
} {
|
||||
if subject == "mesh.mod.*.tool.who" {
|
||||
continue // a wildcard is never asked on
|
||||
}
|
||||
if got := CalledSubject(subject, "shanks.node-tools"); got != want {
|
||||
t.Errorf("%s → %q, want %q", subject, got, want)
|
||||
}
|
||||
}
|
||||
for _, user := range []string{"", "a b", "x.*", "x.>", "a~b", "a..b"} {
|
||||
if got := CalledSubject("mesh.mod.ledger.tool.who", user); got != "" {
|
||||
t.Errorf("a caller %q that is not a bus user was named: %s", user, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The caller is read back from the subject a call arrived on, and only under the pattern that names one.
|
||||
func TestTheCallerIsTheLastTokenOfTheSubjectTheCallArrivedOn(t *testing.T) {
|
||||
pattern := calledPattern("mesh.mod.ledger.tool.who.anchor")
|
||||
if pattern != "mesh.mod.ledger.call.who.anchor.*" {
|
||||
t.Fatalf("pattern %q", pattern)
|
||||
}
|
||||
for subject, want := range map[string]string{
|
||||
"mesh.mod.ledger.call.who.anchor.shanks~node-tools": "shanks.node-tools",
|
||||
"mesh.mod.ledger.call.who.anchor.controller": "controller",
|
||||
"mesh.mod.ledger.call.who.anchor.person~jochen": "person.jochen",
|
||||
"mesh.mod.ledger.call.who.anchor.a~~b": "",
|
||||
"mesh.mod.ledger.call.what.anchor.controller": "",
|
||||
} {
|
||||
if got := callerOn(pattern, subject); got != want {
|
||||
t.Errorf("%s → %q, want %q", subject, got, want)
|
||||
}
|
||||
}
|
||||
if got := callerOn("mesh.mod.ledger.tool.who.anchor", "mesh.mod.ledger.tool.who.anchor"); got != "" {
|
||||
t.Errorf("a call on the subject that names no caller was handed %q", got)
|
||||
}
|
||||
for _, user := range []string{"controller", "shanks.node-tools", "person.jochen", "node.ace"} {
|
||||
if back := UserOfToken(CallerToken(user)); back != user {
|
||||
t.Errorf("%s → %s → %s", user, CallerToken(user), back)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only a pattern of the `call` kind names a caller: a `tool` subscription with a wildcard last — a node seat's
|
||||
// verb on any machine, say — never yields one, whatever token a call arrived with.
|
||||
func TestAToolPatternNeverYieldsACaller(t *testing.T) {
|
||||
for pattern, subject := range map[string]string{
|
||||
"mesh.mod.ledger.tool.who.*": "mesh.mod.ledger.tool.who.controller",
|
||||
"mesh.seat.node-desk.tool.who.*": "mesh.seat.node-desk.tool.who.shanks~node-tools",
|
||||
"mesh.mod.ledger.event.*": "mesh.mod.ledger.event.controller",
|
||||
} {
|
||||
if got := callerOn(pattern, subject); got != "" {
|
||||
t.Errorf("%s on %s yielded caller %q", pattern, subject, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -60,3 +61,96 @@ func TestASlowHandlerAndABurstAreEachHandedOverOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The runtime tells a bundle how its module's consumer stands between events (stdio.OnHeartbeat): a word at each
|
||||
// idle heartbeat while the bus answers, once Missed when the bus ended the loop, and a word again when it is bound again.
|
||||
func TestAConsumerSaysItIsBoundWhileIdleAndOnceWhenTheBusEndedIt(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
restore := bus.IdleHeartbeat
|
||||
bus.IdleHeartbeat = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.IdleHeartbeat = restore })
|
||||
ticks := make(chan struct{}, 64)
|
||||
t.Cleanup(bus.SetTickHook(func() {
|
||||
select {
|
||||
case ticks <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}))
|
||||
mesh.Consumer(t, "anchor", "quiet", []string{"mesh.mod.alpha.event.>"}, time.Second)
|
||||
c, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Logf = func(string, ...any) {}
|
||||
t.Cleanup(c.Close)
|
||||
|
||||
var mu sync.Mutex
|
||||
var beats []bus.Beat
|
||||
stop, err := c.ConsumeAsBeating("quiet", func(bus.Envelope) error { return nil }, func(b bus.Beat) {
|
||||
mu.Lock()
|
||||
beats = append(beats, b)
|
||||
mu.Unlock()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
// A consume nobody asked a heartbeat of says nothing and asks the bus nothing, however many intervals pass.
|
||||
var unaskedBeats atomic.Int32
|
||||
unasked, err := c.ConsumeAsBeatingWhen("quiet", func(bus.Envelope) error { return nil },
|
||||
func(bus.Beat) { unaskedBeats.Add(1) }, func() bool { return false })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(unasked)
|
||||
last := func() (bus.Beat, int) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(beats) == 0 {
|
||||
return bus.Beat{}, 0
|
||||
}
|
||||
return beats[len(beats)-1], len(beats)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
if b, n := last(); n == 0 || b.Missed || b.At.IsZero() {
|
||||
return fmt.Errorf("no word that the consumer is bound yet: %d beats", n)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
mesh.RemoveConsumer(t, bus.EventsStream, "anchor", "quiet")
|
||||
mt.Until(t, func() error {
|
||||
if b, _ := last(); !b.Missed || b.Why == "" {
|
||||
return fmt.Errorf("the consumer is gone from the bus and no Missed word came: %+v", b)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
mu.Lock()
|
||||
missed := 0
|
||||
for _, b := range beats {
|
||||
if b.Missed {
|
||||
missed++
|
||||
}
|
||||
}
|
||||
mu.Unlock()
|
||||
for n := 0; n < 3; n++ { // three more intervals pass, counted by the loop's own tick
|
||||
select {
|
||||
case <-ticks:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the heartbeat loop handled no more intervals")
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
after := 0
|
||||
for _, b := range beats {
|
||||
if b.Missed {
|
||||
after++
|
||||
}
|
||||
}
|
||||
if unaskedBeats.Load() != 0 {
|
||||
t.Errorf("a consume with no heartbeat asked said %d words", unaskedBeats.Load())
|
||||
}
|
||||
if after != missed || missed != 1 {
|
||||
t.Errorf("Missed is said once, not at every beat: %d then %d", missed, after)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
package bus
|
||||
|
||||
// SetTickHook makes the next consumes call f after each idle heartbeat interval was handled; the restore puts it back.
|
||||
func SetTickHook(f func()) (restore func()) {
|
||||
old := tickHook
|
||||
tickHook = f
|
||||
return func() { tickHook = old }
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// **A call the controller refused because it is handing over is asked once more** (novox/hq issue 289):
|
||||
// the controller being replaced did nothing and marked its refusal, and the one after it answers. A
|
||||
// refusal not so marked is final, and a second handover refusal is said, not asked a third time.
|
||||
func TestAHandoverRefusalIsAskedOnceMore(t *testing.T) {
|
||||
was := bus.HandoverPause
|
||||
bus.HandoverPause = 10 * time.Millisecond
|
||||
t.Cleanup(func() { bus.HandoverPause = was })
|
||||
|
||||
url := meshtest.URL(t)
|
||||
seat, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer seat.Close()
|
||||
c, err := bus.Connect(bus.Credential{URL: url, Module: "console", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
serve := func(t *testing.T, verb string, answers ...map[string]any) *atomic.Int32 {
|
||||
t.Helper()
|
||||
var asked atomic.Int32
|
||||
sub, err := seat.Subscribe(bus.SeatToolSubject("mesh-controller", verb, "", ""), func(m *nats.Msg) {
|
||||
n := int(asked.Add(1)) - 1
|
||||
if n >= len(answers) {
|
||||
n = len(answers) - 1
|
||||
}
|
||||
body, _ := json.Marshal(answers[n])
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
_ = seat.Flush()
|
||||
return &asked
|
||||
}
|
||||
handingOver := map[string]any{"error": "the controller is handing over to the next one; nothing was done, ask again",
|
||||
"retry": bus.RetryHandingOver}
|
||||
|
||||
asked := serve(t, "rotate", handingOver, map[string]any{"result": "rotated"})
|
||||
got, err := c.Ask("seat:mesh-controller.rotate", map[string]any{}, "")
|
||||
if err != nil || string(got.Result) != `"rotated"` || asked.Load() != 2 {
|
||||
t.Fatalf("after a handover refusal: %s %v, asked %d times", got.Result, err, asked.Load())
|
||||
}
|
||||
|
||||
asked = serve(t, "push", map[string]any{"error": "refused for its own reason"})
|
||||
if _, err := c.Ask("seat:mesh-controller.push", map[string]any{}, ""); err == nil || asked.Load() != 1 {
|
||||
t.Fatalf("an unmarked refusal was asked again: %v, asked %d times", err, asked.Load())
|
||||
}
|
||||
|
||||
asked = serve(t, "status", handingOver)
|
||||
_, err = c.Ask("seat:mesh-controller.status", map[string]any{}, "")
|
||||
if err == nil || asked.Load() != 2 || !strings.Contains(err.Error(), "both times refused as a handover") {
|
||||
t.Fatalf("two handover refusals: %v, asked %d times", err, asked.Load())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// heardMsg is a delivered event as a consumer's loop hands it, saying how it was settled.
|
||||
type heardMsg struct {
|
||||
jetstream.Msg
|
||||
mu sync.Mutex
|
||||
settled []string
|
||||
}
|
||||
|
||||
func (m *heardMsg) Subject() string { return "mesh.mod.alpha.event.happened" }
|
||||
func (m *heardMsg) Headers() nats.Header { return nats.Header{} }
|
||||
func (m *heardMsg) Data() []byte { return []byte(`{"n":1}`) }
|
||||
func (m *heardMsg) InProgress() error { return nil }
|
||||
func (m *heardMsg) Ack() error { return m.say("ack") }
|
||||
func (m *heardMsg) Nak() error { return m.say("nak") }
|
||||
func (m *heardMsg) NakWithDelay(time.Duration) error { return m.say("nak-delayed") }
|
||||
func (m *heardMsg) Term() error { return m.say("term") }
|
||||
func (m *heardMsg) say(how string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.settled = append(m.settled, how)
|
||||
return nil
|
||||
}
|
||||
|
||||
// An event the bus hands a hearing after it was stopped (a module's bundle loaded again, issue 1945) is
|
||||
// offered again at once and never handed to the module: the module's handler that heard it may be gone.
|
||||
func TestAHearingStoppedHandsNoEventOnAndOffersItAgain(t *testing.T) {
|
||||
delivered := 0
|
||||
h := newHearing(nil, nil, "alpha", func(Envelope) error { delivered++; return nil })
|
||||
h.stop()
|
||||
msg := &heardMsg{}
|
||||
h.handle(msg)
|
||||
if delivered != 0 {
|
||||
t.Errorf("a stopped hearing handed the event to the module")
|
||||
}
|
||||
if len(msg.settled) != 1 || msg.settled[0] != "nak" {
|
||||
t.Errorf("a stopped hearing settled the event as %v, want it offered again at once (nak)", msg.settled)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/ledger"
|
||||
)
|
||||
|
||||
// A ledger on the bus (novox/hq ADR 0597, design 25 § 12.4; issue 1966): its history a stream with a subject per
|
||||
// instance under mesh-sdk's buscompose.LedgerHistorySubject, its state a key-value bucket, its snapshots a subject per
|
||||
// instance under buscompose.LedgerSnapshotSubject, the latest only (the stream that holds them keeps one per subject, as the bus module
|
||||
// creates it). **Only the module's ledger connection publishes on its history and its snapshots** (ADR 0597 §9): what
|
||||
// is read, and the state, go on the runner's own.
|
||||
|
||||
// LedgerStores are the stores of a module's ledger binding: reads and the state on c, the history's appends and the
|
||||
// snapshots' writes on writer, the module's ledger connection. The subjects are mesh-sdk's, the ones the grants part
|
||||
// grants the module's ledger account byte for byte; a binding naming others is refused, naming both.
|
||||
func (c *Conn) LedgerStores(module string, b membership.BusBinding, writer *Conn) (ledger.History, ledger.State, ledger.Snapshots, error) {
|
||||
history, snapshot := buscompose.LedgerHistorySubject(module, b.Local), buscompose.LedgerSnapshotSubject(module, b.Local)
|
||||
for _, named := range []struct{ what, bound, want string }{{"history", b.Subject, history}, {"snapshot", b.Snapshot, snapshot}} {
|
||||
if named.bound != "" && prefixOf(named.bound) != named.want {
|
||||
return nil, nil, nil, fmt.Errorf("%s's ledger %q is bound with the %s subject %s, and its ledger account is granted "+
|
||||
"%s alone (mesh-sdk buscompose); the binding is refused", module, b.Local, named.what, named.bound, named.want)
|
||||
}
|
||||
}
|
||||
return &busHistory{read: c, write: writer, stream: b.History, prefix: history},
|
||||
&busState{c: c, bucket: b.State},
|
||||
&busSnapshots{read: c, write: writer, prefix: snapshot}, nil
|
||||
}
|
||||
|
||||
// prefixOf is a binding's subject as the prefix an instance's key is added to: written with or without its `.>`.
|
||||
func prefixOf(subject string) string {
|
||||
return strings.TrimSuffix(strings.TrimSuffix(subject, ">"), ".")
|
||||
}
|
||||
|
||||
// publishExpecting publishes body on subject on the connection, written only when the subject's last message is still
|
||||
// at expect (0: it has none); ledger.ErrRaced when it is not.
|
||||
func publishExpecting(c *Conn, subject string, body []byte, expect uint64) (uint64, error) {
|
||||
msg := nats.NewMsg(subject)
|
||||
msg.Data = body
|
||||
msg.Header.Set(jetstream.ExpectedLastSubjSeqHeader, strconv.FormatUint(expect, 10))
|
||||
reply, err := c.nc.RequestMsg(msg, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return 0, fmt.Errorf("no stream on the bus keeps %s", subject)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var ack struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Error *struct {
|
||||
ErrCode uint16 `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &ack); err != nil {
|
||||
return 0, fmt.Errorf("the bus answered the write to %s with what is not an acknowledgement: %q", subject, reply.Data)
|
||||
}
|
||||
if ack.Error != nil {
|
||||
if code := jetstream.ErrorCode(ack.Error.ErrCode); code == jetstream.JSErrCodeStreamWrongLastSequence ||
|
||||
code == jetstream.JSErrCodeStreamWrongLastSequenceConstant {
|
||||
return 0, ledger.ErrRaced
|
||||
}
|
||||
return 0, fmt.Errorf("the bus refused the write to %s: %s (%d)", subject, ack.Error.Description, ack.Error.ErrCode)
|
||||
}
|
||||
return ack.Seq, nil
|
||||
}
|
||||
|
||||
// lastOn is the last message on a subject of the stream that keeps it: its bytes and sequence, nil and 0 for none.
|
||||
func lastOn(c *Conn, stream, subject string) ([]byte, uint64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
js, err := c.jetStream()
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if stream == "" {
|
||||
if stream, err = js.StreamNameBySubject(ctx, subject); err != nil {
|
||||
return nil, 0, fmt.Errorf("no stream on the bus keeps %s: %w", subject, err)
|
||||
}
|
||||
}
|
||||
s, err := js.Stream(ctx, stream)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("the stream %s: %w", stream, err)
|
||||
}
|
||||
m, err := s.GetLastMsgForSubject(ctx, subject)
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return m.Data, m.Sequence, nil
|
||||
}
|
||||
|
||||
type busHistory struct {
|
||||
read, write *Conn
|
||||
stream, prefix string
|
||||
}
|
||||
|
||||
func (h *busHistory) Last(key string) (uint64, error) {
|
||||
_, seq, err := lastOn(h.read, h.stream, h.prefix+"."+key)
|
||||
return seq, err
|
||||
}
|
||||
|
||||
func (h *busHistory) Append(key string, op []byte, expect uint64) (uint64, error) {
|
||||
if h.write == nil {
|
||||
return 0, errors.New("the ledger's history is published only on its module's ledger connection, and none is held")
|
||||
}
|
||||
return publishExpecting(h.write, h.prefix+"."+key, op, expect)
|
||||
}
|
||||
|
||||
func (h *busHistory) Read(keys string, from uint64, limit int) ([]ledger.Kept, error) {
|
||||
var out []ledger.Kept
|
||||
for {
|
||||
want := StreamReadMost
|
||||
if limit > 0 && limit-len(out) < want {
|
||||
want = limit - len(out)
|
||||
}
|
||||
page, err := h.read.readStream(h.stream, h.prefix+"."+keys, from, want)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, m := range page.Messages {
|
||||
out = append(out, ledger.Kept{Sequence: m.Sequence, Data: m.Body})
|
||||
}
|
||||
if page.Next == 0 || (limit > 0 && len(out) >= limit) {
|
||||
return out, nil
|
||||
}
|
||||
from = page.Next
|
||||
}
|
||||
}
|
||||
|
||||
type busState struct {
|
||||
c *Conn
|
||||
bucket string
|
||||
}
|
||||
|
||||
func (s *busState) kv(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
js, err := s.c.jetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, s.bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, fmt.Errorf("the ledger's state %s is not on the bus", s.bucket)
|
||||
}
|
||||
return kv, err
|
||||
}
|
||||
|
||||
func (s *busState) Get(key string) ([]byte, uint64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := s.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value(), e.Revision(), nil
|
||||
}
|
||||
|
||||
// isFull is a write a bucket or stream refused because it holds as much as it may.
|
||||
func isFull(err error) bool {
|
||||
msg := err.Error()
|
||||
return strings.Contains(msg, "maximum bytes exceeded") || strings.Contains(msg, "maximum messages exceeded") ||
|
||||
strings.Contains(msg, "insufficient resources")
|
||||
}
|
||||
|
||||
func (s *busState) Write(key string, value []byte, revision uint64) (uint64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := s.kv(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var rev uint64
|
||||
if revision == 0 {
|
||||
rev, err = kv.Create(ctx, key, value)
|
||||
} else {
|
||||
rev, err = kv.Update(ctx, key, value, revision)
|
||||
}
|
||||
switch {
|
||||
case err == nil:
|
||||
return rev, nil
|
||||
case errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) || isWrongSequence(err):
|
||||
return 0, ledger.ErrChanged
|
||||
case isFull(err):
|
||||
return 0, fmt.Errorf("%w: %v", ledger.ErrFull, err)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
func (s *busState) Keys() ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := s.kv(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
type busSnapshots struct {
|
||||
read, write *Conn
|
||||
prefix string
|
||||
}
|
||||
|
||||
func (s *busSnapshots) Get(key string) ([]byte, uint64, error) {
|
||||
return lastOn(s.read, "", s.prefix+"."+key)
|
||||
}
|
||||
|
||||
func (s *busSnapshots) Put(key string, value []byte, expect uint64) error {
|
||||
if s.write == nil {
|
||||
return errors.New("a ledger's snapshot is published only on its module's ledger connection, and none is held")
|
||||
}
|
||||
_, err := publishExpecting(s.write, s.prefix+"."+key, value, expect)
|
||||
return err
|
||||
}
|
||||
|
||||
// Watch hands deliver every instance's record under a key pattern as it is, then each change; a removed key is handed as
|
||||
// nil.
|
||||
func (s *busState) Watch(pattern string, deliver func(key string, value []byte, current bool) error) (func(), error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
kv, err := s.kv(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w, err := kv.Watch(context.Background(), pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hand := func(e jetstream.KeyValueEntry, current bool) {
|
||||
var value []byte
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
value = e.Value()
|
||||
}
|
||||
if err := deliver(e.Key(), value, current); err != nil {
|
||||
s.c.Logf("[mesh-tools] a ledger watch of %s was not taken: %v", s.bucket, err)
|
||||
}
|
||||
}
|
||||
timer := time.NewTimer(StateTimeout)
|
||||
defer timer.Stop()
|
||||
for current := true; current; {
|
||||
select {
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return nil, errors.New("the ledger's state watch ended before its current states were handed over")
|
||||
}
|
||||
if e == nil {
|
||||
current = false
|
||||
continue
|
||||
}
|
||||
hand(e, true)
|
||||
case <-timer.C:
|
||||
_ = w.Stop()
|
||||
return nil, fmt.Errorf("the ledger's state %s did not hand its current states in %s", s.bucket, StateTimeout)
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
for e := range w.Updates() {
|
||||
if e != nil {
|
||||
hand(e, false)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { _ = w.Stop() }, nil
|
||||
}
|
||||
|
||||
// LedgerIndexes are the buckets of a ledger binding's declared indexes, by index name, on the runner's own connection.
|
||||
func (c *Conn) LedgerIndexes(b membership.BusBinding) map[string]ledger.Index {
|
||||
out := map[string]ledger.Index{}
|
||||
for name, bucket := range b.Indexes {
|
||||
out[name] = &busState{c: c, bucket: bucket}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *busState) Entries(pattern string, limit int) ([]ledger.Indexed, error) {
|
||||
if pattern == "" {
|
||||
pattern = ">"
|
||||
}
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var out []ledger.Indexed
|
||||
for _, k := range keys {
|
||||
if limit > 0 && len(out) >= limit {
|
||||
break
|
||||
}
|
||||
if !SubjectMatches(pattern, k) {
|
||||
continue
|
||||
}
|
||||
v, _, err := s.Get(k)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if v != nil {
|
||||
out = append(out, ledger.Indexed{Key: k, Value: v})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *busState) Put(key string, value []byte) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := s.kv(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = kv.Put(ctx, key, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *busState) Delete(key string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := s.kv(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,491 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's log on the bus (novox/hq ADR 0297): a stream of operations, one subject per key, that the
|
||||
// mesh creates from what the module declared under `logs` and issues to the module's own instances
|
||||
// in its membership. It mirrors a bucket (ADR 0201) in how it is reached: what a module may reach is
|
||||
// decided here, from its membership, and refused with the reason before anything is sent.
|
||||
//
|
||||
// **The runtime uses no consumer of a log.** The bus grants it, per issued log, only the publish of an
|
||||
// entry, the stream's info and the direct get: an append is a publish, a read is a batch direct get
|
||||
// (NATS 2.11), the last entry of a key is a direct get by its subject.
|
||||
|
||||
// LogIssued is one log an assignment may reach (ADR 0297): its owner's alone, so there are no reads.
|
||||
type LogIssued struct {
|
||||
Name string `json:"name"`
|
||||
Stream string `json:"stream"`
|
||||
Subject string `json:"subject"`
|
||||
Writes bool `json:"writes,omitempty"`
|
||||
}
|
||||
|
||||
// LogEntry is one operation in a log: its sequence is its identity.
|
||||
type LogEntry struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// LogPage is one page of a read: Next is the sequence to continue from, 0 when nothing follows; Last
|
||||
// is the stream's last sequence.
|
||||
type LogPage struct {
|
||||
Entries []LogEntry `json:"entries"`
|
||||
Next uint64 `json:"next"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
// LogInfo is how full a log is.
|
||||
type LogInfo struct {
|
||||
Entries uint64 `json:"entries"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
MaxBytes uint64 `json:"max_bytes"`
|
||||
First uint64 `json:"first"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
const (
|
||||
// LogValueMax is the most one entry's value may hold (ADR 0297 §3); the stream's MaxMsgSize is 4 KiB
|
||||
// more, for the headers an append carries.
|
||||
LogValueMax = 256 * 1024
|
||||
// LogKeyMax is the longest key.
|
||||
LogKeyMax = 255
|
||||
// LogReadDefault and LogReadMax are a page's entries when none is asked, and at most.
|
||||
LogReadDefault = 100
|
||||
LogReadMax = 1000
|
||||
// LogPageBytes bounds the bytes of one page.
|
||||
LogPageBytes = 4 * 1024 * 1024
|
||||
)
|
||||
|
||||
// logToken is one name of a key: letters, digits, `_` and `-`.
|
||||
var logToken = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
|
||||
func checkLogKey(key string) error {
|
||||
if key == "" || len(key) > LogKeyMax {
|
||||
return fmt.Errorf("%q is not a log key: one or more names of letters, digits, _ and -, joined by "+
|
||||
"dots, at most %d bytes (novox/hq ADR 0297)", key, LogKeyMax)
|
||||
}
|
||||
for _, t := range strings.Split(key, ".") {
|
||||
if !logToken.MatchString(t) {
|
||||
return fmt.Errorf("%q is not a log key: one or more names of letters, digits, _ and -, joined by "+
|
||||
"dots, at most %d bytes (novox/hq ADR 0297)", key, LogKeyMax)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// logFilter is a read's key pattern as a subject filter under the log's subject: `*` for one name, `>`
|
||||
// for the rest; empty is every key.
|
||||
func logFilter(l LogIssued, pattern string) (string, error) {
|
||||
if pattern == "" {
|
||||
return l.Subject + ".>", nil
|
||||
}
|
||||
parts := strings.Split(pattern, ".")
|
||||
for i, p := range parts {
|
||||
if p == "**" && i == len(parts)-1 {
|
||||
// `**` for the rest, as a state's key pattern and mesh-sdk's log reader say it.
|
||||
parts[i], p = ">", ">"
|
||||
}
|
||||
switch {
|
||||
case p == ">" && i == len(parts)-1, p == "*", logToken.MatchString(p):
|
||||
default:
|
||||
return "", fmt.Errorf("%q is not a key pattern: names, `*` for one and `>` for the rest, last "+
|
||||
"(novox/hq ADR 0297)", pattern)
|
||||
}
|
||||
}
|
||||
if len(pattern) > LogKeyMax {
|
||||
return "", fmt.Errorf("%q is longer than a key can be (%d bytes)", pattern, LogKeyMax)
|
||||
}
|
||||
return l.Subject + "." + strings.Join(parts, "."), nil
|
||||
}
|
||||
|
||||
// issuedLog is the log a module may reach by a name, or the reason it may not.
|
||||
func (c *Conn) issuedLog(module, name string) (LogIssued, error) {
|
||||
m := c.Membership(module)
|
||||
if m == nil {
|
||||
return LogIssued{}, fmt.Errorf("%s has no membership issued on %s yet, so no log of it is reachable "+
|
||||
"until the mesh issues one (novox/hq ADR 0297)", module, c.node)
|
||||
}
|
||||
var names []string
|
||||
for _, l := range m.Logs {
|
||||
if l.Name == name {
|
||||
return l, nil
|
||||
}
|
||||
names = append(names, l.Name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
issued := "none"
|
||||
if len(names) > 0 {
|
||||
issued = strings.Join(names, ", ")
|
||||
}
|
||||
return LogIssued{}, fmt.Errorf("%s keeps no log called %q: a module declares the logs it keeps under `logs`, and "+
|
||||
"another module's it reads under `reads` as <module>.<log>; it was issued: %s (novox/hq ADR 0297, ADR 0571)",
|
||||
module, name, issued)
|
||||
}
|
||||
|
||||
// notOnTheBus says a log is issued and its stream is not there yet.
|
||||
func notOnTheBus(l LogIssued) error {
|
||||
return fmt.Errorf("the log %q is issued and its stream %s is not on the bus yet: the mesh creates "+
|
||||
"it from the catalogue on its next raise (novox/hq ADR 0297)", l.Name, l.Stream)
|
||||
}
|
||||
|
||||
// LogAppend writes one entry for a key, as the module, and answers its sequence. With expect, it is
|
||||
// written only when the key's last entry is still at that sequence (0: the key has none); otherwise it
|
||||
// is refused as a lost compare-and-set, the same error a lost state update answers (ADR 0259).
|
||||
func (c *Conn) LogAppend(module, name, key string, value json.RawMessage, expect *uint64) (uint64, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !l.Writes {
|
||||
return 0, fmt.Errorf("%s was issued the log %s without writing it (novox/hq ADR 0297)", module, name)
|
||||
}
|
||||
if err := checkLogKey(key); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(value) == 0 || !json.Valid(value) {
|
||||
return 0, fmt.Errorf("a log entry's value is JSON (novox/hq ADR 0297)")
|
||||
}
|
||||
if len(value) > LogValueMax {
|
||||
return 0, fmt.Errorf("a log entry's value is at most %d bytes (256 KiB), and this one is %d (novox/hq ADR 0297)",
|
||||
LogValueMax, len(value))
|
||||
}
|
||||
if field := credentialField(value); field != "" {
|
||||
return 0, fmt.Errorf("%s's %s.%s carries a field %q, which names a credential: no secret is kept in a log, "+
|
||||
"which keeps it for good. Name the secret and fetch it on request/reply (novox/hq ADR 0201 §13, ADR 0297)",
|
||||
module, name, key, field)
|
||||
}
|
||||
msg := nats.NewMsg(l.Subject + "." + key)
|
||||
msg.Data = value
|
||||
if expect != nil {
|
||||
msg.Header.Set(jetstream.ExpectedLastSubjSeqHeader, strconv.FormatUint(*expect, 10))
|
||||
}
|
||||
reply, err := c.nc.RequestMsg(msg, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return 0, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var ack struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
ErrCode uint16 `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &ack); err != nil {
|
||||
return 0, fmt.Errorf("the bus answered the append to %s with what is not an acknowledgement: %q", name, reply.Data)
|
||||
}
|
||||
if ack.Error != nil {
|
||||
// The key's last entry is no longer the one expected: a lost compare-and-set, answered as a lost
|
||||
// state update is (10164 is the same refusal from a replicated stream).
|
||||
if code := jetstream.ErrorCode(ack.Error.ErrCode); code == jetstream.JSErrCodeStreamWrongLastSequence ||
|
||||
code == jetstream.JSErrCodeStreamWrongLastSequenceConstant {
|
||||
return 0, changed{name + "." + key}
|
||||
}
|
||||
return 0, fmt.Errorf("the bus refused the append to %s.%s: %s (%d)", name, key, ack.Error.Description, ack.Error.ErrCode)
|
||||
}
|
||||
if ack.Seq == 0 {
|
||||
return 0, fmt.Errorf("the bus answered the append to %s.%s with no sequence", name, key)
|
||||
}
|
||||
return ack.Seq, nil
|
||||
}
|
||||
|
||||
// LogRead is the entries from a sequence on, or from a time on (since, when not zero), of the keys matching a pattern, at
|
||||
// most limit of them and about 4 MiB, by one batch direct get. A log another module keeps is read as its own is, by the
|
||||
// name its reads declare, `<module>.<log>` (novox/hq ADR 0571).
|
||||
func (c *Conn) LogRead(module, name string, from uint64, since time.Time, keys string, limit int) (*LogPage, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case limit == 0:
|
||||
limit = LogReadDefault
|
||||
case limit < 0 || limit > LogReadMax:
|
||||
return nil, fmt.Errorf("a read's limit is 1 to %d entries, %d when none is given, not %d (novox/hq ADR 0297)",
|
||||
LogReadMax, LogReadDefault, limit)
|
||||
}
|
||||
filter, err := logFilter(l, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if from != 0 && !since.IsZero() {
|
||||
return nil, fmt.Errorf("a read of %s starts from a sequence or from a time, not both", name)
|
||||
}
|
||||
if from == 0 {
|
||||
from = 1
|
||||
}
|
||||
req := map[string]any{"seq": from, "next_by_subj": filter, "batch": limit, "max_bytes": LogPageBytes}
|
||||
if !since.IsZero() {
|
||||
delete(req, "seq")
|
||||
req["start_time"] = since.UTC()
|
||||
}
|
||||
entries, more, err := c.logBatch(l, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page := &LogPage{Entries: entries}
|
||||
if more && len(entries) > 0 {
|
||||
page.Next = entries[len(entries)-1].Sequence + 1
|
||||
}
|
||||
info, err := c.logInfo(l)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page.Last = info.Last
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// LogNewest is the newest entry of each key matching a pattern, in the log's order, by one batch direct get (mesh-sdk
|
||||
// process.LogReader.Newest).
|
||||
func (c *Conn) LogNewest(module, name, keys string) ([]LogEntry, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter, err := logFilter(l, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, more, err := c.logBatch(l, map[string]any{"multi_last": []string{filter}, "batch": LogReadMax})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if more {
|
||||
return nil, fmt.Errorf("%s holds more than %d keys matching %q: name fewer", name, LogReadMax, keys)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// logBatch is one batch direct get of a log: its entries, and whether more are pending after them.
|
||||
func (c *Conn) logBatch(l LogIssued, request map[string]any) ([]LogEntry, bool, error) {
|
||||
req, _ := json.Marshal(request)
|
||||
inbox := c.nc.NewInbox()
|
||||
sub, err := c.nc.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := c.nc.PublishRequest("$JS.API.DIRECT.GET."+l.Stream, inbox, req); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
entries := []LogEntry{}
|
||||
deadline := time.Now().Add(StateTimeout)
|
||||
for {
|
||||
msg, err := sub.NextMsg(time.Until(deadline))
|
||||
if err != nil {
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, false, notOnTheBus(l)
|
||||
}
|
||||
if errors.Is(err, nats.ErrTimeout) {
|
||||
return nil, false, fmt.Errorf("the read of %s did not end in %s", l.Name, StateTimeout)
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
switch status := msg.Header.Get("Status"); status {
|
||||
case "":
|
||||
e, err := logEntryOf(l, msg)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
entries = append(entries, *e)
|
||||
case "204": // the end of the batch: what is still pending after the last entry sent
|
||||
pending, _ := strconv.ParseUint(msg.Header.Get("Nats-Num-Pending"), 10, 64)
|
||||
return entries, pending > 0, nil
|
||||
case "404": // nothing from there on
|
||||
return entries, false, nil
|
||||
case "503":
|
||||
return nil, false, notOnTheBus(l)
|
||||
default:
|
||||
return nil, false, fmt.Errorf("the read of %s was refused by the bus: %s %s", l.Name, status, msg.Header.Get("Description"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// LogWatch hands deliver every entry of the keys matching a pattern from a sequence on (from), from a time on (since),
|
||||
// or, with neither, each new one, in the log's order, one at a time, through an ordered consumer it lets go of when
|
||||
// stopped (mesh-sdk process.LogReader.Watch). It returns once the entries there at its start are handed over; an error
|
||||
// from deliver is said, and the watch goes on with the next entry.
|
||||
func (c *Conn) LogWatch(module, name, keys string, from uint64, since time.Time, deliver func(LogEntry) error) (func(), error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter, err := logFilter(l, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg := jetstream.OrderedConsumerConfig{FilterSubjects: []string{filter}, DeliverPolicy: jetstream.DeliverNewPolicy}
|
||||
switch {
|
||||
case from != 0 && !since.IsZero():
|
||||
return nil, fmt.Errorf("a watch of %s starts from a sequence or from a time, not both", name)
|
||||
case from != 0:
|
||||
cfg.DeliverPolicy, cfg.OptStartSeq = jetstream.DeliverByStartSequencePolicy, from
|
||||
case !since.IsZero():
|
||||
at := since.UTC()
|
||||
cfg.DeliverPolicy, cfg.OptStartTime = jetstream.DeliverByStartTimePolicy, &at
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
cons, err := js.OrderedConsumer(ctx, l.Stream, cfg)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watch of %s (stream %s): %w", name, l.Stream, err)
|
||||
}
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watch of %s (stream %s): %w", name, l.Stream, err)
|
||||
}
|
||||
it, err := cons.Messages()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watch of %s (stream %s): %w", name, l.Stream, err)
|
||||
}
|
||||
hand := func(m jetstream.Msg) error {
|
||||
meta, err := m.Metadata()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e := LogEntry{Sequence: meta.Sequence.Stream, Key: strings.TrimPrefix(m.Subject(), l.Subject+"."),
|
||||
Value: valueOf(m.Data()), At: meta.Timestamp.UTC()}
|
||||
return deliver(e)
|
||||
}
|
||||
for pending := info.NumPending; pending > 0; pending-- {
|
||||
m, err := it.Next()
|
||||
if err == nil {
|
||||
err = hand(m)
|
||||
}
|
||||
if err != nil {
|
||||
it.Stop()
|
||||
return nil, fmt.Errorf("the watch of %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
var stopped atomic.Bool
|
||||
go func() {
|
||||
for {
|
||||
m, err := it.Next()
|
||||
if err != nil || stopped.Load() {
|
||||
return // stopped, or the connection closed; a message fetched as it stopped is not handed over
|
||||
}
|
||||
if err := hand(m); err != nil {
|
||||
c.Logf("[mesh-tools] %s did not take entry of %s: %v; it is in the log, a read finds it", module, name, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() { stopped.Store(true); it.Stop() }, nil
|
||||
}
|
||||
|
||||
// LogLast is a key's last entry, or nil when it has none.
|
||||
func (c *Conn) LogLast(module, name, key string) (*LogEntry, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkLogKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := c.nc.Request("$JS.API.DIRECT.GET."+l.Stream+"."+l.Subject+"."+key, nil, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch status := reply.Header.Get("Status"); status {
|
||||
case "":
|
||||
return logEntryOf(l, reply)
|
||||
case "404":
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("the last entry of %s.%s could not be read: %s %s", name, key, status, reply.Header.Get("Description"))
|
||||
}
|
||||
}
|
||||
|
||||
// LogInfo is a log's entries, bytes, cap and first and last sequences.
|
||||
func (c *Conn) LogInfo(module, name string) (*LogInfo, error) {
|
||||
l, err := c.issuedLog(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.logInfo(l)
|
||||
}
|
||||
|
||||
func (c *Conn) logInfo(l LogIssued) (*LogInfo, error) {
|
||||
reply, err := c.nc.Request("$JS.API.STREAM.INFO."+l.Stream, nil, StateTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var info struct {
|
||||
Config struct {
|
||||
MaxBytes int64 `json:"max_bytes"`
|
||||
} `json:"config"`
|
||||
State struct {
|
||||
Msgs uint64 `json:"messages"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
} `json:"state"`
|
||||
Error *struct {
|
||||
ErrCode uint16 `json:"err_code"`
|
||||
Description string `json:"description"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &info); err != nil {
|
||||
return nil, fmt.Errorf("the bus answered the info of %s with what is not one: %q", l.Name, reply.Data)
|
||||
}
|
||||
if info.Error != nil {
|
||||
if info.Error.ErrCode == uint16(jetstream.JSErrCodeStreamNotFound) {
|
||||
return nil, notOnTheBus(l)
|
||||
}
|
||||
return nil, fmt.Errorf("the info of the log %s was refused: %s (%d)", l.Name, info.Error.Description, info.Error.ErrCode)
|
||||
}
|
||||
var maxBytes uint64
|
||||
if info.Config.MaxBytes > 0 {
|
||||
maxBytes = uint64(info.Config.MaxBytes)
|
||||
}
|
||||
return &LogInfo{Entries: info.State.Msgs, Bytes: info.State.Bytes, MaxBytes: maxBytes,
|
||||
First: info.State.FirstSeq, Last: info.State.LastSeq}, nil
|
||||
}
|
||||
|
||||
// logEntryOf is an entry a direct get answered: its key is its subject under the log's.
|
||||
func logEntryOf(l LogIssued, msg *nats.Msg) (*LogEntry, error) {
|
||||
subject := msg.Header.Get("Nats-Subject")
|
||||
key, ok := strings.CutPrefix(subject, l.Subject+".")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the bus answered an entry of %s on %q, which is not under %s", l.Name, subject, l.Subject)
|
||||
}
|
||||
seq, err := strconv.ParseUint(msg.Header.Get("Nats-Sequence"), 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus answered an entry of %s with no sequence", l.Name)
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339Nano, msg.Header.Get("Nats-Time-Stamp"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bus answered entry %d of %s with no time: %v", seq, l.Name, err)
|
||||
}
|
||||
return &LogEntry{Sequence: seq, Key: key, Value: valueOf(msg.Data), At: at.UTC()}, nil
|
||||
}
|
||||
@@ -1,13 +1,7 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/wire"
|
||||
)
|
||||
@@ -15,97 +9,8 @@ import (
|
||||
// An answer larger than the bus carries in one message (novox/hq issue 314).
|
||||
//
|
||||
// The bus carries one message of at most its max_payload, and the client library refuses a larger one
|
||||
// before it leaves the process that sent it. Until this, a holder whose answer was too large wrote one
|
||||
// line to its own log and its caller waited out RequestTimeout and read "did not answer" — the
|
||||
// controller's `conditions` and `status` for hours on 2026-10-08, and the operator's channel with them.
|
||||
//
|
||||
// The controller pages such an answer (mesh-controller internal/link, pages.go): it answers an error
|
||||
// carrying `paged` — the call it holds the answer under, its size and its parts — and a caller asks the
|
||||
// same subject again once per part with PageHeader, each part answered raw with PartHeader. Asking the
|
||||
// same subject means no grant beyond the one the caller already had. A module's answer too large is
|
||||
// refused here, loudly, in a message that fits: its instances may be several, and a part asked again
|
||||
// could reach another.
|
||||
|
||||
const (
|
||||
// PageHeader asks for one part of a paged answer: `<call> <part>`, from zero.
|
||||
PageHeader = "Mesh-Page"
|
||||
// PartHeader is on a part: `<part>/<parts>`.
|
||||
PartHeader = "Mesh-Part"
|
||||
// partTries is how often one part is asked: during a handover a part can reach the controller that
|
||||
// did not hold it.
|
||||
partTries = 3
|
||||
)
|
||||
|
||||
// PartTimeout bounds the asking of one part.
|
||||
var PartTimeout = 10 * time.Second
|
||||
|
||||
// paged is what a too-large answer says instead of itself.
|
||||
type paged struct {
|
||||
Call string `json:"call"`
|
||||
Bytes int `json:"bytes"`
|
||||
Parts int `json:"parts"`
|
||||
}
|
||||
|
||||
func pagedIn(data []byte) (paged, bool) {
|
||||
if !bytes.Contains(data, []byte(`"paged"`)) {
|
||||
return paged{}, false
|
||||
}
|
||||
var r struct {
|
||||
Paged *paged `json:"paged"`
|
||||
}
|
||||
if json.Unmarshal(data, &r) != nil || r.Paged == nil || r.Paged.Call == "" || r.Paged.Parts < 1 {
|
||||
return paged{}, false
|
||||
}
|
||||
return *r.Paged, true
|
||||
}
|
||||
|
||||
// whole is a reply's whole answer: the reply itself, or every part of a paged one, joined. A part that
|
||||
// cannot be had is an error naming it, never a shorter answer.
|
||||
func (c *Conn) whole(subject string, data []byte) ([]byte, error) {
|
||||
p, ok := pagedIn(data)
|
||||
if !ok {
|
||||
return data, nil
|
||||
}
|
||||
out := make([]byte, 0, p.Bytes)
|
||||
for n := 0; n < p.Parts; n++ {
|
||||
var part []byte
|
||||
var err error
|
||||
for try := 0; try < partTries; try++ {
|
||||
if part, err = c.askPart(subject, p, n); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the answer was %d bytes, more than the bus carries in one message, paged as %s "+
|
||||
"in %d parts, and part %d could not be had: %w", p.Bytes, p.Call, p.Parts, n, err)
|
||||
}
|
||||
out = append(out, part...)
|
||||
}
|
||||
if len(out) != p.Bytes {
|
||||
return nil, fmt.Errorf("the answer paged as %s was %d bytes, and its parts joined are %d", p.Call, p.Bytes, len(out))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Conn) askPart(subject string, p paged, n int) ([]byte, error) {
|
||||
ask := nats.NewMsg(subject)
|
||||
ask.Header.Set(PageHeader, fmt.Sprintf("%s %d", p.Call, n))
|
||||
ask.Data = []byte(`{}`)
|
||||
reply, err := c.nc.RequestMsg(ask, PartTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if got := reply.Header.Get(PartHeader); got != fmt.Sprintf("%d/%d", n, p.Parts) {
|
||||
var r struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(reply.Data, &r) == nil && r.Error != "" {
|
||||
return nil, errors.New(r.Error)
|
||||
}
|
||||
return nil, fmt.Errorf("asked part %d/%d, answered %q", n, p.Parts, got)
|
||||
}
|
||||
return reply.Data, nil
|
||||
}
|
||||
// before it leaves the process that sent it. A module's answer too large is refused here, loudly, in a
|
||||
// message that fits, so its caller reads why instead of waiting out RequestTimeout.
|
||||
|
||||
// fits is an answer the bus can carry: the answer itself, or — when it is larger than one message —
|
||||
// an error that says so, which is.
|
||||
|
||||
@@ -2,7 +2,6 @@ package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -13,8 +12,8 @@ import (
|
||||
"github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq issue 314: an answer larger than the bus carries in one message arrives whole when the
|
||||
// controller pages it, and a module's is said to be too large — neither is a 30-second silence.
|
||||
// novox/hq issue 314: a module's answer larger than the bus carries in one message is said to be too large,
|
||||
// at once, not a 30-second silence. (The controller's paging ended with the controller, issue 1126.)
|
||||
func TestAnAnswerLargerThanOneMessage(t *testing.T) {
|
||||
url := meshtest.URL(t)
|
||||
seat, err := nats.Connect(url)
|
||||
@@ -27,43 +26,8 @@ func TestAnAnswerLargerThanOneMessage(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
limit := int(seat.MaxPayload())
|
||||
large := strings.Repeat("a condition held past its bound; ", 3*int(seat.MaxPayload())/32)
|
||||
|
||||
// The controller as it pages (mesh-controller internal/link, pages.go): the first answer an error
|
||||
// with `paged`, each part asked again on the same subject with the page header.
|
||||
large := strings.Repeat("a condition held past its bound; ", 3*limit/32)
|
||||
whole, _ := json.Marshal(map[string]any{"result": large})
|
||||
partSize := limit - 1024
|
||||
parts := (len(whole) + partSize - 1) / partSize
|
||||
sub, err := seat.Subscribe(bus.SeatToolSubject("mesh-controller", "conditions", "", ""), func(m *nats.Msg) {
|
||||
if page := m.Header.Get(bus.PageHeader); page != "" {
|
||||
var id string
|
||||
var n int
|
||||
fmt.Sscanf(page, "%s %d", &id, &n)
|
||||
end := min((n+1)*partSize, len(whole))
|
||||
part := nats.NewMsg(m.Reply)
|
||||
part.Header.Set(bus.PartHeader, fmt.Sprintf("%d/%d", n, parts))
|
||||
part.Data = whole[n*partSize : end]
|
||||
_ = m.RespondMsg(part)
|
||||
return
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"error": "too large for one message",
|
||||
"paged": map[string]any{"call": "call-1-1", "bytes": len(whole), "parts": parts}})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
_ = seat.Flush()
|
||||
|
||||
got, err := c.Ask("seat:mesh-controller.conditions", map[string]any{}, "")
|
||||
var said string
|
||||
if err != nil || json.Unmarshal(got.Result, &said) != nil || said != large {
|
||||
t.Fatalf("an answer of %d bytes in %d parts arrived as %d bytes: %v", len(whole), parts, len(got.Result), err)
|
||||
}
|
||||
|
||||
// A module's answer too large is refused here in words, at once.
|
||||
stop, err := c.HandleSubject("mesh.mod.journal.tool.lines", func(json.RawMessage) (any, error) { return large, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/pernode"
|
||||
)
|
||||
|
||||
// A state its owner keeps one bucket per node (novox/hq ADR 0593 §6: each node's engine writes its own `composed-<node>`,
|
||||
// and the provisioner, the assigner and the view read them all) is served to a bundle as one state. The rules (a key's
|
||||
// first part names its node's bucket, keys and a watch over every bucket of the prefix, a bucket created later heard by
|
||||
// the bus's creation advisory) are mesh-sdk's pernode package, written once for a process and for this runtime; this
|
||||
// file adapts its entries to the runtime's shapes. A reader is granted the listing of stream names
|
||||
// (pernode.ListSubject) and the creation advisory (pernode.CreatedAdvisory) beside each bucket's read.
|
||||
|
||||
func perNodeRefusal(module, name, what string) error {
|
||||
return fmt.Errorf("%s's state %s is kept one bucket per node and read only: no %s of it as one state (novox/hq ADR 0593 §6)",
|
||||
module, name, what)
|
||||
}
|
||||
|
||||
func (c *Conn) family(s StateIssued) pernode.Family {
|
||||
return pernode.Family{Read: s.Name, Prefix: s.Bucket, Bus: pernode.OverNATS(c.nc)}
|
||||
}
|
||||
|
||||
func (c *Conn) perNodeGet(s StateIssued, key string) (*StateEntry, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
e, err := c.family(s).Get(ctx, key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
return &StateEntry{Key: e.Key(), Value: valueOf(e.Value()), Revision: e.Revision(), At: e.Created().UTC()}, nil
|
||||
}
|
||||
|
||||
func (c *Conn) perNodeKeys(s StateIssued) ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
return c.family(s).Keys(ctx)
|
||||
}
|
||||
|
||||
// perNodeWatch is StateWatch over every bucket of the family, and over a bucket created later. As in a single bucket's
|
||||
// watch, StateTimeout bounds the time without progress (issue 1726), and a value the bundle does not take is said and
|
||||
// the watch goes on.
|
||||
func (c *Conn) perNodeWatch(module string, s StateIssued, pattern string, deliver func(StateChange) error) (func(), error) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
progress := make(chan struct{}, 1)
|
||||
handle := func(ch pernode.Change) error {
|
||||
select {
|
||||
case progress <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
e := ch.Entry
|
||||
change := StateChange{State: s.Name, Key: e.Key(), Revision: e.Revision(), Current: ch.Current}
|
||||
switch e.Operation() {
|
||||
case jetstream.KeyValuePut:
|
||||
change.Op = "put"
|
||||
change.Value = valueOf(e.Value())
|
||||
default:
|
||||
if ch.Current {
|
||||
return nil
|
||||
}
|
||||
change.Op = "delete"
|
||||
if e.Operation() == jetstream.KeyValuePurge {
|
||||
change.Op = "expire"
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if err := deliver(change); err != nil {
|
||||
c.Logf("[mesh-tools] %s did not take %s %s of %s: %v; its next change, or its next start, reads it again",
|
||||
module, change.Op, change.Key, s.Name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- c.family(s).Watch(ctx, pattern, handle) }()
|
||||
bound := time.NewTimer(StateTimeout)
|
||||
defer bound.Stop()
|
||||
for {
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
return cancel, nil
|
||||
case <-progress:
|
||||
if !bound.Stop() {
|
||||
select {
|
||||
case <-bound.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
bound.Reset(StateTimeout)
|
||||
case <-bound.C:
|
||||
cancel()
|
||||
return nil, fmt.Errorf("the current values of %s did not arrive in %s: none was handed over in that time", s.Name, StateTimeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A module's code asks its machine's provisioner client for its values at runtime (novox/hq ADR 0349 clause 14 as
|
||||
// ADR 0370 changes it, issue 685).
|
||||
//
|
||||
// While a module's code runs here, its asks reach the bus under this runtime's account, and the bus proves only
|
||||
// that account. So the runtime asks its own machine's client on ProvisionAskSubject — which the mesh grants
|
||||
// this machine's tool runner alone to publish (ADR 0370 §3) — naming the module itself, from the bundle that asks
|
||||
// and never from the module's code or arguments (§4). The client answers that module's config values, for what its
|
||||
// manifest declares, and no secret: the vault hands a secret only to a caller the bus proves (§2).
|
||||
|
||||
// ProvisionAskSubject is where this machine's tool runner asks its provisioner client (the node-engine
|
||||
// internal/broker ProvisionerAskSubject): `prov.ask.<node>`.
|
||||
func ProvisionAskSubject(node string) string { return "prov.ask." + node }
|
||||
|
||||
// ProvisionChangedSubjects is every module's word of what moved on a machine (the node-engine
|
||||
// ProvisionerChangedSubject): `prov.changed.<node>.<module>`, a module's name dots and all.
|
||||
func ProvisionChangedSubjects(node string) string { return "prov.changed." + node + ".>" }
|
||||
|
||||
// ProvisionTimeout bounds one ask of the client; a variable so a test need not wait.
|
||||
var ProvisionTimeout = 10 * time.Second
|
||||
|
||||
// ProvisionChanged is what the client says moved of one module: the kind, and the key, or every key of the kind when
|
||||
// empty. Never the value.
|
||||
type ProvisionChanged struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// ProvisionRefused is the client's refusal, in its own words: a key the module does not declare, a module not placed
|
||||
// on the machine, a secret, a kind it does not answer yet.
|
||||
type ProvisionRefused struct{ Reason string }
|
||||
|
||||
func (r *ProvisionRefused) Error() string { return r.Reason }
|
||||
|
||||
// AskProvisioner asks this machine's provisioner client for one value of module: kind `config` or `bound`, and the
|
||||
// key. The module is the caller's to name from the bundle it runs; this never takes it from what a bundle sent.
|
||||
func (c *Conn) AskProvisioner(module, kind, key string) (json.RawMessage, error) {
|
||||
if c.node == "" {
|
||||
return nil, errors.New("this runtime names no machine, so it has no provisioner client to ask")
|
||||
}
|
||||
body, err := json.Marshal(map[string]string{"module": module, "kind": kind, "key": key})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
msg, err := c.nc.Request(ProvisionAskSubject(c.node), body, ProvisionTimeout)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, fmt.Errorf("%s's provisioner client does not answer: it is not running there, or the bus does "+
|
||||
"not let %s ask it", c.node, c.cred.User)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("asking %s's provisioner client for %s's %s %s: %w", c.node, module, kind, key, err)
|
||||
}
|
||||
var answered struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &answered); err != nil {
|
||||
return nil, fmt.Errorf("%s's provisioner client answered something that is not its answer: %w", c.node, err)
|
||||
}
|
||||
if answered.Error != "" {
|
||||
return nil, &ProvisionRefused{Reason: answered.Error}
|
||||
}
|
||||
if len(answered.Value) == 0 {
|
||||
return nil, fmt.Errorf("%s's provisioner client answered no value and no reason", c.node)
|
||||
}
|
||||
return answered.Value, nil
|
||||
}
|
||||
|
||||
// OnProvisionChanged hands told every word of this machine's provisioner client of what moved, by module, until
|
||||
// stopped.
|
||||
func (c *Conn) OnProvisionChanged(told func(module string, changed ProvisionChanged)) (func(), error) {
|
||||
if c.node == "" {
|
||||
return nil, errors.New("this runtime names no machine, so it hears no provisioner client")
|
||||
}
|
||||
prefix := "prov.changed." + c.node + "."
|
||||
sub, err := c.nc.Subscribe(ProvisionChangedSubjects(c.node), func(m *nats.Msg) {
|
||||
var changed ProvisionChanged
|
||||
if json.Unmarshal(m.Data, &changed) != nil || changed.Kind == "" {
|
||||
return
|
||||
}
|
||||
told(strings.TrimPrefix(m.Subject, prefix), changed)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() { _ = sub.Unsubscribe() }, c.nc.Flush()
|
||||
}
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
// and anything else is refused here, with the reason, before it is sent. A bundle reaches this only through
|
||||
// its own standard input and output: no tool call does.
|
||||
|
||||
// SeatTraffic is what a membership lists of a module's seat traffic (the controller's broker.SeatTraffic).
|
||||
// SeatTraffic is what a membership lists of a module's seat traffic (the grants part's record).
|
||||
type SeatTraffic struct {
|
||||
Publish []string `json:"publish,omitempty"`
|
||||
Subscribe []string `json:"subscribe,omitempty"`
|
||||
@@ -32,7 +32,7 @@ type SeatTraffic struct {
|
||||
Workers []SeatWorker `json:"workers,omitempty"`
|
||||
Records []string `json:"records,omitempty"`
|
||||
// Kinds are the holders of every kinded bench the module uses or watches, with what each promises: the
|
||||
// controller's record of their claims (novox/hq ADR 0259 §5).
|
||||
// record of their claims (novox/hq ADR 0259 §5).
|
||||
Kinds []KindHeld `json:"kinds,omitempty"`
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ type KindHeld struct {
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
|
||||
// SeatKinds is the kinds the module's membership names, as the controller issued them.
|
||||
// SeatKinds is the kinds the module's membership names, as the grants part issued them.
|
||||
func (c *Conn) SeatKinds(module string) ([]KindHeld, error) {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
@@ -249,7 +249,7 @@ type Work struct {
|
||||
|
||||
// SeatTake reads a worker the module's seat traffic lists and hands each message to deliver: acknowledged
|
||||
// when deliver returns nil, offered again after NakDelay when it returns an error, ended when it is not
|
||||
// JSON. The worker is the controller's to create; this binds to it by name and never makes one.
|
||||
// JSON. The worker is the mesh's to create; this binds to it by name and never makes one.
|
||||
func (c *Conn) SeatTake(module, consumer string, deliver func(Work) error) (func(), error) {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
@@ -370,7 +370,13 @@ func (c *Conn) SeatRecord(module, bucket, key string) (*StateEntry, error) {
|
||||
}
|
||||
var revision uint64
|
||||
fmt.Sscan(reply.Header.Get("Nats-Sequence"), &revision)
|
||||
return &StateEntry{Key: module + "." + key, Value: valueOf(reply.Data), Revision: revision}, nil
|
||||
var at time.Time
|
||||
if stamp := reply.Header.Get("Nats-Time-Stamp"); stamp != "" {
|
||||
if t, err := time.Parse(time.RFC3339Nano, stamp); err == nil {
|
||||
at = t.UTC()
|
||||
}
|
||||
}
|
||||
return &StateEntry{Key: module + "." + key, Value: valueOf(reply.Data), Revision: revision, At: at}, nil
|
||||
}
|
||||
|
||||
// StateCreate writes one key only when it has no value, and answers its revision; a key that has one is
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// **A holder that pulls one piece of work when it has room** (novox/hq ADR 0525 §4, §6; issue 1887 A11). SeatTake
|
||||
// hands a bundle every piece a worker holds, one at a time and prefetched; a build module must instead take one ask
|
||||
// only when it has a free slot, from its workers in their order (delivery, then check, then recheck), and decide each
|
||||
// ask's fate when its step ends. SeatPull is one fetch of at most one message from a worker the module's membership
|
||||
// lists, never a prefetch; SeatSettle gives a pulled message its fate. A message the bundle never settles (its process
|
||||
// died) is delivered again by the worker's own acknowledgement wait.
|
||||
|
||||
// MaxPullWait bounds how long one pull waits for a message: under the bundle's 30 s wait for its answer.
|
||||
var MaxPullWait = 25 * time.Second
|
||||
|
||||
// Pulled is one message a module pulled from a seat's worker, named by an id the runtime gave it.
|
||||
type Pulled struct {
|
||||
ID string `json:"id"`
|
||||
Worker string `json:"worker"`
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
Delivered uint64 `json:"delivered"`
|
||||
}
|
||||
|
||||
// pulls are the messages modules pulled and have not settled, by id; each kept with the module it was pulled for, so
|
||||
// one module never settles another's.
|
||||
type pulls struct {
|
||||
mu sync.Mutex
|
||||
held map[string]pulledMsg
|
||||
cons map[string]jetstream.Consumer // by module and consumer
|
||||
}
|
||||
|
||||
type pulledMsg struct {
|
||||
module string
|
||||
msg jetstream.Msg
|
||||
}
|
||||
|
||||
var seatPulls = &pulls{held: map[string]pulledMsg{}, cons: map[string]jetstream.Consumer{}}
|
||||
|
||||
func pullID() string {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// SeatPull fetches at most one message from consumer for module, waiting at most wait (none when 0): nil when none
|
||||
// came. The consumer must be one the module's membership lists as its worker (novox/hq ADR 0259); the runtime binds to
|
||||
// it by name and never makes one.
|
||||
func (c *Conn) SeatPull(module, consumer string, wait time.Duration) (*Pulled, error) {
|
||||
t, err := c.trafficOf(module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var w *SeatWorker
|
||||
var names []string
|
||||
for i := range t.Workers {
|
||||
names = append(names, t.Workers[i].Consumer)
|
||||
if t.Workers[i].Consumer == consumer {
|
||||
w = &t.Workers[i]
|
||||
}
|
||||
}
|
||||
if w == nil {
|
||||
return nil, fmt.Errorf("%s pulls no work from %s: its membership lists %s (novox/hq ADR 0259)", module,
|
||||
consumer, strings.Join(names, ", "))
|
||||
}
|
||||
if wait > MaxPullWait {
|
||||
wait = MaxPullWait
|
||||
}
|
||||
bound, err := c.pullConsumer(module, *w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var batch jetstream.MessageBatch
|
||||
if wait <= 0 {
|
||||
batch, err = bound.FetchNoWait(1)
|
||||
} else {
|
||||
batch, err = bound.Fetch(1, jetstream.FetchMaxWait(wait))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pulling %s's worker %s: %w", module, w.Consumer, err)
|
||||
}
|
||||
msg, ok := <-batch.Messages()
|
||||
if !ok {
|
||||
if err := batch.Error(); err != nil {
|
||||
return nil, fmt.Errorf("pulling %s's worker %s: %w", module, w.Consumer, err)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if !json.Valid(msg.Data()) {
|
||||
_ = msg.Term()
|
||||
return nil, nil
|
||||
}
|
||||
headers := map[string]string{}
|
||||
for k := range msg.Headers() {
|
||||
headers[k] = msg.Headers().Get(k)
|
||||
}
|
||||
delivered := uint64(1)
|
||||
if meta, merr := msg.Metadata(); merr == nil && meta != nil {
|
||||
delivered = meta.NumDelivered
|
||||
}
|
||||
p := &Pulled{ID: pullID(), Worker: w.Consumer, Subject: msg.Subject(), Body: json.RawMessage(msg.Data()),
|
||||
Headers: headers, Delivered: delivered}
|
||||
seatPulls.mu.Lock()
|
||||
seatPulls.held[p.ID] = pulledMsg{module: module, msg: msg}
|
||||
seatPulls.mu.Unlock()
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (c *Conn) pullConsumer(module string, w SeatWorker) (jetstream.Consumer, error) {
|
||||
key := module + " " + w.Consumer
|
||||
seatPulls.mu.Lock()
|
||||
bound, ok := seatPulls.cons[key]
|
||||
seatPulls.mu.Unlock()
|
||||
if ok {
|
||||
return bound, nil
|
||||
}
|
||||
js, err := jetstream.New(c.nc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
bound, err = js.Consumer(ctx, w.Stream, w.Consumer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("binding %s's worker %s on %s: %w", module, w.Consumer, w.Stream, err)
|
||||
}
|
||||
seatPulls.mu.Lock()
|
||||
seatPulls.cons[key] = bound
|
||||
seatPulls.mu.Unlock()
|
||||
return bound, nil
|
||||
}
|
||||
|
||||
// The fates SeatSettle gives a pulled message.
|
||||
const (
|
||||
SettleAck = "ack" // done: never delivered again
|
||||
SettleNak = "nak" // handed back, to be delivered again at once
|
||||
SettleTerm = "term" // dropped
|
||||
SettleProgress = "progress" // still being worked: its acknowledgement wait starts again
|
||||
)
|
||||
|
||||
// SeatSettle gives the message module pulled under id its fate. A message pulled for another module, or settled
|
||||
// already, is refused.
|
||||
func (c *Conn) SeatSettle(module, id, fate string) error {
|
||||
seatPulls.mu.Lock()
|
||||
p, ok := seatPulls.held[id]
|
||||
if ok && p.module != module {
|
||||
ok = false
|
||||
}
|
||||
if ok && fate != SettleProgress {
|
||||
delete(seatPulls.held, id)
|
||||
}
|
||||
seatPulls.mu.Unlock()
|
||||
if !ok {
|
||||
return fmt.Errorf("%s holds no pulled message %s", module, id)
|
||||
}
|
||||
switch fate {
|
||||
case SettleAck:
|
||||
return p.msg.Ack()
|
||||
case SettleNak:
|
||||
return p.msg.Nak()
|
||||
case SettleTerm:
|
||||
return p.msg.Term()
|
||||
case SettleProgress:
|
||||
return p.msg.InProgress()
|
||||
}
|
||||
seatPulls.mu.Lock()
|
||||
seatPulls.held[id] = p // an unknown fate settles nothing
|
||||
seatPulls.mu.Unlock()
|
||||
return fmt.Errorf("a pulled message's fate is ack, nak, term or progress, not %q", fate)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// 1887/A11: a holder pulls one message at a time, only from a worker its membership lists, never a prefetch; it settles
|
||||
// only what it pulled, and a message handed back comes again.
|
||||
func TestAHolderPullsOneMessageWhenItHasRoom(t *testing.T) {
|
||||
_, conn, js := seatMesh(t)
|
||||
for _, id := range []string{"a1", "a2"} {
|
||||
if _, err := conn.SeatPublish("mesh-delivery", "mesh.seat.operator-channel.accept.ask.mesh-delivery",
|
||||
json.RawMessage(`{"id":"`+id+`"}`), id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
first, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 0)
|
||||
if err != nil || first == nil || string(first.Body) != `{"id":"a1"}` || first.Delivered != 1 {
|
||||
t.Fatalf("the first pull = %+v, %v", first, err)
|
||||
}
|
||||
// One pulled, one still waiting on the stream: nothing was fetched ahead.
|
||||
info, err := js.ConsumerInfo("SEAT_OPERATOR_CHANNEL", "SEAT_OPERATOR_CHANNEL_worker")
|
||||
if err != nil || info.NumPending != 1 || info.NumAckPending != 1 {
|
||||
t.Fatalf("after one pull: pending %d, in hand %d (%v); want 1 and 1", info.NumPending, info.NumAckPending, err)
|
||||
}
|
||||
second, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 0)
|
||||
if err != nil || second == nil || string(second.Body) != `{"id":"a2"}` {
|
||||
t.Fatalf("the second pull = %+v, %v", second, err)
|
||||
}
|
||||
if none, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 0); err != nil || none != nil {
|
||||
t.Fatalf("an empty worker answered %+v, %v", none, err)
|
||||
}
|
||||
began := time.Now()
|
||||
if none, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", 200*time.Millisecond); err != nil || none != nil || time.Since(began) < 150*time.Millisecond {
|
||||
t.Fatalf("a waiting pull answered %+v, %v after %v", none, err, time.Since(began))
|
||||
}
|
||||
if err := conn.SeatSettle("telegram", first.ID, bus.SettleAck); err == nil {
|
||||
t.Error("a module settled another's message")
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", first.ID, bus.SettleProgress); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", first.ID, bus.SettleAck); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", first.ID, bus.SettleAck); err == nil {
|
||||
t.Error("a message was settled twice")
|
||||
}
|
||||
if err := conn.SeatSettle("messenger", second.ID, bus.SettleNak); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := conn.SeatPull("messenger", "SEAT_OPERATOR_CHANNEL_worker", time.Second)
|
||||
if err != nil || again == nil || string(again.Body) != `{"id":"a2"}` || again.Delivered != 2 {
|
||||
t.Fatalf("the message handed back = %+v, %v", again, err)
|
||||
}
|
||||
if _, err := conn.SeatPull("telegram", "SEAT_OPERATOR_CHANNEL_worker", 0); err == nil || !strings.Contains(err.Error(), "membership lists") {
|
||||
t.Errorf("a channel pulled the router's work: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -98,7 +98,7 @@ func TestAModuleSubmitsAndSaysOnlyUnderItsOwnNameAndKind(t *testing.T) {
|
||||
t.Fatalf("an asker could not ask under its own name: %v", err)
|
||||
}
|
||||
for module, subject := range map[string]string{
|
||||
"mesh-delivery": "mesh.seat.operator-channel.accept.ask.mesh-controller",
|
||||
"mesh-delivery": "mesh.seat.operator-channel.accept.ask.mesh-module-assigner",
|
||||
"telegram": "mesh.seat.intake.event.choice.desktop",
|
||||
"messenger": "mesh.seat.intake.event.choice.telegram",
|
||||
} {
|
||||
@@ -206,13 +206,16 @@ func TestAnAskerReadsItsOwnRecordAndNoOther(t *testing.T) {
|
||||
if _, err := kv.Put("mesh-delivery.a1", []byte(`{"state":"open"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put("mesh-controller.a1", []byte(`{"state":"open"}`)); err != nil {
|
||||
if _, err := kv.Put("mesh-module-assigner.a1", []byte(`{"state":"open"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := conn.SeatRecord("mesh-delivery", "messenger_asks", "a1")
|
||||
if err != nil || got == nil || string(got.Value) != `{"state":"open"}` {
|
||||
t.Fatalf("the asker could not read its own ask: %v %v", got, err)
|
||||
}
|
||||
if got.At.IsZero() || time.Since(got.At) > time.Minute {
|
||||
t.Errorf("the record does not say when the bus wrote it: at %v", got.At)
|
||||
}
|
||||
if missing, err := conn.SeatRecord("mesh-delivery", "messenger_asks", "a2"); err != nil || missing != nil {
|
||||
t.Fatalf("a missing ask read as %v, %v", missing, err)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's state on the bus (novox/hq ADR 0201): key-value buckets the controller creates from what
|
||||
// A module's state on the bus (novox/hq ADR 0201): key-value buckets the mesh creates from what
|
||||
// the module declared, and issues to each assignment in its membership by the name the module uses —
|
||||
// its own state by the local name, another module's as `<module>.<name>`.
|
||||
//
|
||||
@@ -28,6 +28,10 @@ type StateIssued struct {
|
||||
Name string `json:"name"`
|
||||
Bucket string `json:"bucket"`
|
||||
Writes bool `json:"writes,omitempty"`
|
||||
// PerNode says the state is kept one bucket per node (novox/hq ADR 0593 §6, mesh-sdk membership.StateIssued): Bucket is
|
||||
// then the buckets' prefix, each bucket the prefix and a node's name, and a key's first part names its node. It is read
|
||||
// only, served as one state (per_node.go).
|
||||
PerNode bool `json:"per-node,omitempty"`
|
||||
}
|
||||
|
||||
// StateEntry is one key's current value.
|
||||
@@ -35,9 +39,12 @@ type StateEntry struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Revision uint64 `json:"revision"`
|
||||
// At is when the bus wrote this value (the entry's created time), as stdio.StateEntry.At reads it.
|
||||
At time.Time `json:"at,omitzero"`
|
||||
}
|
||||
|
||||
// StateChange is one change a watch delivers: a key put or deleted.
|
||||
// StateChange is one change a watch delivers: a key put, deleted, or expired (its own time to live, or the
|
||||
// bucket's age, ended; issue 1952).
|
||||
type StateChange struct {
|
||||
State string `json:"state"`
|
||||
Key string `json:"key"`
|
||||
@@ -88,7 +95,7 @@ func (c *Conn) bucket(ctx context.Context, s StateIssued) (jetstream.KeyValue, e
|
||||
kv, err := js.KeyValue(ctx, s.Bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the controller "+
|
||||
return nil, fmt.Errorf("the state %q is issued and its bucket is not on the bus yet: the mesh "+
|
||||
"creates it from the catalogue on its next raise", s.Name)
|
||||
}
|
||||
return nil, err
|
||||
@@ -113,6 +120,9 @@ func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) {
|
||||
if err := checkKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.PerNode {
|
||||
return c.perNodeGet(s, key)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
@@ -126,7 +136,102 @@ func (c *Conn) StateGet(module, name, key string) (*StateEntry, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &StateEntry{Key: e.Key(), Value: valueOf(e.Value()), Revision: e.Revision()}, nil
|
||||
return &StateEntry{Key: e.Key(), Value: valueOf(e.Value()), Revision: e.Revision(), At: e.Created().UTC()}, nil
|
||||
}
|
||||
|
||||
// StateKept is one value a bucket still keeps for a key: a put, or a delete (a purge reads as one).
|
||||
type StateKept struct {
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Op string `json:"op"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// StateHistory is every value the bucket still keeps for one key, oldest first, or none (novox/hq ADR
|
||||
// 0307). It is a read like StateGet: served to the owner and to every reader, and writes nothing.
|
||||
func (c *Conn) StateHistory(module, name, key string) ([]StateKept, error) {
|
||||
s, err := c.issuedState(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.PerNode {
|
||||
return nil, perNodeRefusal(module, name, "history")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := kv.History(ctx, key)
|
||||
kept := []StateKept{}
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return kept, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
k := StateKept{Key: e.Key(), Revision: e.Revision(), Op: "delete", At: e.Created().UTC()}
|
||||
if e.Operation() == jetstream.KeyValuePut {
|
||||
k.Op = "put"
|
||||
k.Value = valueOf(e.Value())
|
||||
}
|
||||
kept = append(kept, k)
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// StateInfo is how full a state's bucket is against its caps (tools-are-how-a-module-is-spoken-to): a bundle's tools
|
||||
// say fill against cap from it.
|
||||
type StateInfo struct {
|
||||
State string `json:"state"`
|
||||
Bucket string `json:"bucket"`
|
||||
// Bytes is what the bucket holds now, every kept value and marker counted; Keys how many keys it holds, a key
|
||||
// whose last value is a delete or expiry marker counted until the marker goes.
|
||||
Bytes uint64 `json:"bytes"`
|
||||
Keys uint64 `json:"keys"`
|
||||
// MaxBytes and MaxValueBytes are the bucket's caps; -1 is none.
|
||||
MaxBytes int64 `json:"max_bytes"`
|
||||
MaxValueBytes int32 `json:"max_value_bytes"`
|
||||
// History is how many values a key keeps; TTLSeconds the bucket's age for a value (0 none); KeyTTL whether a key may
|
||||
// carry its own time to live (issue 1952).
|
||||
History int64 `json:"history"`
|
||||
TTLSeconds int64 `json:"ttl_seconds"`
|
||||
KeyTTL bool `json:"key_ttl"`
|
||||
}
|
||||
|
||||
// StateInfo reads a state's bucket's fill and caps. It is a read like StateGet: served to the owner and every reader.
|
||||
func (c *Conn) StateInfo(module, name string) (*StateInfo, error) {
|
||||
s, err := c.issuedState(module, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.PerNode {
|
||||
return nil, perNodeRefusal(module, name, "fill")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
status, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b, ok := status.(*jetstream.KeyValueBucketStatus)
|
||||
if !ok || b.StreamInfo() == nil {
|
||||
return nil, fmt.Errorf("the bus answered no stream behind %s's state %s", module, name)
|
||||
}
|
||||
info := b.StreamInfo()
|
||||
return &StateInfo{State: name, Bucket: status.Bucket(), Bytes: info.State.Bytes, Keys: uint64(info.State.NumSubjects),
|
||||
MaxBytes: info.Config.MaxBytes, MaxValueBytes: info.Config.MaxMsgSize, History: status.History(),
|
||||
TTLSeconds: int64(status.TTL() / time.Second), KeyTTL: info.Config.AllowMsgTTL && status.LimitMarkerTTL() > 0}, nil
|
||||
}
|
||||
|
||||
// StatePut writes one key, as the module, where the module keeps the state. It answers the revision.
|
||||
@@ -155,6 +260,61 @@ func (c *Conn) StatePut(module, name, key string, value json.RawMessage) (uint64
|
||||
return kv.Put(ctx, key, value)
|
||||
}
|
||||
|
||||
// MinKeyTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds (mesh-sdk
|
||||
// process.MinTTL).
|
||||
const MinKeyTTL = time.Second
|
||||
|
||||
// StateCreateTTL writes one key only when it has no value, with its own time to live (novox/hq ADR 0564 clauses 4 and
|
||||
// 7, issue 1952; mesh-sdk process.State.CreateTTL): the bus removes it when ttl passes and every watch hears the
|
||||
// removal as an expiry, so a bundle's wake needs no timer. A key that has a value is refused as ErrStateChanged. The
|
||||
// bucket must take keys with their own time to live (a state declared `key-ttl`); one that does not is refused, naming
|
||||
// it, before anything is written.
|
||||
func (c *Conn) StateCreateTTL(module, name, key string, value json.RawMessage, ttl time.Duration) (uint64, error) {
|
||||
if ttl < MinKeyTTL {
|
||||
return 0, fmt.Errorf("%s's state %s, key %s: a time to live of %s is shorter than the bus keeps (%s)",
|
||||
module, name, key, ttl, MinKeyTTL)
|
||||
}
|
||||
return c.stateWrite(module, name, key, value, func(ctx context.Context, kv jetstream.KeyValue) (uint64, error) {
|
||||
if err := takesKeyTTL(ctx, kv, module, name); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return kv.Create(ctx, key, value, jetstream.KeyTTL(ttl))
|
||||
})
|
||||
}
|
||||
|
||||
// StateRenewTTL writes one key whole with a fresh time to live, whatever it held (mesh-sdk process.State.RenewTTL): a
|
||||
// time to live is set only when a key is created, so a live key is deleted first, and its watches hear the delete and
|
||||
// then the new value.
|
||||
func (c *Conn) StateRenewTTL(module, name, key string, value json.RawMessage, ttl time.Duration) (uint64, error) {
|
||||
rev, err := c.StateCreateTTL(module, name, key, value, ttl)
|
||||
if !errors.Is(err, ErrStateChanged) {
|
||||
return rev, err
|
||||
}
|
||||
if err := c.StateDelete(module, name, key); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.StateCreateTTL(module, name, key, value, ttl)
|
||||
}
|
||||
|
||||
// takesKeyTTL refuses a bucket that does not allow a time to live per key, or leaves no marker when one ends: there the
|
||||
// server would refuse the write, or the key would vanish and no watch would hear it.
|
||||
func takesKeyTTL(ctx context.Context, kv jetstream.KeyValue, module, name string) error {
|
||||
status, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allows := status.LimitMarkerTTL() > 0
|
||||
if b, ok := status.(*jetstream.KeyValueBucketStatus); ok && b.StreamInfo() != nil {
|
||||
allows = allows && b.StreamInfo().Config.AllowMsgTTL
|
||||
}
|
||||
if !allows {
|
||||
return fmt.Errorf("%s's state %s does not take keys with their own time to live: its bucket %s allows none, or "+
|
||||
"leaves no marker a watch hears when one ends; %s declares the state `key-ttl` for it (novox/hq ADR 0564, "+
|
||||
"issue 1952)", module, name, status.Bucket(), module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StateDelete removes one key, as the module, where the module keeps the state. A key that was not
|
||||
// there is not an error: what is asked for is that it is gone.
|
||||
func (c *Conn) StateDelete(module, name, key string) error {
|
||||
@@ -180,12 +340,20 @@ func (c *Conn) StateKeys(module, name string) ([]string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.PerNode {
|
||||
return c.perNodeKeys(s)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
defer cancel()
|
||||
kv, err := c.bucket(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return keysOf(ctx, kv)
|
||||
}
|
||||
|
||||
// keysOf is every key with a current value in one bucket, sorted.
|
||||
func keysOf(ctx context.Context, kv jetstream.KeyValue) ([]string, error) {
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -217,7 +385,12 @@ func (c *Conn) writable(module, name string) (StateIssued, error) {
|
||||
|
||||
// StateWatch hands deliver the current value of every key matching the pattern — none that is
|
||||
// deleted — and then every change, in order (ADR 0201). It returns once the current values are
|
||||
// delivered; deliver is called from one goroutine, one change at a time, and an error from it is said
|
||||
// delivered. **StateTimeout bounds the time without progress, not the whole set** (novox/hq issue
|
||||
// 1726): each current value is a round trip to the bundle, so a state of a few hundred keys outlasts
|
||||
// any fixed bound for all of them; the watch then failed at every attempt, the bundle dropped its
|
||||
// handler, and the newest keys, handed over last, were never taken (the checker's owed, whose newest
|
||||
// verdicts never reached mesh/check). It fails only when no current value arrives, or one is not
|
||||
// taken, within StateTimeout; deliver is called from one goroutine, one change at a time, and an error from it is said
|
||||
// and the watch goes on: state is not a queue, and the next change, or the next start, reads it again.
|
||||
// The pattern is a key, with `*` for one name and `**` for the rest; empty is every key.
|
||||
func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange) error) (stop func(), err error) {
|
||||
@@ -242,12 +415,20 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
|
||||
}
|
||||
filter = strings.Join(parts, ".")
|
||||
}
|
||||
if s.PerNode {
|
||||
return c.perNodeWatch(module, s, pattern, deliver)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), StateTimeout)
|
||||
kv, err := c.bucket(ctx, s)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.watchBucket(module, name, kv, filter, deliver)
|
||||
}
|
||||
|
||||
// watchBucket is StateWatch over one bucket: the current values, then every change.
|
||||
func (c *Conn) watchBucket(module, name string, kv jetstream.KeyValue, filter string, deliver func(StateChange) error) (stop func(), err error) {
|
||||
watching, stopWatching := context.WithCancel(context.Background())
|
||||
w, err := kv.Watch(watching, filter)
|
||||
if err != nil {
|
||||
@@ -262,9 +443,23 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
|
||||
})
|
||||
}
|
||||
current := make(chan struct{})
|
||||
// handed says a current value was handed over or passed by: progress, which restarts the bound.
|
||||
handed := make(chan struct{}, 1)
|
||||
progress := func() {
|
||||
select {
|
||||
case handed <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
initial := true
|
||||
for e := range w.Updates() {
|
||||
// A watch given up, or stopped, hands over nothing more, though values stay buffered in the
|
||||
// watcher: they would reach a watch id the bundle no longer holds (issue 774: 1257 orphan
|
||||
// changes after 51 failed watches).
|
||||
if watching.Err() != nil {
|
||||
break
|
||||
}
|
||||
if e == nil {
|
||||
// The end of what was there when the watch began.
|
||||
if initial {
|
||||
@@ -273,6 +468,11 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
|
||||
}
|
||||
continue
|
||||
}
|
||||
if initial {
|
||||
// Arrival from the bus is progress before the bundle is asked: the bound measures the current
|
||||
// values coming in, and restarts when the bundle has taken one.
|
||||
progress()
|
||||
}
|
||||
change := StateChange{State: name, Key: e.Key(), Revision: e.Revision(), Current: initial}
|
||||
switch e.Operation() {
|
||||
case jetstream.KeyValuePut:
|
||||
@@ -282,25 +482,50 @@ func (c *Conn) StateWatch(module, name, pattern string, deliver func(StateChange
|
||||
// A deletion among the current values is a key that is not there: not handed over
|
||||
// (measured, research 024 — the server sends its marker among the initial values).
|
||||
if initial {
|
||||
progress()
|
||||
continue
|
||||
}
|
||||
change.Op = "delete"
|
||||
if e.Operation() == jetstream.KeyValuePurge {
|
||||
// The marker the server leaves when a key's time to live ends (Nats-Marker-Reason MaxAge,
|
||||
// read by the client as a purge): nothing else purges a module's state, since neither the
|
||||
// runtime nor mesh-sdk's State offers a purge (issue 1952).
|
||||
change.Op = "expire"
|
||||
}
|
||||
}
|
||||
if watching.Err() != nil {
|
||||
break
|
||||
}
|
||||
if err := deliver(change); err != nil {
|
||||
c.Logf("[mesh-tools] %s did not take %s %s of %s: %v; its next change, or its next start, reads it again",
|
||||
module, change.Op, change.Key, name, err)
|
||||
}
|
||||
if initial {
|
||||
progress()
|
||||
}
|
||||
}
|
||||
if initial {
|
||||
close(current)
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-current:
|
||||
return stop, nil
|
||||
case <-time.After(StateTimeout):
|
||||
stop()
|
||||
return nil, fmt.Errorf("the current values of %s did not arrive in %s", name, StateTimeout)
|
||||
bound := time.NewTimer(StateTimeout)
|
||||
defer bound.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-current:
|
||||
return stop, nil
|
||||
case <-handed:
|
||||
if !bound.Stop() {
|
||||
select {
|
||||
case <-bound.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
bound.Reset(StateTimeout)
|
||||
case <-bound.C:
|
||||
stop()
|
||||
return nil, fmt.Errorf("the current values of %s did not arrive in %s: none was handed over in that time", name, StateTimeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,7 +543,10 @@ func valueOf(raw []byte) json.RawMessage {
|
||||
var credentialEndings = []string{"password", "passwd", "secret", "token", "credential", "credentials",
|
||||
"authorization", "apikey", "privatekey", "accesskey", "cookie"}
|
||||
|
||||
// credentialField is the first field anywhere in a JSON value whose name says it is a credential, or "".
|
||||
// credentialField is the first field anywhere in a JSON value whose name says it is a credential, or "". The one shape a
|
||||
// credential-shaped name may hold is a provider reference, an object of exactly `node` and `module`, both strings: a
|
||||
// the assignment's binding of the provision "secret" to the vault (issue 1126) names who provides it, never its value. Any
|
||||
// other shape under such a name is refused.
|
||||
func credentialField(value json.RawMessage) string {
|
||||
var v any
|
||||
if json.Unmarshal(value, &v) != nil {
|
||||
@@ -336,7 +564,7 @@ func credentialField(value json.RawMessage) string {
|
||||
for _, k := range keys {
|
||||
norm := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(k))
|
||||
for _, end := range credentialEndings {
|
||||
if strings.HasSuffix(norm, end) {
|
||||
if strings.HasSuffix(norm, end) && !providerReference(t[k]) {
|
||||
return k
|
||||
}
|
||||
}
|
||||
@@ -355,3 +583,17 @@ func credentialField(value json.RawMessage) string {
|
||||
}
|
||||
return walk(v)
|
||||
}
|
||||
|
||||
// providerReference says whether v is exactly {"node": <name>, "module": <name>}, each a name as mesh-sdk names them.
|
||||
func providerReference(v any) bool {
|
||||
m, ok := v.(map[string]any)
|
||||
if !ok || len(m) != 2 {
|
||||
return false
|
||||
}
|
||||
node, okN := m["node"].(string)
|
||||
module, okM := m["module"].(string)
|
||||
return okN && okM && referenceName.MatchString(node) && referenceName.MatchString(module)
|
||||
}
|
||||
|
||||
// referenceName is a node's or a module's name as mesh-sdk names them: lower case, digits and dashes.
|
||||
var referenceName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
//go:build integration
|
||||
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// **A state its owner keeps one bucket per node is served to a bundle as one state** (novox/hq ADR 0593 §6: each node's
|
||||
// engine writes its own `composed-<node>`, and the provisioner reads them all). Get goes to the bucket the key's first
|
||||
// part names, Keys and Watch over every bucket of the prefix, a bucket created during the watch included, and a bucket
|
||||
// that only starts like the prefix is never read.
|
||||
func TestAPerNodeStateIsServedAcrossEveryNodesBucket(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
a, b := mesh.Bucket(t, "composed-a"), mesh.Bucket(t, "composed-b")
|
||||
other := mesh.Bucket(t, "composedx-z")
|
||||
for kv, key := range map[interface {
|
||||
Put(string, []byte) (uint64, error)
|
||||
}]string{a: "a.m1.f1", b: "b.m2.f2", other: "z.m3.f3"} {
|
||||
if _, err := kv.Put(key, []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
prov := mt.MembershipOf("mesh-provisioner", "anchor", false, nil)
|
||||
prov.State = []bus.StateIssued{{Name: "mesh-host.composed", Bucket: "composed-", PerNode: true}}
|
||||
mesh.Issue(t, prov)
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("mesh-provisioner")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("mesh-provisioner") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if e, err := conn.StateGet("mesh-provisioner", "mesh-host.composed", "b.m2.f2"); err != nil || e == nil || e.Key != "b.m2.f2" || e.At.IsZero() || time.Since(e.At) > time.Minute {
|
||||
t.Fatalf("get from the second node's bucket: %+v, %v", e, err)
|
||||
}
|
||||
if e, err := conn.StateGet("mesh-provisioner", "mesh-host.composed", "q.m.f"); err != nil || e != nil {
|
||||
t.Fatalf("a key of a node with no bucket should read as none: %+v, %v", e, err)
|
||||
}
|
||||
if e, err := conn.StateGet("mesh-provisioner", "mesh-host.composed", "z.m3.f3"); err != nil || e != nil {
|
||||
t.Fatalf("the look-alike bucket was read: %+v, %v", e, err)
|
||||
}
|
||||
if _, err := conn.StateGet("mesh-provisioner", "mesh-host.composed", "nonode"); err == nil {
|
||||
t.Fatal("a key naming no node should be refused")
|
||||
}
|
||||
if keys, err := conn.StateKeys("mesh-provisioner", "mesh-host.composed"); err != nil || !reflect.DeepEqual(keys, []string{"a.m1.f1", "b.m2.f2"}) {
|
||||
t.Fatalf("keys across the buckets: %v, %v", keys, err)
|
||||
}
|
||||
if _, err := conn.StatePut("mesh-provisioner", "mesh-host.composed", "a.m1.f9", []byte(`{}`)); err == nil {
|
||||
t.Fatal("a per-node state is read only")
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var current, live []string
|
||||
stop, err := conn.StateWatch("mesh-provisioner", "mesh-host.composed", "", func(ch bus.StateChange) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if ch.Current {
|
||||
current = append(current, ch.Key)
|
||||
} else {
|
||||
live = append(live, ch.Key)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mu.Lock()
|
||||
sort.Strings(current)
|
||||
if !reflect.DeepEqual(current, []string{"a.m1.f1", "b.m2.f2"}) {
|
||||
t.Fatalf("current values: %v", current)
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
// A third node's bucket, created during the watch, and a change in a first one.
|
||||
c := mesh.Bucket(t, "composed-c")
|
||||
if _, err := c.Put("c.m4.f4", []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := a.Put("a.m1.f5", []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
got := strings.Join(append(append([]string(nil), current...), live...), ",")
|
||||
for _, want := range []string{"a.m1.f5", "c.m4.f4"} {
|
||||
if !strings.Contains(got, want) {
|
||||
return fmt.Errorf("%s not heard yet: current %v, changes %v", want, current, live)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -15,6 +15,14 @@ func TestACredentialNamedFieldIsFoundAnywhereInAValue(t *testing.T) {
|
||||
`{"clientSecret":"s"}`: "clientSecret",
|
||||
`{"maxTokens":4096,"name":"a","generation":3}`: "",
|
||||
`"just a string"`: "",
|
||||
// Issue 1126: an assignment binds the provision "secret" to the vault; only an exact provider reference passes.
|
||||
`{"bindings":{"secret":{"node":"anchor","module":"mesh-vault"}}}`: "",
|
||||
`{"secret":{"value":"s"}}`: "secret",
|
||||
`{"secret":{"node":"anchor","module":"mesh-vault","value":"s"}}`: "secret",
|
||||
`{"secret":{"node":"anchor","module":7}}`: "secret",
|
||||
`{"secret":{"node":"anchor","module":"hunter2 Bearer s3cr3t"}}`: "secret",
|
||||
`{"secret":["s"]}`: "secret",
|
||||
`{"bindings":{"secret":{"token":"s"}}}`: "secret",
|
||||
} {
|
||||
if got := credentialField(json.RawMessage(value)); got != want {
|
||||
t.Errorf("%s: found %q, want %q", value, got, want)
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
//go:build integration
|
||||
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// **A watch whose current values take longer than the bound to hand over still holds** (novox/hq issue
|
||||
// 1726). The gitea module watches the checker's owed: one key per pull request head, each a round trip
|
||||
// to the bundle. With a few hundred keys the hand-over outlasted StateTimeout at every attempt, the
|
||||
// watch was refused, and the newest keys, handed over last, never reached mesh/check: a group's passing
|
||||
// verdict was never written on its members' heads. The bound is on progress: a watch fails only when
|
||||
// no current value is handed over within it.
|
||||
func TestAWatchHoldsWhileItsCurrentValuesKeepArriving(t *testing.T) {
|
||||
was := bus.StateTimeout
|
||||
bus.StateTimeout = 400 * time.Millisecond
|
||||
t.Cleanup(func() { bus.StateTimeout = was })
|
||||
|
||||
conn, kv := owedWatcher(t)
|
||||
// A group of three pull requests judged last, after many older heads: their keys come last.
|
||||
const older = 30
|
||||
var keys []string
|
||||
for i := 0; i < older; i++ {
|
||||
keys = append(keys, fmt.Sprintf("novox/mesh-catalog.%d.%040x.status=mesh/check", 400+i, i))
|
||||
}
|
||||
for _, member := range []string{"novox/mesh-sdk.76.34f210e5", "novox/mesh-module-assigner.453.c06f75d0", "novox/mesh-delivery.33.28ee22ef"} {
|
||||
keys = append(keys, member+strings.Repeat("0", 32)+".status=mesh/check")
|
||||
}
|
||||
for _, k := range keys {
|
||||
if _, err := kv.Put(k, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"success"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var current, live []string
|
||||
// Each value takes 40 ms to be taken: 33 values take about 1.3 s, three times the bound.
|
||||
stop, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(ch bus.StateChange) error {
|
||||
if ch.Current {
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if ch.Current {
|
||||
current = append(current, ch.Key)
|
||||
} else {
|
||||
live = append(live, ch.Key)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a watch whose current values kept arriving was refused: %v", err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mu.Lock()
|
||||
got := append([]string(nil), current...)
|
||||
mu.Unlock()
|
||||
if len(got) != len(keys) {
|
||||
t.Fatalf("the watch returned after %d of %d current values", len(got), len(keys))
|
||||
}
|
||||
for i, k := range keys {
|
||||
if got[i] != k {
|
||||
t.Fatalf("current value %d is %s, want %s", i, got[i], k)
|
||||
}
|
||||
}
|
||||
|
||||
// And it holds: a verdict written afterwards reaches it as a change.
|
||||
next := "novox/mesh-sdk.77." + strings.Repeat("a", 40) + ".status=mesh/check"
|
||||
if _, err := kv.Put(next, []byte(`{"v":1,"act":"status","context":"mesh/check","state":"pending"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(live) != 1 || live[0] != next {
|
||||
return fmt.Errorf("live changes so far: %v", live)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// A watch whose bundle takes nothing for longer than the bound is still refused, so a bundle that hangs
|
||||
// is said, not waited on for ever.
|
||||
func TestAWatchIsRefusedWhenNoCurrentValueIsTakenWithinTheBound(t *testing.T) {
|
||||
was := bus.StateTimeout
|
||||
bus.StateTimeout = 300 * time.Millisecond
|
||||
t.Cleanup(func() { bus.StateTimeout = was })
|
||||
|
||||
conn, kv := owedWatcher(t)
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
release := make(chan struct{})
|
||||
t.Cleanup(func() { close(release) })
|
||||
began := time.Now()
|
||||
_, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", func(bus.StateChange) error {
|
||||
<-release
|
||||
return nil
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "did not arrive in") {
|
||||
t.Fatalf("a watch whose first value was never taken: want refused, got %v", err)
|
||||
}
|
||||
if took := time.Since(began); took > 3*time.Second {
|
||||
t.Fatalf("the refusal took %s, far past the bound", took)
|
||||
}
|
||||
}
|
||||
|
||||
// **A membership the grants part wrote in its old wrapper is served whole** (issue 774). Until 2026-10-30 the part wrote
|
||||
// its pass record, the membership inside, into its state memberships; the tool runner read the value as a bare
|
||||
// membership, and served no tool and no state. This is what the live mesh held: read through mesh-sdk's codec, its tools
|
||||
// are answered where the membership says and its state is reached. A held-back one's wrapper is refused, and the module has
|
||||
// no membership then (774/S32/AC2: no stream is read in its place).
|
||||
func TestAMembershipTheGrantsPartWroteWrappedIsServed(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Bucket(t, "gitea_repos")
|
||||
gitea := mt.MembershipOf("gitea", "anchor", false, nil)
|
||||
gitea.State = []bus.StateIssued{{Name: "repos", Bucket: "gitea_repos", Writes: true}}
|
||||
mesh.IssueInGrantsWrapped(t, gitea, "published")
|
||||
// The old wrapper of a membership the part held back carries its unpublished composition: refused, never served.
|
||||
heldBack := mt.MembershipOf("pages", "anchor", true, nil)
|
||||
mesh.IssueInGrantsWrapped(t, heldBack, "not published, a shadow difference: publishing would lose the last holder of x")
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var logged []string
|
||||
conn.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("gitea")
|
||||
mt.Until(t, func() error {
|
||||
if m := conn.Membership("gitea"); m == nil || len(m.Serves) == 0 {
|
||||
return fmt.Errorf("gitea's membership is %+v, want the one inside the wrapper", m)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if got, want := fmt.Sprint(conn.ServedOn("gitea", "gitea_list")), "[{mesh.mod.gitea.tool.gitea_list.anchor }]"; got != want {
|
||||
t.Errorf("gitea_list is served on %s, want %s", got, want)
|
||||
}
|
||||
if _, err := conn.StatePut("gitea", "repos", "novox.hq", []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatalf("the state the membership issues is refused: %v", err)
|
||||
}
|
||||
if e, err := conn.StateGet("gitea", "repos", "novox.hq"); err != nil || e == nil || string(e.Value) != `{"v":1}` || e.At.IsZero() || time.Since(e.At) > time.Minute {
|
||||
t.Errorf("the state reads back %+v, %v", e, err)
|
||||
}
|
||||
conn.Follow("pages")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if m := conn.Membership("pages"); m != nil || !strings.Contains(strings.Join(logged, "\n"), "no membership issued for pages") {
|
||||
t.Errorf("a held-back membership's wrapper was served or not said: %+v, %q", m, logged)
|
||||
}
|
||||
}
|
||||
|
||||
// owedWatcher is the gitea module's view of the checker's owed: the bucket, and a runtime connection
|
||||
// that serves gitea with the owed issued for reading.
|
||||
func owedWatcher(t *testing.T) (*bus.Conn, interface {
|
||||
Put(string, []byte) (uint64, error)
|
||||
}) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
kv := mesh.Bucket(t, "mesh-delivery-checker_owed")
|
||||
gitea := mt.MembershipOf("gitea", "anchor", false, nil)
|
||||
gitea.State = []bus.StateIssued{{Name: "mesh-delivery-checker.owed", Bucket: "mesh-delivery-checker_owed"}}
|
||||
mesh.Issue(t, gitea)
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("gitea")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("gitea") == nil {
|
||||
return errors.New("gitea not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return conn, kv
|
||||
}
|
||||
|
||||
// **A watch given up, or stopped, hands over nothing more** (issue 774). The runtime's watch stopped its watcher
|
||||
// when the current values did not arrive within the bound, but the watcher still held values, and its goroutine
|
||||
// went on handing them over: to a watch id the bundle no longer held (on the provisioner, 51 failed watches were
|
||||
// followed by 1257 orphan changes). Each row blocks the bundle on its first value, gives the watch up, lets the
|
||||
// bundle go and counts what it is still handed.
|
||||
func TestAWatchGivenUpHandsOverNothingMore(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
refused bool // the bound gives it up, or the caller stops it
|
||||
}{
|
||||
{"refused at the bound", true},
|
||||
{"stopped by its caller", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
was := bus.StateTimeout
|
||||
bus.StateTimeout = 300 * time.Millisecond
|
||||
t.Cleanup(func() { bus.StateTimeout = was })
|
||||
conn, kv := owedWatcher(t)
|
||||
put := func(i int) {
|
||||
if _, err := kv.Put(fmt.Sprintf("novox/hq.%d.%040x.status=mesh/check", i+1, i), []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if tc.refused {
|
||||
for i := 0; i < 4; i++ {
|
||||
put(i)
|
||||
}
|
||||
}
|
||||
var calls atomic.Int32
|
||||
first := make(chan struct{}, 1)
|
||||
release := make(chan struct{})
|
||||
deliver := func(ch bus.StateChange) error {
|
||||
if calls.Add(1) == 1 {
|
||||
first <- struct{}{}
|
||||
<-release
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var stop func()
|
||||
if tc.refused {
|
||||
if _, err := conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err == nil {
|
||||
t.Fatal("the watch whose bundle took nothing was not refused")
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
if stop, err = conn.StateWatch("gitea", "mesh-delivery-checker.owed", "", deliver); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
put(i)
|
||||
}
|
||||
select {
|
||||
case <-first:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the bundle was handed no change")
|
||||
}
|
||||
stop()
|
||||
}
|
||||
// Let the watcher's server side deliver what it held, then free the bundle.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
close(release)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if n := calls.Load(); n != 1 {
|
||||
t.Fatalf("the bundle was handed %d changes of a watch given up, want 1 (the one in hand)", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A module's own stream, as the runtime publishes to it and hears it (novox/hq ADR 0363 §8, issue 725).
|
||||
//
|
||||
// A module may declare a stream of its own; the mesh creates it and says, in the module's
|
||||
// membership, which of its events go there and on which subject (`streams`), and, in the membership of
|
||||
// each module granted to hear it, which durable consumer on it to bind (`hears`). The runtime publishes and
|
||||
// binds exactly as the membership says. A membership naming neither is what every membership was before:
|
||||
// every event on EVENTS, and only the consumer on EVENTS bound.
|
||||
|
||||
// StreamIssued is one of a module's own streams as its membership names it: the events it carries, as the
|
||||
// bus spells them (`*` one token, `>` the rest), and the subject an event of them is published on, `{event}`
|
||||
// standing for the event.
|
||||
type StreamIssued struct {
|
||||
Stream string `json:"stream"`
|
||||
Events []string `json:"events"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
|
||||
// HeardIssued is one module stream an assignment hears: the stream and the durable consumer on it the
|
||||
// runtime binds. The consumer is the mesh's to make, to filter and to delete.
|
||||
type HeardIssued struct {
|
||||
Stream string `json:"stream"`
|
||||
Consumer string `json:"consumer"`
|
||||
}
|
||||
|
||||
// eventSubject is where a module's event is published: on the subject of the first of its membership's
|
||||
// streams that carries it, else on EVENTS — `mesh.mod.<module>.event.<event>`, as every event was before.
|
||||
func eventSubject(m *Membership, module, event string) string {
|
||||
if m != nil {
|
||||
for _, s := range m.Streams {
|
||||
if s.Subject == "" || !strings.Contains(s.Subject, "{event}") {
|
||||
continue
|
||||
}
|
||||
for _, pattern := range s.Events {
|
||||
if eventMatches(pattern, event) {
|
||||
return strings.ReplaceAll(s.Subject, "{event}", event)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return "mesh.mod." + module + ".event." + event
|
||||
}
|
||||
|
||||
// eventMatches says whether an event's name falls under a pattern as the bus matches subjects: `*` is one
|
||||
// token, `>` (or `**`, the manifest's spelling) the one or more tokens left.
|
||||
func eventMatches(pattern, event string) bool {
|
||||
if pattern == "" || event == "" {
|
||||
return false
|
||||
}
|
||||
ps, es := strings.Split(pattern, "."), strings.Split(event, ".")
|
||||
for i, p := range ps {
|
||||
if p == ">" || p == "**" {
|
||||
return i == len(ps)-1 && len(es) > i
|
||||
}
|
||||
if i >= len(es) || (p != "*" && p != es[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(ps) == len(es)
|
||||
}
|
||||
|
||||
// RebindEvery is how often a consumer the membership names and the bus did not let the runtime bind is
|
||||
// tried again: the mesh makes the consumer and sends the user list carrying its grant in the
|
||||
// composition that issues the membership, so one may arrive before the other (ADR 0363 §6).
|
||||
var RebindEvery = 30 * time.Second
|
||||
|
||||
// hearing is one module's consumers: the one on EVENTS, and each on a module stream its membership names.
|
||||
// Every event of all of them is handed to the module one at a time, as the consumer on EVENTS alone did.
|
||||
type hearing struct {
|
||||
c *Conn
|
||||
js jetstream.JetStream
|
||||
module string
|
||||
deliver func(Envelope) error
|
||||
// serial hands the module one event at a time across its consumers (novox/hq issue 387).
|
||||
serial sync.Mutex
|
||||
|
||||
// halted is stopped, read by handle without mu: the bus's loops hand events while mu is held.
|
||||
halted atomic.Bool
|
||||
|
||||
mu sync.Mutex
|
||||
stopped bool
|
||||
bound map[HeardIssued]jetstream.ConsumeContext
|
||||
// made is when the bus made each bound consumer: a consumer deleted and made again under its name has
|
||||
// another time, and a loop bound to the old one is not reading the new one.
|
||||
made map[HeardIssued]time.Time
|
||||
said map[HeardIssued]bool
|
||||
// why is the last error the bus gave a consumer's loop, said when the loop ends on it.
|
||||
// Guarded by whyMu, not mu: the bus hands errors from its own goroutines, possibly while mu is held.
|
||||
whyMu sync.Mutex
|
||||
why map[HeardIssued]error
|
||||
kick chan struct{}
|
||||
done chan struct{}
|
||||
unhear func()
|
||||
}
|
||||
|
||||
func newHearing(c *Conn, js jetstream.JetStream, module string, deliver func(Envelope) error) *hearing {
|
||||
return &hearing{c: c, js: js, module: module, deliver: deliver,
|
||||
bound: map[HeardIssued]jetstream.ConsumeContext{}, made: map[HeardIssued]time.Time{}, said: map[HeardIssued]bool{}, why: map[HeardIssued]error{},
|
||||
kick: make(chan struct{}, 1), done: make(chan struct{})}
|
||||
}
|
||||
|
||||
// bind reads one durable consumer by name, and on failure says what it was doing. Errors the bus gives
|
||||
// the loop afterwards are handed to onErr, when given.
|
||||
func (h *hearing) bind(stream, durable string, onErr func(error)) (jetstream.ConsumeContext, time.Time, string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
consumer, err := h.js.Consumer(ctx, stream, durable)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, "binding", err
|
||||
}
|
||||
var created time.Time
|
||||
if info := consumer.CachedInfo(); info != nil {
|
||||
created = info.Created
|
||||
}
|
||||
opts := []jetstream.PullConsumeOpt{jetstream.PullMaxMessages(1)}
|
||||
if onErr != nil {
|
||||
opts = append(opts, jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { onErr(err) }))
|
||||
}
|
||||
reading, err := consumer.Consume(h.handle, opts...)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, "reading", err
|
||||
}
|
||||
return reading, created, "", nil
|
||||
}
|
||||
|
||||
// handle hands one delivered event to the module: acknowledged when it took it, offered again after NakDelay
|
||||
// when it did not, terminated when it is not an event at all.
|
||||
func (h *hearing) handle(msg jetstream.Msg) {
|
||||
env, ok := envelopeOf(msg.Subject(), msg.Headers(), msg.Data())
|
||||
if !ok {
|
||||
// Unparseable: redelivering bytes no version of a handler can read is a loop.
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
// Kept alive while the bundle works on it, as a worker's work is (seat.go): the consumer's ack wait
|
||||
// (30 seconds) is for a runtime that died, not a handler that is slow — the runtime gives a handler
|
||||
// launch.EventTimeout. Without it an event handled longer than the ack wait was offered again while
|
||||
// it was handled, and counted toward max-deliver (novox/hq issue 387). Kept alive too while it waits
|
||||
// for an event from another of the module's consumers to be handled.
|
||||
done, stopped := make(chan struct{}), make(chan struct{})
|
||||
go func() {
|
||||
defer close(stopped)
|
||||
tick := time.NewTicker(EventHeartbeat)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
_ = msg.InProgress()
|
||||
}
|
||||
}
|
||||
}()
|
||||
h.serial.Lock()
|
||||
halted := h.halted.Load()
|
||||
var err error
|
||||
if !halted {
|
||||
err = h.deliver(env)
|
||||
}
|
||||
h.serial.Unlock()
|
||||
close(done)
|
||||
// No in-progress after the answer: one after a Nak would stretch its delay to the whole ack wait.
|
||||
<-stopped
|
||||
if halted {
|
||||
// **A hearing stopped hands nothing on** (issue 1945): the module's bundle that subscribed may be
|
||||
// gone, loaded again in its place; offered again at once, to the consumer bound after it.
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
_ = msg.NakWithDelay(NakDelay)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
}
|
||||
|
||||
// start binds what the membership names now, and follows it: a new membership binds what it adds and stops
|
||||
// what it no longer names, and a consumer not bound yet is tried again every RebindEvery.
|
||||
func (h *hearing) start() {
|
||||
h.reconcile()
|
||||
h.unhear = h.c.onMembership(func(m Membership) {
|
||||
if m.Module != h.module {
|
||||
return
|
||||
}
|
||||
h.nudge()
|
||||
})
|
||||
go func() {
|
||||
tick := time.NewTicker(RebindEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-h.done:
|
||||
return
|
||||
case <-h.kick:
|
||||
case <-tick.C:
|
||||
}
|
||||
h.reconcile()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// forget drops the last error a consumer's loop was given.
|
||||
func (h *hearing) forget(heard HeardIssued) {
|
||||
h.whyMu.Lock()
|
||||
delete(h.why, heard)
|
||||
h.whyMu.Unlock()
|
||||
}
|
||||
|
||||
// nudge asks the loop to reconcile now.
|
||||
func (h *hearing) nudge() {
|
||||
select {
|
||||
case h.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// stop stops every consumer on a module stream this module reads; the one on EVENTS is its caller's.
|
||||
func (h *hearing) stop() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.stopped {
|
||||
return
|
||||
}
|
||||
h.stopped = true
|
||||
h.halted.Store(true)
|
||||
close(h.done)
|
||||
if h.unhear != nil {
|
||||
h.unhear()
|
||||
}
|
||||
for heard, reading := range h.bound {
|
||||
reading.Stop()
|
||||
delete(h.bound, heard)
|
||||
delete(h.made, heard)
|
||||
}
|
||||
}
|
||||
|
||||
// replaced says whether the consumer a loop is bound to is gone from the bus or was made again since. A bus
|
||||
// that cannot be asked right now says no: the loop is left as it is.
|
||||
func (h *hearing) replaced(heard HeardIssued) bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
consumer, err := h.js.Consumer(ctx, heard.Stream, heard.Consumer)
|
||||
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
return true
|
||||
}
|
||||
if err != nil || consumer.CachedInfo() == nil {
|
||||
return false
|
||||
}
|
||||
created := h.made[heard]
|
||||
return !created.IsZero() && !consumer.CachedInfo().Created.Equal(created)
|
||||
}
|
||||
|
||||
// reconcile makes what is bound what the module's membership names now.
|
||||
func (h *hearing) reconcile() {
|
||||
want := map[HeardIssued]bool{}
|
||||
if m := h.c.Membership(h.module); m != nil {
|
||||
for _, heard := range m.Hears {
|
||||
if heard.Stream != "" && heard.Consumer != "" {
|
||||
want[heard] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.stopped {
|
||||
return
|
||||
}
|
||||
for heard, reading := range h.bound {
|
||||
if !want[heard] {
|
||||
reading.Stop()
|
||||
delete(h.bound, heard)
|
||||
delete(h.made, heard)
|
||||
h.c.Logf("[mesh-tools] %s no longer hears %s: its membership stopped naming its consumer %s",
|
||||
h.module, heard.Stream, heard.Consumer)
|
||||
}
|
||||
}
|
||||
for heard := range h.said {
|
||||
if !want[heard] {
|
||||
delete(h.said, heard)
|
||||
h.forget(heard)
|
||||
}
|
||||
}
|
||||
// **A loop the bus ended is no longer reading** (nats.go ends a pull loop on its own when its consumer is
|
||||
// deleted): the mesh deletes and makes again under the same name a restricted consumer it cannot
|
||||
// narrow (ADR 0363 §6), and the membership, unchanged, says nothing — so it is bound again here.
|
||||
// **A loop the bus did not end is asked about**: nats.go learns of a deleted consumer only from the answer
|
||||
// to a pull it has pending, and a consumer deleted and made again between two pulls, or while none is
|
||||
// pending, leaves the loop waiting out its heartbeat (30 seconds) on a consumer that is gone. So each
|
||||
// bound consumer is looked up by name: gone, or made at another time, ends its loop here.
|
||||
for heard, reading := range h.bound {
|
||||
if h.replaced(heard) {
|
||||
h.whyMu.Lock()
|
||||
if h.why[heard] == nil {
|
||||
h.why[heard] = errors.New("the consumer was deleted or made again")
|
||||
}
|
||||
h.whyMu.Unlock()
|
||||
reading.Stop()
|
||||
}
|
||||
}
|
||||
for heard, reading := range h.bound {
|
||||
select {
|
||||
case <-reading.Closed():
|
||||
delete(h.bound, heard)
|
||||
delete(h.made, heard)
|
||||
h.whyMu.Lock()
|
||||
why := h.why[heard]
|
||||
h.whyMu.Unlock()
|
||||
if why == nil {
|
||||
why = errors.New("no reason given")
|
||||
}
|
||||
h.c.Logf("[mesh-tools] %s stopped hearing %s: the bus ended reading its consumer %s (%v); bound again as it exists",
|
||||
h.module, heard.Stream, heard.Consumer, why)
|
||||
h.said[heard] = true
|
||||
default:
|
||||
}
|
||||
}
|
||||
missing := make([]HeardIssued, 0, len(want))
|
||||
for heard := range want {
|
||||
if h.bound[heard] == nil {
|
||||
missing = append(missing, heard)
|
||||
}
|
||||
}
|
||||
sort.Slice(missing, func(i, j int) bool { return missing[i].Stream < missing[j].Stream })
|
||||
for _, heard := range missing {
|
||||
heard := heard
|
||||
h.forget(heard)
|
||||
reading, created, doing, err := h.bind(heard.Stream, heard.Consumer, func(err error) {
|
||||
h.whyMu.Lock()
|
||||
h.why[heard] = err
|
||||
h.whyMu.Unlock()
|
||||
})
|
||||
if err != nil {
|
||||
// Said once, not at every attempt; said again when it is bound.
|
||||
if !h.said[heard] {
|
||||
h.c.Logf("[mesh-tools] %s does not hear %s yet: %s its consumer %s failed: %v; tried again every %s",
|
||||
h.module, heard.Stream, doing, heard.Consumer, err, RebindEvery)
|
||||
h.said[heard] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
h.bound[heard] = reading
|
||||
h.made[heard] = created
|
||||
delete(h.said, heard)
|
||||
h.c.Logf("[mesh-tools] %s hears %s on its consumer %s", h.module, heard.Stream, heard.Consumer)
|
||||
// When the bus ends the loop, look again at once rather than at the next tick.
|
||||
go func() {
|
||||
select {
|
||||
case <-reading.Closed():
|
||||
h.nudge()
|
||||
case <-h.done:
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package bus
|
||||
|
||||
import "testing"
|
||||
|
||||
// novox/hq ADR 0363 §2, §3: an event goes to the stream whose pattern carries it, as the bus matches subjects.
|
||||
func TestAnEventFallsUnderAStreamsPatternAsTheBusMatchesIt(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
pattern, event string
|
||||
want bool
|
||||
}{
|
||||
{"mail.*.message.added", "mail.home.message.added", true},
|
||||
{"mail.*.message.added", "mail.home.message.removed", false},
|
||||
{"mail.*.message.added", "mail.home.work.message.added", false},
|
||||
{"mail.>", "mail.home.message.added", true},
|
||||
{"mail.**", "mail.home", true},
|
||||
{"mail.>", "mail", false},
|
||||
{"status", "status", true},
|
||||
{"status", "status.changed", false},
|
||||
{"", "status", false},
|
||||
} {
|
||||
if got := eventMatches(c.pattern, c.event); got != c.want {
|
||||
t.Errorf("%q under %q: %v, want %v", c.event, c.pattern, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A membership naming no stream publishes every event on EVENTS, exactly as before (issue 725).
|
||||
func TestAnEventNoStreamCarriesIsPublishedOnEvents(t *testing.T) {
|
||||
mail := &Membership{Module: "mail", Streams: []StreamIssued{{Stream: "MAIL", Events: []string{"mail.*.message.added"},
|
||||
Subject: "mesh.stream.mail.event.{event}"}}}
|
||||
for _, c := range []struct {
|
||||
m *Membership
|
||||
event string
|
||||
want string
|
||||
}{
|
||||
{nil, "mail.home.message.added", "mesh.mod.mail.event.mail.home.message.added"},
|
||||
{&Membership{Module: "mail"}, "mail.home.message.added", "mesh.mod.mail.event.mail.home.message.added"},
|
||||
{mail, "mail.home.message.added", "mesh.stream.mail.event.mail.home.message.added"},
|
||||
{mail, "status", "mesh.mod.mail.event.status"},
|
||||
} {
|
||||
if got := eventSubject(c.m, "mail", c.event); got != c.want {
|
||||
t.Errorf("%s: %s, want %s", c.event, got, c.want)
|
||||
}
|
||||
}
|
||||
if got := keyOf("mesh.stream.mail.event.mail.home.message.added"); got != "mail.mail.home.message.added" {
|
||||
t.Errorf("a stream's event reaches its consumer as %q, not as on EVENTS", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// streamMesh is a mesh with the mail module's own stream MAIL, and a connection on anchor following the
|
||||
// memberships of the modules named.
|
||||
func streamMesh(t *testing.T, memberships ...bus.Membership) (*mt.Mesh, *bus.Conn) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
mesh.ModuleStream(t, "mail", "MAIL")
|
||||
for _, m := range memberships {
|
||||
mesh.Issue(t, m)
|
||||
}
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
for _, m := range memberships {
|
||||
conn.Follow(m.Module)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
for _, m := range memberships {
|
||||
if conn.Membership(m.Module) == nil {
|
||||
return errors.New(m.Module + " not issued yet")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return mesh, conn
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §2, §8 (issue 725): an event one of the module's streams carries is published on that
|
||||
// stream's subject and kept by that stream alone; any other event of it goes to EVENTS as before.
|
||||
func TestAnEventADeclaredStreamCarriesIsPublishedOnItsSubjectAlone(t *testing.T) {
|
||||
mail := mt.MembershipOf("mail", "anchor", true, nil)
|
||||
mail.Streams = []bus.StreamIssued{{Stream: "MAIL", Events: []string{"mail.*.message.added"},
|
||||
Subject: "mesh.stream.mail.event.{event}"}}
|
||||
mesh, conn := streamMesh(t, mail)
|
||||
|
||||
landed := mesh.NextEvent(t, "mesh.stream.mail.event.>")
|
||||
if err := conn.PublishAs("mail", bus.Envelope{Key: "mail.home.message.added", Body: json.RawMessage(`{"id":1}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case got := <-landed:
|
||||
if got != "mesh.stream.mail.event.mail.home.message.added" {
|
||||
t.Errorf("published on %s", got)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the event did not land on its stream's subject")
|
||||
}
|
||||
if err := conn.PublishAs("mail", bus.Envelope{Key: "status", Body: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := mesh.Messages(t, "MAIL"); n != 1 {
|
||||
t.Errorf("MAIL holds %d messages, want the one event it carries", n)
|
||||
}
|
||||
if n := mesh.Messages(t, "EVENTS"); n != 1 {
|
||||
t.Errorf("EVENTS holds %d messages, want only the event no stream carries", n)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §8 (issue 725): a module hears, beside EVENTS, every module stream its membership names,
|
||||
// one event at a time, with the key it reads on EVENTS; a membership naming none hears EVENTS alone.
|
||||
func TestAModuleHearsEveryStreamItsMembershipNames(t *testing.T) {
|
||||
reader := mt.MembershipOf("reader", "anchor", false, nil)
|
||||
mesh, conn := streamMesh(t, reader)
|
||||
mesh.Consumer(t, "anchor", "reader", []string{"mesh.mod.alpha.event.>"}, 5*time.Second)
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.mail.home.>"}, 5*time.Second)
|
||||
|
||||
var mu sync.Mutex
|
||||
heard := map[string]int{}
|
||||
handling := 0
|
||||
stop, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||
mu.Lock()
|
||||
handling++
|
||||
if handling > 1 {
|
||||
t.Error("two events were handed to the module at once")
|
||||
}
|
||||
mu.Unlock()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
mu.Lock()
|
||||
handling--
|
||||
heard[env.Key]++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
count := func(key string) int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return heard[key]
|
||||
}
|
||||
|
||||
// Its membership names no stream: EVENTS alone, as before.
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 1})
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error {
|
||||
if count("alpha.happened") != 1 {
|
||||
return errors.New("the event on EVENTS was not heard")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if _, notYet := mesh.PendingOn(t, "MAIL", "anchor", "reader"); notYet != 1 || count("mail.mail.home.message.added") != 0 {
|
||||
t.Fatalf("a module whose membership names no stream heard MAIL (%d not yet delivered)", notYet)
|
||||
}
|
||||
|
||||
// Its membership now names its consumer on MAIL: bound, and what waited there is heard.
|
||||
reader.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "anchor_reader"}}
|
||||
mesh.Issue(t, reader)
|
||||
mt.Until(t, func() error {
|
||||
if count("mail.mail.home.message.added") != 1 {
|
||||
return errors.New("the event on MAIL was not heard")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for n := 2; n <= 4; n++ {
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": n})
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": n})
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
if a, m := count("alpha.happened"), count("mail.mail.home.message.added"); a != 4 || m != 4 {
|
||||
return fmt.Errorf("heard %d on EVENTS and %d on MAIL, want 4 and 4", a, m)
|
||||
}
|
||||
for _, stream := range []string{"EVENTS", "MAIL"} {
|
||||
if ack, notYet := mesh.PendingOn(t, stream, "anchor", "reader"); ack != 0 || notYet != 0 {
|
||||
return fmt.Errorf("%s still holds %d unacknowledged, %d undelivered", stream, ack, notYet)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// A membership that stops naming it stops what it reads there.
|
||||
reader.Hears = nil
|
||||
mesh.Issue(t, reader)
|
||||
mt.Until(t, func() error {
|
||||
if mesh.Consumers(t, "MAIL") != 1 {
|
||||
return errors.New("MAIL's consumer is gone")
|
||||
}
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 0})
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if _, notYet := mesh.PendingOn(t, "MAIL", "anchor", "reader"); notYet == 0 {
|
||||
return errors.New("MAIL is still read")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// The membership as the grants part keeps it (ADR 0571 clause 7, 10): read first and applied live, and nothing
|
||||
// published on the controller's subject is heard.
|
||||
granted := reader
|
||||
granted.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "from-grants"}}
|
||||
mesh.Issue(t, granted)
|
||||
again, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again.Logf = func(string, ...any) {}
|
||||
t.Cleanup(again.Close)
|
||||
again.Follow("reader")
|
||||
consumerOf := func() string {
|
||||
if m := again.Membership("reader"); m != nil && len(m.Hears) == 1 {
|
||||
return m.Hears[0].Consumer
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if got := consumerOf(); got != "from-grants" {
|
||||
t.Fatalf("the membership read at the follow names %q, want the grants part's", got)
|
||||
}
|
||||
granted.Hears[0].Consumer = "from-grants-2"
|
||||
mesh.Issue(t, granted)
|
||||
mt.Until(t, func() error {
|
||||
if got := consumerOf(); got != "from-grants-2" {
|
||||
return fmt.Errorf("the membership names %q, want the grants part's change", got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// 774/S32/AC1: a copy said on the controller's subject (mesh.assignment) changes nothing; there is no stream to read it from.
|
||||
streamed := reader
|
||||
streamed.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "from-subject"}}
|
||||
mesh.Publish(t, "mesh.assignment.anchor.reader", streamed)
|
||||
again.Flush()
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if got := consumerOf(); got != "from-grants-2" {
|
||||
t.Fatalf("a membership said on the controller's subject was applied: the membership names %q", got)
|
||||
}
|
||||
// Review g1 round 1: memberships that cannot be read at the follow (a bus not ready after its boot) are read again until
|
||||
// they can be; the follow, which runs once per module, does not give the module up for good.
|
||||
was := bus.GrantsRetry
|
||||
bus.GrantsRetry = 50 * time.Millisecond
|
||||
t.Cleanup(func() { bus.GrantsRetry = was })
|
||||
mesh.DropGrants(t)
|
||||
late, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
late.Logf = func(string, ...any) {}
|
||||
t.Cleanup(late.Close)
|
||||
late.Follow("reader")
|
||||
if m := late.Membership("reader"); m != nil {
|
||||
t.Fatalf("a membership was served from a state not there: %+v", m)
|
||||
}
|
||||
mesh.Issue(t, granted)
|
||||
mt.Until(t, func() error {
|
||||
if m := late.Membership("reader"); m == nil || len(m.Hears) != 1 || m.Hears[0].Consumer != "from-grants-2" {
|
||||
return fmt.Errorf("the membership is %+v, want the one written once the state is there", m)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §6 (issue 725): a consumer the membership names before the controller made it is bound once
|
||||
// it exists, without the module's events on EVENTS waiting for it.
|
||||
func TestAConsumerNamedBeforeItExistsIsBoundOnceItDoes(t *testing.T) {
|
||||
restore := bus.RebindEvery
|
||||
bus.RebindEvery = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.RebindEvery = restore })
|
||||
reader := mt.MembershipOf("reader", "anchor", false, nil)
|
||||
reader.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "anchor_reader"}}
|
||||
mesh, conn := streamMesh(t, reader)
|
||||
mesh.Consumer(t, "anchor", "reader", []string{"mesh.mod.alpha.event.>"}, 5*time.Second)
|
||||
|
||||
var mu sync.Mutex
|
||||
heard := map[string]int{}
|
||||
stop, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||
mu.Lock()
|
||||
heard[env.Key]++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a consumer on a module stream not made yet kept the module from hearing EVENTS: %v", err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{})
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.>"}, 5*time.Second)
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{})
|
||||
mt.Until(t, func() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if heard["alpha.happened"] != 1 || heard["mail.mail.home.message.added"] != 1 {
|
||||
return fmt.Errorf("heard %v", heard)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// novox/hq ADR 0363 §6 (issue 725): a consumer the controller deletes and makes again under the same name, its
|
||||
// membership unchanged, is bound again, and what is published after is heard — the module does not go deaf.
|
||||
func TestAConsumerDeletedAndMadeAgainIsHeardAgain(t *testing.T) {
|
||||
restore := bus.RebindEvery
|
||||
bus.RebindEvery = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.RebindEvery = restore })
|
||||
reader := mt.MembershipOf("reader", "anchor", false, nil)
|
||||
reader.Hears = []bus.HeardIssued{{Stream: "MAIL", Consumer: "anchor_reader"}}
|
||||
mesh, conn := streamMesh(t, reader)
|
||||
mesh.Consumer(t, "anchor", "reader", []string{"mesh.mod.alpha.event.>"}, 5*time.Second)
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.>"}, 5*time.Second)
|
||||
logs := &syncLogs{}
|
||||
conn.Logf = logs.Logf
|
||||
|
||||
var mu sync.Mutex
|
||||
heard := 0
|
||||
stop, err := conn.ConsumeAs("reader", func(env bus.Envelope) error {
|
||||
if env.Key == "mail.mail.home.message.added" {
|
||||
mu.Lock()
|
||||
heard++
|
||||
mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
count := func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return heard
|
||||
}
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error {
|
||||
if count() != 1 {
|
||||
return errors.New("the first event on MAIL was not heard")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
mesh.RemoveConsumer(t, "MAIL", "anchor", "reader")
|
||||
mt.Until(t, func() error {
|
||||
if !logs.Has("stopped hearing MAIL") {
|
||||
return errors.New("the end of the loop was not said")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
mesh.ConsumerOn(t, "MAIL", "anchor", "reader", []string{"mesh.stream.mail.event.>"}, 5*time.Second)
|
||||
mesh.Emit(t, "mesh.stream.mail.event.mail.home.message.added", map[string]any{"n": 2})
|
||||
mt.Until(t, func() error {
|
||||
if count() != 2 {
|
||||
return fmt.Errorf("heard %d events on MAIL after its consumer was made again, want 2", count())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// syncLogs keeps what the runtime says, from any goroutine.
|
||||
type syncLogs struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (l *syncLogs) Logf(format string, args ...any) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.lines = append(l.lines, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *syncLogs) Has(fragment string) bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
for _, line := range l.lines {
|
||||
if strings.Contains(line, fragment) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// Waiting on the bus for one event (novox/hq ADR 0367): what the mesh MCP server's `mesh_wait` does.
|
||||
//
|
||||
// **A subscription for the call, and nothing kept** (§2): no durable consumer, no queue, no stream. It
|
||||
// subscribes the subjects when called and unsubscribes them before it answers, so an event published
|
||||
// between two calls is not heard — a session reads the state from its owner after every wake (§4).
|
||||
|
||||
// WaitIssued is one event family the mesh issued this account to wait on (ADR 0367 §2): the name a session
|
||||
// asks for, and the subject it lands on. Only the tool runner's membership carries any.
|
||||
type WaitIssued struct {
|
||||
Event string `json:"event"`
|
||||
Subject string `json:"subject"`
|
||||
}
|
||||
|
||||
// Heard is one event a wait heard: the subject it arrived on, its body and when it arrived.
|
||||
type Heard struct {
|
||||
Subject string
|
||||
Body json.RawMessage
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// HearFirst subscribes the subjects for this call alone and answers the first message whose body match
|
||||
// accepts (every one, when match is nil), or ok false once `within` has passed. Every subscription is
|
||||
// closed before it returns. A subscription the bus refuses is said as an error naming the subject, at once;
|
||||
// a caller gone — ctx done — ends the wait at once with ctx's error, so nothing waits on for nobody.
|
||||
func (c *Conn) HearFirst(ctx context.Context, subjects []string, within time.Duration, match func(json.RawMessage) bool) (heard Heard, ok bool, err error) {
|
||||
if len(subjects) == 0 {
|
||||
return Heard{}, false, fmt.Errorf("a wait names at least one subject")
|
||||
}
|
||||
got := make(chan Heard, 1)
|
||||
refused := make(chan string, len(subjects))
|
||||
var subs []*nats.Subscription
|
||||
defer func() {
|
||||
for _, s := range subs {
|
||||
_ = s.Unsubscribe()
|
||||
}
|
||||
c.mu.Lock()
|
||||
for _, subject := range subjects {
|
||||
delete(c.waits[subject], refused)
|
||||
if len(c.waits[subject]) == 0 {
|
||||
delete(c.waits, subject)
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}()
|
||||
c.mu.Lock()
|
||||
if c.waits == nil {
|
||||
c.waits = map[string]map[chan string]bool{}
|
||||
}
|
||||
for _, subject := range subjects {
|
||||
if c.waits[subject] == nil {
|
||||
c.waits[subject] = map[chan string]bool{}
|
||||
}
|
||||
c.waits[subject][refused] = true
|
||||
}
|
||||
c.mu.Unlock()
|
||||
for _, subject := range subjects {
|
||||
sub, err := c.nc.Subscribe(subject, func(msg *nats.Msg) {
|
||||
body := json.RawMessage(msg.Data)
|
||||
if !json.Valid(body) {
|
||||
quoted, _ := json.Marshal(string(msg.Data))
|
||||
body = quoted
|
||||
}
|
||||
if match != nil && !match(body) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case got <- Heard{Subject: msg.Subject, Body: body, At: time.Now().UTC()}:
|
||||
default:
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return Heard{}, false, fmt.Errorf("could not wait on %s: %w", subject, err)
|
||||
}
|
||||
subs = append(subs, sub)
|
||||
}
|
||||
// The bus has every subscription — or has said which it refused — once this returns.
|
||||
if err := c.nc.FlushTimeout(5 * time.Second); err != nil {
|
||||
return Heard{}, false, fmt.Errorf("the bus did not take the wait: %w", err)
|
||||
}
|
||||
timer := time.NewTimer(within)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case h := <-got:
|
||||
return h, true, nil
|
||||
case subject := <-refused:
|
||||
return Heard{}, false, fmt.Errorf("the bus refused this account a subscription on %s: it is not in the "+
|
||||
"account's wait grant (novox/hq ADR 0367 §2)", subject)
|
||||
case <-timer.C:
|
||||
return Heard{}, false, nil
|
||||
case <-ctx.Done():
|
||||
return Heard{}, false, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// waitRefused tells every wait on a subject the bus refused that it was refused; true when one was waiting.
|
||||
func (c *Conn) waitRefused(subject string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
told := false
|
||||
for ch := range c.waits[subject] {
|
||||
select {
|
||||
case ch <- subject:
|
||||
default:
|
||||
}
|
||||
told = true
|
||||
}
|
||||
return told
|
||||
}
|
||||
|
||||
// Subscriptions is how many subscriptions this connection holds now: what a wait leaves behind is none.
|
||||
func (c *Conn) Subscriptions() int { return c.nc.NumSubscriptions() }
|
||||
|
||||
// subjectRefused is the subject a subscription refusal names, or "".
|
||||
func subjectRefused(err error) string {
|
||||
m := refusedSubject.FindStringSubmatch(err.Error())
|
||||
if len(m) < 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.Trim(m[1], `"`)
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package bus_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0367 §2, §5: the bus refuses the tool runner's account any subject its wait grant does not
|
||||
// name — another asker's `decided` among them — and a wait on one says the bus refused it, at once.
|
||||
func TestTheBusRefusesAWaitOutsideTheGrant(t *testing.T) {
|
||||
granted := "mesh.seat.operator-channel.event.decided.claude-code"
|
||||
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, NoLog: true, NoSigs: true,
|
||||
Users: []*server.User{{Username: "desk.node-tools", Password: "pw", Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: []string{"$JS.API.DIRECT.GET.>", "mesh.>"}},
|
||||
Subscribe: &server.SubjectPermission{Allow: []string{granted, "_INBOX.desk.node-tools.>", "mesh.assignment.desk.*"}},
|
||||
}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
c := dial(t, s.ClientURL(), "node-tools", "desk", "desk.node-tools")
|
||||
|
||||
start := time.Now()
|
||||
_, ok, err := c.HearFirst(context.Background(), []string{"mesh.seat.operator-channel.event.decided.shop"}, 5*time.Second, nil)
|
||||
if ok || err == nil || !strings.Contains(err.Error(), "refused") {
|
||||
t.Errorf("a wait on another asker's decided: heard %v, %v", ok, err)
|
||||
}
|
||||
if time.Since(start) > 3*time.Second {
|
||||
t.Errorf("the refusal took %s to be said", time.Since(start))
|
||||
}
|
||||
|
||||
done := make(chan bus.Heard, 1)
|
||||
go func() {
|
||||
h, ok, err := c.HearFirst(context.Background(), []string{granted}, 5*time.Second, func(body json.RawMessage) bool { return true })
|
||||
if ok && err == nil {
|
||||
done <- h
|
||||
}
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if err := c.Say(granted, []byte(`{"ask":"a1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case h := <-done:
|
||||
if h.Subject != granted || string(h.Body) != `{"ask":"a1"}` {
|
||||
t.Errorf("heard %+v", h)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a wait on the granted subject heard nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// A wait whose caller is gone ends at once, its subscription closed (the review of issue 745).
|
||||
func TestAWaitEndsWhenItsCallerIsGone(t *testing.T) {
|
||||
c := dial(t, mt.URL(t), "node-tools", "desk", "")
|
||||
before := c.Subscriptions()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
time.AfterFunc(200*time.Millisecond, cancel)
|
||||
start := time.Now()
|
||||
_, ok, err := c.HearFirst(ctx, []string{"mesh.mod.mesh-knowledge.event.reviewed"}, time.Minute, nil)
|
||||
if ok || !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("a cancelled wait answered %v, %v", ok, err)
|
||||
}
|
||||
if took := time.Since(start); took > 2*time.Second {
|
||||
t.Errorf("a cancelled wait ended after %s", took)
|
||||
}
|
||||
if after := c.Subscriptions(); after != before {
|
||||
t.Errorf("%d subscriptions before, %d after", before, after)
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,8 @@ package console
|
||||
|
||||
// The mesh's tools found by address, not announced whole (novox/hq ADR 0195, to-be 34 §3a).
|
||||
//
|
||||
// The console announces six tools — five to find and call, and mesh_runtimes to say which runtimes
|
||||
// discovery heard. Everything the mesh answers is reached through them by one address per layer:
|
||||
// The console announces seven tools — five to find and call, mesh_runtimes to say which runtimes
|
||||
// discovery heard, and mesh_wait to wait on the bus for an event (novox/hq ADR 0367). Everything the mesh answers is reached through them by one address per layer:
|
||||
//
|
||||
// <seat>.<verb> a seat held once for the mesh — its holder answers
|
||||
// <node>/<seat>.<verb> a seat held once per machine — that machine's holder answers
|
||||
@@ -17,6 +17,12 @@ package console
|
||||
// session — so a tool that arrived a minute ago is found without the client reconnecting.
|
||||
|
||||
import (
|
||||
assigner "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
meshcatalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
@@ -26,31 +32,35 @@ import (
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/firstview"
|
||||
)
|
||||
|
||||
// IndexKept is how long what the mesh answered is kept before it is asked again: long enough that
|
||||
// one agent turn's search, describe and call ask once, short enough that nothing goes stale.
|
||||
var IndexKept = 5 * time.Second
|
||||
|
||||
// The six tools the console announces. Names of the API's kind — letters, digits, `_`, `-`.
|
||||
// The seven tools the console announces. Names of the API's kind — letters, digits, `_`, `-`.
|
||||
const (
|
||||
verbOverview = "mesh_overview"
|
||||
verbMachine = "mesh_machine"
|
||||
verbSearch = "mesh_search"
|
||||
verbDescribe = "mesh_describe"
|
||||
verbCall = "mesh_call"
|
||||
verbRuntimes = "mesh_runtimes"
|
||||
verbOverview = "mesh_overview"
|
||||
verbMachine = "mesh_machine"
|
||||
verbSearch = "mesh_search"
|
||||
verbDescribe = "mesh_describe"
|
||||
verbCall = "mesh_call"
|
||||
verbRuntimes = "mesh_runtimes"
|
||||
verbFirstViews = "mesh_first_views"
|
||||
verbFirstAgain = "mesh_first_view_again"
|
||||
verbWait = "mesh_wait"
|
||||
)
|
||||
|
||||
// searchCap is how many matches a search answers before it says how many more there were.
|
||||
const searchCap = 25
|
||||
|
||||
const grammar = "Addresses: `<seat>.<verb>` for a seat held once for the mesh (e.g. `mesh-controller.nodes`); " +
|
||||
const grammar = "Addresses: `<seat>.<verb>` for a seat held once for the mesh (e.g. `mesh-module-assigner.assignments`); " +
|
||||
"`<node>/<seat>.<verb>` for a seat every machine holds (e.g. `ace/node-packet-filter.rules`); " +
|
||||
"`<node>/<module>.<tool>` for a module on one machine (e.g. `novox/postgres.postgres_list_databases`); " +
|
||||
"and `<module>.<tool>` also for a module whose instances are interchangeable."
|
||||
|
||||
// discovery is the six tools as tools/list announces them.
|
||||
// discovery is the seven tools as tools/list announces them.
|
||||
func discovery() []map[string]any {
|
||||
str := func(desc string) map[string]any { return map[string]any{"type": "string", "description": desc} }
|
||||
obj := func(props map[string]any, required ...string) map[string]any {
|
||||
@@ -84,12 +94,22 @@ func discovery() []map[string]any {
|
||||
"arrive, its size in bytes, how many modules and tools it announced, whether it was shortened to fit the " +
|
||||
"bus, and when it was last heard — and every runtime or machine expected that was not heard. Read-only; " +
|
||||
"for when an address is said to be missing or on another machine."},
|
||||
{"name": verbFirstViews, "inputSchema": obj(map[string]any{}),
|
||||
"description": "The tool runner's first views of the four states it reads of the indexer and the assigner (the " +
|
||||
"assigner's assignments and seat-holders, the indexer's manifests and modules): for each, the owner, the tool " +
|
||||
"asked, the revision held and when it was made, how many keys, when it was asked and answered, how often " +
|
||||
"it was asked and how often by hand, whether its condition is raised, and why it was last asked. Read-only."},
|
||||
{"name": verbFirstAgain, "inputSchema": obj(map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why it is asked again, kept in the log"},
|
||||
}), "description": "Ask the tool runner's four first views again, now, and say what each answers. A hand act, recorded in " +
|
||||
"the runner's log with its reason; for when a picture is doubted, or after an owner's state was made again."},
|
||||
waitTool(),
|
||||
}
|
||||
}
|
||||
|
||||
func isDiscovery(name string) bool {
|
||||
switch name {
|
||||
case verbOverview, verbMachine, verbSearch, verbDescribe, verbCall, verbRuntimes:
|
||||
case verbOverview, verbMachine, verbSearch, verbDescribe, verbCall, verbRuntimes, verbFirstViews, verbFirstAgain, verbWait:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -125,15 +145,55 @@ type index struct {
|
||||
NotAnswering []string
|
||||
Listing *Listing // the flat catalogue the call path resolves subjects with
|
||||
Discovery announce.Discovery
|
||||
// Recorded is where the controller's records place each module that declares tools.
|
||||
// Recorded is where the assigner places each module that declares tools.
|
||||
Recorded map[string][]string
|
||||
// Unheard is every runtime whose answer this index lacks: it said it was there and did not say
|
||||
// what it serves in time, or it answered before and not now. An address it might answer is never
|
||||
// called missing while it is here (2026-10-05).
|
||||
Unheard []unheard
|
||||
// Replaces is what each verb and tool replaces, from the controller's records (novox/hq ADR 0245):
|
||||
// Placed is the state seat-holders as the seat mesh-module-assigner's holder wrote it (novox/hq ADR 0571 clause 4), by its
|
||||
// keys `<seat>` and `<seat>.<node>`, read with assigner.HoldersOf; PlacedWhy says why it could not be read.
|
||||
Placed map[string][]byte
|
||||
PlacedWhy string
|
||||
// Replaces is what each verb and tool replaces (novox/hq ADR 0245):
|
||||
// `seat:<seat>.<verb>` and `<module>.<tool>`.
|
||||
Replaces map[string][]string
|
||||
// Earlier is every runtime not heard this round whose last answer, within RuntimeForgotten, this
|
||||
// index holds instead (novox/hq issue 554), by runtimeKey: when it was heard.
|
||||
Earlier map[string]time.Time
|
||||
// Stale is every module on a machine (`mod:<module>@<node>`) and every seat's holder
|
||||
// (`seat:<seat>@<node>`) this index knows only from an earlier answer: from whom, and when.
|
||||
Stale map[string]staleFrom
|
||||
}
|
||||
|
||||
// staleFrom is where an address known only from an earlier round of discovery was heard.
|
||||
type staleFrom struct {
|
||||
Runtime string
|
||||
Machine string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// say is the sentence a call or a description carries for it.
|
||||
func (f staleFrom) say() string {
|
||||
who := f.Runtime
|
||||
if f.Machine != "" {
|
||||
who += " on " + f.Machine
|
||||
}
|
||||
return fmt.Sprintf("found from what %s announced %s ago: it did not answer the latest round of discovery",
|
||||
who, time.Since(f.At).Round(time.Second))
|
||||
}
|
||||
|
||||
// earlier is a runtime's last answer, used for a round that did not hear it.
|
||||
type earlier struct {
|
||||
Heard announce.Heard
|
||||
At time.Time
|
||||
}
|
||||
|
||||
func endpointKey(e announce.Endpoint) string {
|
||||
if e.Kind == announce.KindSeat {
|
||||
return "seat:" + e.Seat + "@" + e.Node
|
||||
}
|
||||
return "mod:" + e.Module + "@" + e.Node
|
||||
}
|
||||
|
||||
// unheard is one runtime discovery did not hear in full, and why.
|
||||
@@ -185,42 +245,8 @@ func findTool(tools []Tool, name string) *Tool {
|
||||
return nil
|
||||
}
|
||||
|
||||
// controllerOutput is a controller seat verb's answer: the command's printed output.
|
||||
func controllerOutput(conn *bus.Conn, verb string) (string, error) {
|
||||
got, err := conn.Ask("seat:mesh-controller."+verb, map[string]any{}, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var r struct {
|
||||
Output string `json:"output"`
|
||||
OK *bool `json:"ok"`
|
||||
}
|
||||
if json.Unmarshal(got.Result, &r) != nil {
|
||||
return "", fmt.Errorf("mesh-controller.%s answered something that is not its output", verb)
|
||||
}
|
||||
if r.OK != nil && !*r.OK {
|
||||
return "", fmt.Errorf("mesh-controller.%s: %s", verb, strings.TrimSpace(r.Output))
|
||||
}
|
||||
return r.Output, nil
|
||||
}
|
||||
|
||||
// jsonIn is the JSON document a command printed, after any lines it said first: a seat verb runs
|
||||
// the controller's command, and a command may warn before it answers.
|
||||
func jsonIn(output string) string {
|
||||
t := strings.TrimSpace(output)
|
||||
if strings.HasPrefix(t, "{") || strings.HasPrefix(t, "[") {
|
||||
return t
|
||||
}
|
||||
for _, open := range []string{"\n{", "\n["} {
|
||||
if i := strings.Index(output, open); i >= 0 {
|
||||
return strings.TrimSpace(output[i+1:])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// recordedModule is a module as the controller's records hold it (`module list --json`): where the
|
||||
// mesh assigned it, and whether it declares tools — what should announce itself, and where.
|
||||
// recordedModule is a module as the bus holds it: where the assigner seat places it, and whether its manifest declares
|
||||
// tools — what should announce itself, and where (novox/hq ADR 0571).
|
||||
type recordedModule struct {
|
||||
Module string `json:"module"`
|
||||
On []string `json:"on"`
|
||||
@@ -229,42 +255,68 @@ type recordedModule struct {
|
||||
Replaces map[string][]string `json:"replaces,omitempty"`
|
||||
}
|
||||
|
||||
// recordedMachine is a machine as the controller's records hold it (`node list --json`).
|
||||
type recordedMachine struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// indexOn asks the mesh what it holds (novox/hq ADR 0197): what answers, from every runtime's own
|
||||
// announcement on the bus — one `$SRV.INFO` request — and what should, from the controller's records
|
||||
// announcement on the bus — one `$SRV.INFO` request — and what should, from the indexer's records
|
||||
// read as JSON. Nothing is inferred from a roster and nothing is parsed from print.
|
||||
func indexOn(conn *bus.Conn) (*index, error) {
|
||||
//
|
||||
// `held` is what the console holds already, so an instance whose announcement is unchanged is not
|
||||
// asked for it again; `before` is the last answer of every runtime this round did not hear, which the
|
||||
// index holds instead, marked with its age (novox/hq issue 554).
|
||||
func indexOn(conn *bus.Conn, src Source, held announce.Known, before func(announce.Discovery) []earlier) (*index, error) {
|
||||
var wg sync.WaitGroup
|
||||
var nodesOut, modulesOut string
|
||||
var toolsAnswer json.RawMessage
|
||||
wg.Add(3)
|
||||
go func() { defer wg.Done(); nodesOut, _ = controllerOutput(conn, "nodes") }()
|
||||
go func() { defer wg.Done(); modulesOut, _ = controllerOutput(conn, "modules") }()
|
||||
// The seats' verbs as the records define them, for what each replaces (novox/hq ADR 0245): answered
|
||||
// in-process from the records, never a command.
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if got, err := conn.Ask("seat:mesh-controller.tools", map[string]any{}, ""); err == nil {
|
||||
toolsAnswer = got.Result
|
||||
}
|
||||
}()
|
||||
d, err := announce.Gather(conn)
|
||||
var placed map[string][]byte
|
||||
var placedErr error
|
||||
var recorded []recordedModule
|
||||
var known []string
|
||||
var recordsWhy []string
|
||||
wg.Add(2)
|
||||
// Which modules there are and where they are placed, and which machines the mesh has, from the bus (ADR 0571
|
||||
// clauses 4 and 5): the indexer's modules and manifests and the assigner seat's assignments, never any other record.
|
||||
go func() { defer wg.Done(); recorded, known, recordsWhy = recordsOn(src) }()
|
||||
// Who holds which seat on which node, from the assigner (ADR 0571 clause 4): no other call.
|
||||
go func() { defer wg.Done(); placed, placedErr = seatHoldersOn(src) }()
|
||||
d, err := announce.Gather(conn, held)
|
||||
wg.Wait()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var endpoints []announce.Endpoint
|
||||
fresh := map[string]bool{}
|
||||
for _, h := range d.Heard {
|
||||
endpoints = append(endpoints, announce.Endpoints(h.Info)...)
|
||||
for _, e := range announce.Endpoints(h.Info) {
|
||||
fresh[endpointKey(e)] = true
|
||||
endpoints = append(endpoints, e)
|
||||
}
|
||||
}
|
||||
// **A runtime not heard this round is not gone** (novox/hq issue 554): a round in which no answer
|
||||
// crossed the laptop's link in time replaced a full index with an empty one, and every seat and
|
||||
// module call was refused. What it announced last, within RuntimeForgotten, stands — said with its age.
|
||||
stale := map[string]staleFrom{}
|
||||
earlierAt := map[string]time.Time{}
|
||||
if before != nil {
|
||||
for _, b := range before(d) {
|
||||
earlierAt[runtimeKey(b.Heard.Info.Name, b.Heard.Info.ID)] = b.At
|
||||
for _, e := range announce.Endpoints(b.Heard.Info) {
|
||||
k := endpointKey(e)
|
||||
if fresh[k] {
|
||||
continue
|
||||
}
|
||||
if _, have := stale[k]; !have {
|
||||
stale[k] = staleFrom{Runtime: b.Heard.Info.Name, Machine: b.Heard.Machine(), At: b.At}
|
||||
}
|
||||
endpoints = append(endpoints, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
x, announced, machines := indexOfEndpoints(endpoints)
|
||||
l := x.Listing
|
||||
x.Discovery = d
|
||||
x.Stale, x.Earlier = stale, earlierAt
|
||||
x.Placed = placed
|
||||
if placedErr != nil {
|
||||
x.PlacedWhy = placedErr.Error()
|
||||
}
|
||||
for _, s := range d.Silent {
|
||||
x.Unheard = append(x.Unheard, unheard{Runtime: s.Name, Machine: s.Machine,
|
||||
Why: fmt.Sprintf("it answered PING; what it serves did not arrive within %s", announce.Patience)})
|
||||
@@ -272,25 +324,24 @@ func indexOn(conn *bus.Conn) (*index, error) {
|
||||
|
||||
// What should have answered: every assignment of a module that declares tools. Silence is named;
|
||||
// a module with no tools is never a name here.
|
||||
var recorded []recordedModule
|
||||
recordedOK := json.Unmarshal([]byte(jsonIn(modulesOut)), &recorded) == nil
|
||||
x.Replaces = replacesOf(toolsAnswer, recorded)
|
||||
if recordedOK {
|
||||
for _, m := range recorded {
|
||||
if !m.Tools {
|
||||
continue
|
||||
}
|
||||
x.Recorded[m.Module] = append([]string{}, m.On...)
|
||||
for _, n := range m.On {
|
||||
// A holder of a seat held once for the mesh announces no machine — the seat is the mesh's,
|
||||
// not a machine's — so what it announced without one answers for wherever it is assigned.
|
||||
if !announced[m.Module][n] && !announced[m.Module][""] {
|
||||
l.NotAnswering = append(l.NotAnswering, m.Module+" on "+n)
|
||||
}
|
||||
x.Replaces = replacesOf(catalogue.Seats(), recorded)
|
||||
for _, m := range recorded {
|
||||
if !m.Tools {
|
||||
continue
|
||||
}
|
||||
x.Recorded[m.Module] = append([]string{}, m.On...)
|
||||
for _, n := range m.On {
|
||||
// A holder of a seat held once for the mesh announces no machine — the seat is the mesh's,
|
||||
// not a machine's — so what it announced without one answers for wherever it is assigned.
|
||||
if !announced[m.Module][n] && !announced[m.Module][""] {
|
||||
l.NotAnswering = append(l.NotAnswering, m.Module+" on "+n)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
l.NotAnswering = append(l.NotAnswering, "mesh-controller (its records of the modules did not answer, so what is missing cannot be said)")
|
||||
}
|
||||
l.NotAnswering = append(l.NotAnswering, recordsWhy...)
|
||||
if x.PlacedWhy != "" {
|
||||
l.NotAnswering = append(l.NotAnswering, firstview.SeatHoldersRead+" (who holds which seat by assignment could not be read: "+
|
||||
x.PlacedWhy+")")
|
||||
}
|
||||
for _, u := range x.Unheard {
|
||||
l.NotAnswering = append(l.NotAnswering, "the runtime "+sayUnheard([]unheard{u}))
|
||||
@@ -298,13 +349,8 @@ func indexOn(conn *bus.Conn) (*index, error) {
|
||||
sort.Strings(l.NotAnswering)
|
||||
x.NotAnswering = l.NotAnswering
|
||||
|
||||
var known []recordedMachine
|
||||
if json.Unmarshal([]byte(jsonIn(nodesOut)), &known) == nil {
|
||||
for _, n := range known {
|
||||
if n.Name != "" {
|
||||
machines[n.Name] = true
|
||||
}
|
||||
}
|
||||
for _, n := range known {
|
||||
machines[n] = true
|
||||
}
|
||||
for n := range machines {
|
||||
x.Machines = append(x.Machines, n)
|
||||
@@ -313,6 +359,142 @@ func indexOn(conn *bus.Conn) (*index, error) {
|
||||
return x, nil
|
||||
}
|
||||
|
||||
// Source is where the console reads the four states of the indexer and the assigner it indexes the mesh by: the assigner's
|
||||
// assignments and seat-holders, the indexer's modules and manifests, by `<owner>.<state>` (firstview.AssignmentsRead and the
|
||||
// rest). The tool runner's is its first views (firstview.Views), never the owners' state (novox/hq ADR 0615 §1).
|
||||
type Source interface {
|
||||
// State is every current value of a state by key; an error says why it cannot be told yet.
|
||||
State(name string) (map[string][]byte, error)
|
||||
// Entry is one value of a state, nil when it holds none.
|
||||
Entry(name, key string) ([]byte, error)
|
||||
}
|
||||
|
||||
// seatHoldersOn reads the state seat-holders whole (novox/hq ADR 0571 clause 4), as firstview.SeatHoldersRead.
|
||||
func seatHoldersOn(src Source) (map[string][]byte, error) {
|
||||
return src.State(firstview.SeatHoldersRead)
|
||||
}
|
||||
|
||||
// The indexer's states the console reads (novox/hq ADR 0571 clauses 4 and 5): every module, and each manifest by its
|
||||
// digest, by the names a reader's manifest declares.
|
||||
var (
|
||||
modulesRead = firstview.ModulesRead
|
||||
manifestsRead = firstview.ManifestsRead
|
||||
)
|
||||
|
||||
// manifestFacts is what the console takes of a manifest: the tools it declares and what each replaces. A manifest's
|
||||
// digest never names another, so what is read once is kept.
|
||||
type manifestFacts struct {
|
||||
Tools []string `json:"tools"`
|
||||
Replaces map[string][]string `json:"replaces"`
|
||||
}
|
||||
|
||||
var manifestsKept sync.Map // digest → manifestFacts
|
||||
|
||||
// recordsOn is every module the indexer holds, with where the assigner seat places it and whether it declares tools,
|
||||
// and every machine the assignments name (moduleNames, assigner.MachineNames). What cannot be read is said, one line
|
||||
// per state, for the listing's not answering.
|
||||
func recordsOn(src Source) (recorded []recordedModule, machines []string, why []string) {
|
||||
held, err := src.State(firstview.AssignmentsRead)
|
||||
if err != nil {
|
||||
why = append(why, firstview.AssignmentsRead+" (where modules are placed could not be read, so what is missing cannot "+
|
||||
"be said: "+err.Error()+")")
|
||||
}
|
||||
machines = assigner.MachineNames(held)
|
||||
on := map[string][]string{}
|
||||
// The assignments' keys in either form, a node key and its module keys or the form before (research 072 row 37).
|
||||
assembled, err := assigner.AssignmentsOf(held)
|
||||
if err != nil {
|
||||
why = append(why, firstview.AssignmentsRead+" (not every key is an assignment: "+err.Error()+")")
|
||||
}
|
||||
for _, node := range machines {
|
||||
for _, m := range assembled[node].Modules {
|
||||
on[m.Module] = append(on[m.Module], node)
|
||||
}
|
||||
}
|
||||
modules, err := src.State(modulesRead)
|
||||
if err != nil {
|
||||
return nil, machines, append(why, modulesRead+" (which modules there are could not be read: "+err.Error()+")")
|
||||
}
|
||||
names, err := moduleNames(modules)
|
||||
if err != nil {
|
||||
return nil, machines, append(why, modulesRead+" ("+err.Error()+")")
|
||||
}
|
||||
for _, name := range names {
|
||||
r := recordedModule{Module: name, On: on[name]}
|
||||
if m, err := meshcatalogclient.DecodeModule(modules[name]); err == nil && m.Manifest != "" {
|
||||
facts, err := manifestOf(src, m.Manifest)
|
||||
if err != nil {
|
||||
why = append(why, name+" (its manifest could not be read, so whether it answers tools is not known: "+err.Error()+")")
|
||||
} else {
|
||||
r.Tools, r.Replaces = len(facts.Tools) > 0, facts.Replaces
|
||||
}
|
||||
}
|
||||
recorded = append(recorded, r)
|
||||
}
|
||||
return recorded, machines, why
|
||||
}
|
||||
|
||||
// moduleNames is every module the indexer holds, sorted, from its state `modules` as a reader holds its keys (key to
|
||||
// value; a deleted key absent). A key that is not a module's is refused, naming it.
|
||||
func moduleNames(keys map[string][]byte) ([]string, error) {
|
||||
out := make([]string, 0, len(keys))
|
||||
for k := range keys {
|
||||
module, err := meshcatalogclient.ParseModulesKey(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, module)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// manifestOf is what the console takes of the manifest of a digest, read from the indexer's manifests once.
|
||||
func manifestOf(src Source, digest string) (manifestFacts, error) {
|
||||
if kept, ok := manifestsKept.Load(digest); ok {
|
||||
return kept.(manifestFacts), nil
|
||||
}
|
||||
entry, err := src.Entry(manifestsRead, digest)
|
||||
if err != nil {
|
||||
return manifestFacts{}, err
|
||||
}
|
||||
if entry == nil {
|
||||
return manifestFacts{}, fmt.Errorf("the indexer holds no manifest %s", digest)
|
||||
}
|
||||
m, err := meshcatalogclient.DecodeManifest(entry)
|
||||
if err != nil {
|
||||
return manifestFacts{}, err
|
||||
}
|
||||
var facts manifestFacts
|
||||
if err := json.Unmarshal([]byte(m.Manifest), &facts); err != nil {
|
||||
return manifestFacts{}, err
|
||||
}
|
||||
manifestsKept.Store(digest, facts)
|
||||
return facts, nil
|
||||
}
|
||||
|
||||
// placedHolders is who holds seat (on node, for a seat held per machine) by assignment, or none.
|
||||
func (x *index) placedHolders(seat, node string) []meshmoduleassigner.SeatHoldersHoldersItem {
|
||||
if len(x.Placed) == 0 {
|
||||
return nil
|
||||
}
|
||||
hs, err := assigner.HoldersOf(x.Placed, seat, node)
|
||||
if err != nil || len(hs) > 0 || node != "" {
|
||||
return hs
|
||||
}
|
||||
// A seat held per machine, asked without one: every node's key of it.
|
||||
var all []meshmoduleassigner.SeatHoldersHoldersItem
|
||||
for key := range x.Placed {
|
||||
if s, n, ok := strings.Cut(key, "."); ok && s == seat {
|
||||
if more, err := assigner.HoldersOf(x.Placed, seat, n); err == nil {
|
||||
all = append(all, more...)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool { return all[i].Node+"/"+all[i].Module < all[j].Node+"/"+all[j].Module })
|
||||
return all
|
||||
}
|
||||
|
||||
func containsHolder(hs []holder, h holder) bool {
|
||||
for _, x := range hs {
|
||||
if x == h {
|
||||
@@ -325,6 +507,10 @@ func containsHolder(hs []holder, h holder) bool {
|
||||
func (s *Surface) index() (*index, error) { return s.indexAsked(false) }
|
||||
|
||||
// indexAsked is what the mesh answered, asked again when `fresh` or when the kept answer is old.
|
||||
//
|
||||
// **A round that heard nothing is not kept** (novox/hq issue 554): the index it gives holds what every
|
||||
// runtime announced last, marked with its age, and the next question asks again rather than living
|
||||
// with it for IndexKept.
|
||||
func (s *Surface) indexAsked(fresh bool) (*index, error) {
|
||||
s.mu.Lock()
|
||||
if !fresh && s.idx != nil && time.Since(s.idxAt) <= IndexKept {
|
||||
@@ -333,13 +519,17 @@ func (s *Surface) indexAsked(fresh bool) (*index, error) {
|
||||
return x, nil
|
||||
}
|
||||
s.mu.Unlock()
|
||||
x, err := indexOn(s.conn)
|
||||
x, err := indexOn(s.conn, s.src, s.held, s.before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.remember(x)
|
||||
s.idx, s.idxAt = x, time.Now()
|
||||
s.sayRound(x)
|
||||
s.idx = x
|
||||
if !x.Discovery.Empty() {
|
||||
s.idxAt = time.Now()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return x, nil
|
||||
}
|
||||
@@ -450,6 +640,26 @@ type target struct {
|
||||
Node string
|
||||
Tool Tool
|
||||
Seat bool
|
||||
// Stale is set when the target is known only from an earlier round of discovery (novox/hq issue 554).
|
||||
Stale *staleFrom
|
||||
}
|
||||
|
||||
// staleOf is where a target was heard, when every place it answers is known only from an earlier round.
|
||||
func (x *index) staleOf(keys []string) *staleFrom {
|
||||
if len(keys) == 0 || len(x.Stale) == 0 {
|
||||
return nil
|
||||
}
|
||||
var first *staleFrom
|
||||
for _, k := range keys {
|
||||
f, ok := x.Stale[k]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if first == nil {
|
||||
first = &f
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// resolve turns an address into exactly one target, or says why it cannot.
|
||||
@@ -487,12 +697,17 @@ func resolve(x *index, address string) (target, error) {
|
||||
return target{}, fmt.Errorf("%s is held once per machine: write <node>/%s — it is held on %s",
|
||||
prefix, rest, orNobody(nodesOf(s.Holders)))
|
||||
}
|
||||
return target{Address: node + "/" + rest, Key: "seat:" + rest + "@" + node, Name: rest, Node: node, Tool: *verb, Seat: true}, nil
|
||||
return target{Address: node + "/" + rest, Key: "seat:" + rest + "@" + node, Name: rest, Node: node, Tool: *verb, Seat: true,
|
||||
Stale: x.staleOf([]string{"seat:" + prefix + "@" + node})}, nil
|
||||
}
|
||||
if node != "" {
|
||||
return target{}, fmt.Errorf("%s is held once for the whole mesh: write %s, without a machine", prefix, rest)
|
||||
}
|
||||
return target{Address: rest, Key: "seat:" + rest, Name: rest, Tool: *verb, Seat: true}, nil
|
||||
var holders []string
|
||||
for _, h := range s.Holders {
|
||||
holders = append(holders, "seat:"+prefix+"@"+h.Node)
|
||||
}
|
||||
return target{Address: rest, Key: "seat:" + rest, Name: rest, Tool: *verb, Seat: true, Stale: x.staleOf(holders)}, nil
|
||||
}
|
||||
|
||||
m := x.Modules[prefix]
|
||||
@@ -505,10 +720,23 @@ func resolve(x *index, address string) (target, error) {
|
||||
return target{}, fmt.Errorf("%s is assigned to %s in the mesh's records, but nothing that answered discovery serves it: %s. "+
|
||||
"Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, strings.Join(on, ", "), why)
|
||||
}
|
||||
if hs := x.placedHolders(prefix, node); len(hs) > 0 {
|
||||
var held []string
|
||||
for _, h := range hs {
|
||||
held = append(held, h.Module+" on "+h.Node)
|
||||
}
|
||||
return target{}, fmt.Errorf("the seat %s is held by %s by assignment, but no holder of it announced itself on the bus. "+
|
||||
"Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, strings.Join(held, ", "))
|
||||
}
|
||||
if len(x.Unheard) > 0 {
|
||||
return target{}, fmt.Errorf("nothing that answered discovery is called %s, but not every runtime answered: %s — "+
|
||||
"%s may be theirs. Ask again in a moment; mesh_runtimes shows which runtimes answered", prefix, sayUnheard(x.Unheard), prefix)
|
||||
}
|
||||
if x.Discovery.Empty() {
|
||||
return target{}, fmt.Errorf("nothing answered discovery at all within %s, not even PING, and this console holds no "+
|
||||
"earlier answer: whether anything is called %s is not known. Ask again in a moment; mesh_runtimes shows which "+
|
||||
"runtimes answered", x.Discovery.Waited.Round(time.Millisecond), prefix)
|
||||
}
|
||||
return target{}, fmt.Errorf("nothing in the mesh is called %s: no seat, and no module that answers tools. "+
|
||||
"mesh_search finds a tool by words", prefix)
|
||||
}
|
||||
@@ -521,7 +749,11 @@ func resolve(x *index, address string) (target, error) {
|
||||
return target{}, fmt.Errorf("%s keeps state on each machine it runs on, so a call names the machine: "+
|
||||
"write <node>/%s — it runs on %s", prefix, rest, orNobody(m.On))
|
||||
}
|
||||
return target{Address: rest, Key: rest, Name: rest, Tool: *tool}, nil
|
||||
var on []string
|
||||
for _, n := range m.On {
|
||||
on = append(on, "mod:"+prefix+"@"+n)
|
||||
}
|
||||
return target{Address: rest, Key: rest, Name: rest, Tool: *tool, Stale: x.staleOf(on)}, nil
|
||||
}
|
||||
if len(m.On) > 0 && !contains(m.On, node) {
|
||||
if us := x.unheardOn(node); len(us) > 0 || contains(x.Recorded[prefix], node) {
|
||||
@@ -534,7 +766,8 @@ func resolve(x *index, address string) (target, error) {
|
||||
}
|
||||
return target{}, fmt.Errorf("%s does not run on %s; it runs on %s", prefix, node, orNobody(m.On))
|
||||
}
|
||||
return target{Address: node + "/" + rest, Key: rest + "@" + node, Name: rest, Node: node, Tool: *tool}, nil
|
||||
return target{Address: node + "/" + rest, Key: rest + "@" + node, Name: rest, Node: node, Tool: *tool,
|
||||
Stale: x.staleOf([]string{"mod:" + prefix + "@" + node})}, nil
|
||||
}
|
||||
|
||||
// unheardOnAny is what is unheard on any of these machines.
|
||||
@@ -600,7 +833,7 @@ func firstLine(s string) string {
|
||||
// the machine, `node` is taken out: the machine is in the address. Where it does not — a seat held
|
||||
// once for the mesh, or an interchangeable module — `node` is the tool's own argument, and is kept.
|
||||
//
|
||||
// **Taking it out everywhere made the mesh's own verbs lie** (novox/hq issue 244): the controller's
|
||||
// **Taking it out everywhere made the mesh's own verbs lie** (novox/hq issue 244): the old controller verbs
|
||||
// `push`, `plan`, `assign`, `pin` and `settings` take the machine they act on as `node`, and were
|
||||
// described without it — `push` as taking nothing at all, so a push naming one machine arrived as a
|
||||
// push of every machine behind.
|
||||
@@ -707,10 +940,16 @@ func failure(text string) map[string]any {
|
||||
}
|
||||
|
||||
// discover answers one of the discovery verbs.
|
||||
func (s *Surface) discover(name string, args map[string]any) map[string]any {
|
||||
func (s *Surface) discover(ctx context.Context, name string, args map[string]any) map[string]any {
|
||||
if name == verbRuntimes {
|
||||
return s.runtimesAnswer()
|
||||
}
|
||||
if name == verbFirstViews || name == verbFirstAgain {
|
||||
return s.firstViewsAnswer(ctx, name, args)
|
||||
}
|
||||
if name == verbWait {
|
||||
return s.waitAnswer(ctx, args)
|
||||
}
|
||||
x, err := s.index()
|
||||
if err != nil {
|
||||
return failure("the mesh's discovery failed: " + err.Error())
|
||||
@@ -861,6 +1100,9 @@ func (s *Surface) discover(name string, args map[string]any) map[string]any {
|
||||
if r := x.Replaces[key]; len(r) > 0 {
|
||||
described["instead_of"] = r
|
||||
}
|
||||
if t.Stale != nil {
|
||||
described["found"] = t.Stale.say()
|
||||
}
|
||||
return answerText(described)
|
||||
|
||||
case verbCall:
|
||||
@@ -880,12 +1122,19 @@ func (s *Surface) discover(name string, args map[string]any) map[string]any {
|
||||
got, err = callTool(s.conn, t.Key, callArgs, nil, x.Listing)
|
||||
}
|
||||
if err != nil {
|
||||
return failure(whyItFailed(t.Address, err))
|
||||
why := whyItFailed(t.Address, err)
|
||||
if t.Stale != nil {
|
||||
why += " (" + t.Stale.say() + ")"
|
||||
}
|
||||
return failure(why)
|
||||
}
|
||||
content := []map[string]any{{"type": "text", "text": pretty(got.Result)}}
|
||||
if got.Node != "" {
|
||||
content = append(content, map[string]any{"type": "text", "text": "answered by " + got.Node})
|
||||
}
|
||||
if t.Stale != nil {
|
||||
content = append(content, map[string]any{"type": "text", "text": t.Stale.say()})
|
||||
}
|
||||
return map[string]any{"content": content}
|
||||
}
|
||||
return failure("no discovery verb " + name)
|
||||
|
||||
@@ -36,9 +36,10 @@ func call(t *testing.T, endpoint, tool string, args map[string]any) (string, boo
|
||||
return text(t, post(t, endpoint, string(body)))
|
||||
}
|
||||
|
||||
// novox/hq ADR 0195: the console announces six tools, and everything the mesh answers is reached
|
||||
// novox/hq ADR 0195 and ADR 0367: the console announces seven tools, and everything the mesh answers is reached
|
||||
// through them by one address per layer.
|
||||
func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
was := IndexKept
|
||||
IndexKept = 0 // every discovery asks the mesh, so a module arriving mid-test is found
|
||||
t.Cleanup(func() { IndexKept = was })
|
||||
@@ -69,35 +70,34 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
|
||||
// The controller: its records as JSON, as its seat verbs answer them, and its own seat's verbs
|
||||
// announced on the bus like every runtime's.
|
||||
controller := connect(t, "mesh-controller", "bench")
|
||||
out := func(s string) map[string]any { return map[string]any{"output": s, "ok": true} }
|
||||
serve := func(verb string, answer func() any) {
|
||||
stop, err := controller.HandleSubject("mesh.seat.mesh-controller.tool."+verb, func(json.RawMessage) (any, error) {
|
||||
return answer(), nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
// The controller: its own seat's verbs announced on the bus like every runtime's; its records are asked by no one.
|
||||
controller := connect(t, "mesh-module-assigner", "bench")
|
||||
// Its verb nodes, called through the console as any seat's verb is.
|
||||
stopNodes, err := controller.HandleSubject("mesh.seat.mesh-module-assigner.tool.nodes", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"ok": true, "output": `[{"name":"bench","mode":"converged"},{"name":"desk","mode":"converged"}]`}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serve("nodes", func() any {
|
||||
return out("the bus's user list leaves out 2 user(s)\n" +
|
||||
`[{"name":"bench","heard":"3m ago","mode":"converged","id":"1f2e"},{"name":"desk","heard":"here","mode":"converged","id":"9a8b"}]`)
|
||||
t.Cleanup(stopNodes)
|
||||
// What the mesh holds, as the indexer and the assigner write it (ADR 0571 clauses 4, 5): never the controller's records.
|
||||
tools := map[string][]string{"alpha": {"one", "two"}, "beta": {"three"}, "gamma": {"given"}, "delta": nil,
|
||||
"epsilon": {"seven"}, "mesh-module-assigner": {"nodes"}}
|
||||
mesh.Placed(t, tools, map[string][]string{"desk": {"alpha", "beta", "delta"}, "bench": {"epsilon", "mesh-module-assigner"}})
|
||||
console := mt.MembershipOf("node-tools", "desk", false, nil)
|
||||
console.State = mt.ConsoleReads()
|
||||
mesh.Issue(t, console)
|
||||
mt.Until(t, func() error {
|
||||
if m := nodeTools.Membership("node-tools"); m == nil || len(m.State) == 0 {
|
||||
return fmt.Errorf("the console's reads are not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
var gammaOn atomic.Value
|
||||
gammaOn.Store("[]")
|
||||
serve("modules", func() any {
|
||||
return out(fmt.Sprintf(`[{"module":"alpha","on":["desk"],"tools":true},{"module":"beta","on":["desk"],"tools":true},`+
|
||||
`{"module":"gamma","on":%s,"tools":true},{"module":"delta","on":["desk"],"tools":false},`+
|
||||
`{"module":"epsilon","on":["bench"],"tools":true},{"module":"mesh-controller","on":["bench"],"tools":true}]`, gammaOn.Load().(string)))
|
||||
})
|
||||
stopAnn, err := announce.Serve(controller, announce.Service{Name: "mesh-controller", ID: "bench"}, func() []announce.Endpoint {
|
||||
return []announce.Endpoint{{Kind: announce.KindSeat, Module: "mesh-controller", Seat: "mesh-controller", Scope: "mesh",
|
||||
stopAnn, err := announce.Serve(controller, announce.Service{Name: "mesh-module-assigner", ID: "bench"}, func() []announce.Endpoint {
|
||||
return []announce.Endpoint{{Kind: announce.KindSeat, Module: "mesh-module-assigner", Seat: "mesh-module-assigner", Scope: "mesh",
|
||||
// As the live controller announces: a mesh seat's holder names no machine.
|
||||
Tool: "nodes", Node: "", Description: "Every machine the mesh knows.", Schema: json.RawMessage(`{}`),
|
||||
Subject: "mesh.seat.mesh-controller.tool.nodes"}}
|
||||
Subject: "mesh.seat.mesh-module-assigner.tool.nodes"}}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -106,14 +106,14 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
controller.Flush()
|
||||
watcher.Flush()
|
||||
|
||||
up, err := Serve(NewSurface(nodeTools, "desk.node-tools"), "127.0.0.1:0")
|
||||
up, err := Serve(NewSurface(nodeTools, busSource{nodeTools}, "desk.node-tools"), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer up.Close()
|
||||
endpoint := "http://" + up.Address + "/mcp"
|
||||
|
||||
// Six tools, nothing else.
|
||||
// Seven tools, nothing else (ADR 0367).
|
||||
listed := post(t, endpoint, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`)["result"].(map[string]any)
|
||||
var names []string
|
||||
for _, x := range listed["tools"].([]any) {
|
||||
@@ -125,19 +125,19 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if got := strings.Join(names, ","); got != "mesh_overview,mesh_machine,mesh_search,mesh_describe,mesh_call,mesh_runtimes" {
|
||||
if got := strings.Join(names, ","); got != "mesh_overview,mesh_machine,mesh_search,mesh_describe,mesh_call,mesh_runtimes,mesh_first_views,mesh_first_view_again,mesh_wait" {
|
||||
t.Errorf("announced %s", got)
|
||||
}
|
||||
|
||||
// The overview names the mesh's seats, the machines' seats and the machines.
|
||||
overview, isErr := call(t, endpoint, "mesh_overview", nil)
|
||||
if isErr || !strings.Contains(overview, "mesh-controller.nodes") || !strings.Contains(overview, "<node>/node-shelf.list") ||
|
||||
if isErr || !strings.Contains(overview, "mesh-module-assigner.nodes") || !strings.Contains(overview, "<node>/node-shelf.list") ||
|
||||
!strings.Contains(overview, `"bench"`) || !strings.Contains(overview, `"desk"`) {
|
||||
t.Errorf("overview: %s", overview)
|
||||
}
|
||||
// Silence is named only where tools should have answered: epsilon declares tools on bench and
|
||||
// nothing there announced it; delta declares none and is never a name.
|
||||
if strings.Contains(overview, "mesh-controller on bench") {
|
||||
if strings.Contains(overview, "mesh-module-assigner on bench") {
|
||||
t.Errorf("a mesh seat's holder that announced no machine is called silent on its machine:\n%s", overview)
|
||||
}
|
||||
if !strings.Contains(overview, "epsilon on bench") || strings.Contains(overview, "delta") {
|
||||
@@ -150,7 +150,7 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
}
|
||||
|
||||
// A mesh seat, a node seat, an assignment and an interchangeable module, each by address.
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "mesh-controller.nodes"}); isErr || !strings.Contains(got, "converged") {
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "mesh-module-assigner.nodes"}); isErr || !strings.Contains(got, "converged") {
|
||||
t.Errorf("mesh seat: %s", got)
|
||||
}
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "desk/node-shelf.list"}); isErr || !strings.Contains(got, `"a"`) {
|
||||
@@ -166,11 +166,11 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
|
||||
// Refused, by name, where the address does not say enough or says the wrong thing.
|
||||
for address, want := range map[string]string{
|
||||
"beta.three": "keeps state on each machine it runs on, so a call names the machine: write <node>/beta.three — it runs on desk",
|
||||
"node-shelf.list": "held once per machine: write <node>/node-shelf.list — it is held on desk",
|
||||
"desk/mesh-controller.nodes": "held once for the whole mesh",
|
||||
"bench/beta.three": "beta does not run on bench; it runs on desk",
|
||||
"nonsense": "is not an address",
|
||||
"beta.three": "keeps state on each machine it runs on, so a call names the machine: write <node>/beta.three — it runs on desk",
|
||||
"node-shelf.list": "held once per machine: write <node>/node-shelf.list — it is held on desk",
|
||||
"desk/mesh-module-assigner.nodes": "held once for the whole mesh",
|
||||
"bench/beta.three": "beta does not run on bench; it runs on desk",
|
||||
"nonsense": "is not an address",
|
||||
} {
|
||||
got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": address})
|
||||
if !isErr || !strings.Contains(got, want) {
|
||||
@@ -201,7 +201,7 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
t.Fatalf("gamma was found before it served: %s", got)
|
||||
}
|
||||
mesh.Issue(t, mt.MembershipOf("gamma", "desk", false, nil))
|
||||
gammaOn.Store(`["desk"]`)
|
||||
mesh.Placed(t, tools, map[string][]string{"desk": {"alpha", "beta", "delta", "gamma"}, "bench": {"epsilon", "mesh-module-assigner"}})
|
||||
late := connect(t, "node-tools", "desk")
|
||||
stopLate, err := runtime.Run(late, []runtime.Served{{Module: "gamma", Entrypoints: []string{mt.Fixture("env-gamma.serve.mjs")}}},
|
||||
nil, (&mt.Logs{}).Logf)
|
||||
@@ -232,12 +232,3 @@ func TestTheMeshsToolsAreFoundByAddress(t *testing.T) {
|
||||
t.Errorf("an old name: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControllersRecordsAreReadAsJSON(t *testing.T) {
|
||||
if got := jsonIn("a warning printed first\n[{\"name\":\"ace\"}]"); got != `[{"name":"ace"}]` {
|
||||
t.Errorf("an array after a warning: %q", got)
|
||||
}
|
||||
if got := jsonIn(`{"seats":[]}`); got != `{"seats":[]}` {
|
||||
t.Errorf("an object: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
@@ -61,8 +63,8 @@ func toolKey(name string, seats Seats) string {
|
||||
|
||||
// toolsOn is the flat catalogue (MESH_CONSOLE_FLAT=1): what announced itself on the bus, every
|
||||
// tool and seat verb, and the assignments with tools that did not (novox/hq ADR 0197).
|
||||
func toolsOn(conn *bus.Conn) (*Listing, error) {
|
||||
x, err := indexOn(conn)
|
||||
func toolsOn(conn *bus.Conn, src Source) (*Listing, error) {
|
||||
x, err := indexOn(conn, src, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -109,7 +111,6 @@ func subjectListed(name, node string, l *Listing) string {
|
||||
}
|
||||
|
||||
var (
|
||||
noResponders = regexp.MustCompile(`(?i)no responders|503`)
|
||||
// The bus's own two refusals, by their whole phrase. "authorization" alone also matched a tool's own
|
||||
// answer that merely mentions the word — the runtime refusing a state value with an Authorization
|
||||
// header (novox/hq ADR 0201) read as "this account may not call", which sent the reader the wrong way.
|
||||
@@ -117,9 +118,6 @@ var (
|
||||
timedOut = regexp.MustCompile(`(?i)timeout`)
|
||||
)
|
||||
|
||||
// controllerSeat is the seat whose holder keeps what came of every call (novox/hq issue 265).
|
||||
const controllerSeat = "mesh-controller"
|
||||
|
||||
// whyItFailed says why a call failed, so the remedy is in the words.
|
||||
func whyItFailed(key string, err error) string {
|
||||
if err == nil {
|
||||
@@ -127,7 +125,8 @@ func whyItFailed(key string, err error) string {
|
||||
}
|
||||
msg := err.Error()
|
||||
switch {
|
||||
case noResponders.MatchString(msg):
|
||||
// By type, not text: a tool's own error that mentions 503 is the tool's answer (issue 1529).
|
||||
case errors.Is(err, bus.ErrNoResponders) || errors.Is(err, nats.ErrNoResponders):
|
||||
extra := ""
|
||||
if strings.HasPrefix(key, "seat:") {
|
||||
extra = ", or nothing holds that seat"
|
||||
@@ -143,12 +142,6 @@ func whyItFailed(key string, err error) string {
|
||||
said := key + " did not answer within " + bus.RequestTimeout.String() + ". That is not a failure: " +
|
||||
"something is serving it, so it may still be running and may already have done what was " +
|
||||
"asked — check before repeating a call that changes the mesh."
|
||||
if strings.HasPrefix(strings.TrimPrefix(key, "seat:"), controllerSeat+".") {
|
||||
// The controller answers every call within seconds, with an id while it runs, so silence
|
||||
// from it is an answer lost on the way — which it keeps.
|
||||
said += " The controller answers every call within seconds, so its answer was lost on the " +
|
||||
"way; " + controllerSeat + ".calls lists its recent calls and what came of each."
|
||||
}
|
||||
return said
|
||||
}
|
||||
return key + " failed: " + msg
|
||||
|
||||
@@ -3,6 +3,7 @@ package console
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -42,6 +43,7 @@ func post(t *testing.T, endpoint string, body string) map[string]any {
|
||||
// As node-tools, the runtime serves the bundles and is the console on loopback: the listing is what
|
||||
// the modules and the mesh's records answered, and a call reaches the module on the machine named.
|
||||
func TestTheConsoleListsAndCallsOverHTTP(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "desk", true, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "desk", false, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
@@ -55,16 +57,21 @@ func TestTheConsoleListsAndCallsOverHTTP(t *testing.T) {
|
||||
}
|
||||
defer stop()
|
||||
|
||||
// The controller's records (ADR 0197): ghost declares tools on desk and nothing announces it.
|
||||
controller := connect(t, "mesh-controller", "")
|
||||
stopSeat, _ := controller.HandleSubject("mesh.seat.mesh-controller.tool.modules", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"ok": true, "output": `[{"module":"alpha","on":["desk"],"tools":true},` +
|
||||
`{"module":"beta","on":["desk"],"tools":true},{"module":"ghost","on":["desk"],"tools":true}]`}, nil
|
||||
// What the mesh holds, as the indexer and the assigner write it (ADR 0571): ghost declares tools on desk and nothing
|
||||
// announces it.
|
||||
mesh.Placed(t, map[string][]string{"alpha": {"one", "two"}, "beta": {"three"}, "ghost": {"boo"}},
|
||||
map[string][]string{"desk": {"alpha", "beta", "ghost"}})
|
||||
console := mt.MembershipOf("node-tools", "desk", false, nil)
|
||||
console.State = mt.ConsoleReads()
|
||||
mesh.Issue(t, console)
|
||||
mt.Until(t, func() error {
|
||||
if m := nodeTools.Membership("node-tools"); m == nil || len(m.State) == 0 {
|
||||
return fmt.Errorf("the console's reads are not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
defer stopSeat()
|
||||
controller.Flush()
|
||||
|
||||
flat := NewSurface(nodeTools, "desk.node-tools")
|
||||
flat := NewSurface(nodeTools, busSource{nodeTools}, "desk.node-tools")
|
||||
flat.Flat = true // the whole catalogue, as before ADR 0195: still reachable, no longer announced
|
||||
up, err := Serve(flat, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
@@ -125,7 +132,7 @@ func TestTheConsoleListsAndCallsOverHTTP(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTheConsoleListensOnLoopbackAndNowhereElse(t *testing.T) {
|
||||
if _, err := Serve(NewSurface(nil, "x"), "0.0.0.0:0"); err == nil || !strings.Contains(err.Error(), "loopback and nowhere else") {
|
||||
if _, err := Serve(NewSurface(nil, nil, "x"), "0.0.0.0:0"); err == nil || !strings.Contains(err.Error(), "loopback and nowhere else") {
|
||||
t.Errorf("a non-loopback console was not refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// novox/hq issue 554, reproduced: the laptop's console heard every runtime, then a round in which
|
||||
// nothing answered discovery in time — and every seat and module call was refused ("it answered
|
||||
// discovery 2s ago, not now") for the five seconds that round was kept, again and again. Now a call
|
||||
// is resolved against what the runtime announced last, says how old that is, and reaches the tool; the
|
||||
// empty round is logged once, is not kept, and the next question asks again.
|
||||
func TestACallAfterARoundThatHeardNothingUsesWhatWasHeardBefore(t *testing.T) {
|
||||
keptWas, patienceWas := IndexKept, announce.Patience
|
||||
IndexKept, announce.Patience = 0, 1200*time.Millisecond
|
||||
t.Cleanup(func() { IndexKept, announce.Patience = keptWas, patienceWas })
|
||||
mt.New(t)
|
||||
|
||||
// The tools answer on their own connection; discovery is answered on another, which goes quiet.
|
||||
tools := connect(t, "node-tools", "laptop")
|
||||
stopTool, err := tools.HandleSubject("mesh.mod.slack.tool.slack_check.laptop", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"ok": true}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopTool)
|
||||
stopSeat, err := tools.HandleSubject("mesh.seat.issue-tracker.tool.list", func(json.RawMessage) (any, error) {
|
||||
return map[string]any{"issues": []int{554}}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopSeat)
|
||||
tools.Flush()
|
||||
endpoints := []announce.Endpoint{
|
||||
{Kind: announce.KindTool, Module: "slack", Tool: "slack_check", Node: "laptop", Subject: "mesh.mod.slack.tool.slack_check.laptop"},
|
||||
{Kind: announce.KindSeat, Module: "issue-tracker", Seat: "issue-tracker", Tool: "list", Scope: "mesh", Node: "laptop",
|
||||
Schema: json.RawMessage(`{}`), Subject: "mesh.seat.issue-tracker.tool.list"},
|
||||
}
|
||||
announcing := connect(t, "node-tools", "laptop")
|
||||
stopAnnounce, err := announce.Serve(announcing, announce.Service{Name: "node-tools", ID: "laptop",
|
||||
Metadata: map[string]string{"node": "laptop"}}, func() []announce.Endpoint { return endpoints })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
announcing.Flush()
|
||||
|
||||
console := connect(t, "node-tools", "laptop-console")
|
||||
var mu sync.Mutex
|
||||
var logged []string
|
||||
console.Logf = func(f string, a ...any) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
logged = append(logged, fmt.Sprintf(f, a...))
|
||||
}
|
||||
said := func(words string) int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range logged {
|
||||
if strings.Contains(l, words) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
surface := NewSurface(console, busSource{console}, "laptop.node-tools")
|
||||
up, err := Serve(surface, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer up.Close()
|
||||
endpoint := "http://" + up.Address + "/mcp"
|
||||
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/slack.slack_check"}); isErr || strings.Contains(got, "announced") {
|
||||
t.Fatalf("while discovery hears the runtime: %s", got)
|
||||
}
|
||||
|
||||
// Discovery goes quiet: no PING arrives at all.
|
||||
stopAnnounce()
|
||||
announcing.Flush()
|
||||
for _, address := range []string{"laptop/slack.slack_check", "issue-tracker.list"} {
|
||||
got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": address})
|
||||
if isErr || !strings.Contains(got, "found from what node-tools on laptop announced") ||
|
||||
!strings.Contains(got, "ago: it did not answer the latest round of discovery") {
|
||||
t.Errorf("%s after a round that heard nothing: %s", address, got)
|
||||
}
|
||||
}
|
||||
if got, isErr := call(t, endpoint, "mesh_describe", map[string]any{"address": "issue-tracker.list"}); isErr || !strings.Contains(got, `"found"`) {
|
||||
t.Errorf("describe after a round that heard nothing: %s", got)
|
||||
}
|
||||
if n := said("discovery heard nothing at all"); n != 1 {
|
||||
t.Errorf("the empty round was logged %d time(s), want once: %v", n, logged)
|
||||
}
|
||||
// A runtime really gone: the call is sent on what it announced, and fails with the bus's own refusal,
|
||||
// saying where the address was found.
|
||||
stopTool()
|
||||
tools.Flush()
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "laptop/slack.slack_check"}); !isErr ||
|
||||
!strings.Contains(got, "nothing serves") || !strings.Contains(got, "found from what node-tools on laptop announced") {
|
||||
t.Errorf("a gone runtime's tool: %s", got)
|
||||
}
|
||||
surface.mu.Lock()
|
||||
kept := surface.idx != nil && !surface.idx.Discovery.Empty() && time.Since(surface.idxAt) < time.Second
|
||||
surface.mu.Unlock()
|
||||
if kept {
|
||||
t.Error("a round that heard nothing was kept as the answer")
|
||||
}
|
||||
|
||||
// Discovery answers again: the calls are plain again, and that is logged once.
|
||||
stopAgain, err := announce.Serve(announcing, announce.Service{Name: "node-tools", ID: "laptop",
|
||||
Metadata: map[string]string{"node": "laptop"}}, func() []announce.Endpoint { return endpoints })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stopAgain)
|
||||
announcing.Flush()
|
||||
if got, isErr := call(t, endpoint, "mesh_call", map[string]any{"address": "issue-tracker.list"}); isErr || strings.Contains(got, "announced") {
|
||||
t.Errorf("once discovery hears the runtime again: %s", got)
|
||||
}
|
||||
if n := said("discovery hears every runtime again"); n != 1 {
|
||||
t.Errorf("the recovery was logged %d time(s): %v", n, logged)
|
||||
}
|
||||
}
|
||||
|
||||
// An address known only from an earlier answer says how old it is; one heard fresh on its machine is
|
||||
// plain, even where the same module is known only from an earlier answer on another.
|
||||
func TestAnAddressKnownOnlyFromAnEarlierAnswerSaysHowOld(t *testing.T) {
|
||||
at := time.Now().Add(-20 * time.Second)
|
||||
x := &index{Modules: map[string]*moduleInfo{"slack": {Module: "slack", On: []string{"laptop"},
|
||||
Tools: []Tool{{Module: "slack", Name: "slack_check"}}}}, Recorded: map[string][]string{},
|
||||
Stale: map[string]staleFrom{"mod:slack@laptop": {Runtime: "node-tools", Machine: "laptop", At: at}},
|
||||
Discovery: announce.Discovery{Pinged: 0}}
|
||||
got, err := resolve(x, "laptop/slack.slack_check")
|
||||
if err != nil || got.Stale == nil || !strings.Contains(got.Stale.say(), "announced 20s ago") {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
// A module heard fresh on one machine and only earlier on another: the fresh one is plain.
|
||||
x.Modules["slack"].On = []string{"desk", "laptop"}
|
||||
if got, err := resolve(x, "desk/slack.slack_check"); err != nil || got.Stale != nil {
|
||||
t.Errorf("a fresh machine's module marked stale: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// FirstViewer is the tool runner's four first views as the console shows them (novox/hq ADR 0615 §6; issue 1966 TLS-6): a
|
||||
// Source that also is one is asked by mesh_first_views and mesh_first_view_again.
|
||||
type FirstViewer interface {
|
||||
Status() []views.Status
|
||||
Again(ctx context.Context, why string) []error
|
||||
}
|
||||
|
||||
// firstViewsAnswer is mesh_first_views and mesh_first_view_again.
|
||||
func (s *Surface) firstViewsAnswer(ctx context.Context, name string, args map[string]any) map[string]any {
|
||||
fv, ok := s.src.(FirstViewer)
|
||||
if !ok {
|
||||
return failure("this tool runner holds no first views: it reads the indexer's and the assigner's states through " +
|
||||
"another source")
|
||||
}
|
||||
if name == verbFirstViews {
|
||||
return answerText(map[string]any{"first views": fv.Status()})
|
||||
}
|
||||
why, _ := args["why"].(string)
|
||||
if why = strings.TrimSpace(why); why == "" {
|
||||
why = "asked by hand with no reason given"
|
||||
}
|
||||
// A hand act, recorded through the seat mesh-hand-acts (novox/hq ADR 0615 §6, ADR 0565 §10) before it is done, and in the
|
||||
// runner's log. One that cannot be recorded is said, and the ask is not made.
|
||||
log.Printf("[mesh-tools] hand act: %s asked the four first views again: %s", s.who, why)
|
||||
if err := s.recordAct(map[string]any{"what": "asked the tool runner's four first views again", "why": why,
|
||||
"cause": "first-view-again", "verb": verbFirstAgain}); err != nil {
|
||||
return failure("the hand act was not recorded through mesh-hand-acts, so the first views were not asked again: " + err.Error())
|
||||
}
|
||||
out := map[string]any{"asked again": why, "recorded": "mesh-hand-acts.hand-act"}
|
||||
if errs := fv.Again(ctx, why); len(errs) > 0 {
|
||||
var msgs []string
|
||||
for _, e := range errs {
|
||||
msgs = append(msgs, e.Error())
|
||||
}
|
||||
out["not answered"] = msgs
|
||||
}
|
||||
out["first views"] = fv.Status()
|
||||
return answerText(out)
|
||||
}
|
||||
|
||||
// recordAct records one act done by hand through the seat mesh-hand-acts (its verb hand-act); Surface.Record replaces the call
|
||||
// in a test.
|
||||
func (s *Surface) recordAct(args map[string]any) error {
|
||||
if s.Record != nil {
|
||||
return s.Record(args)
|
||||
}
|
||||
if s.conn == nil {
|
||||
return errNoBus
|
||||
}
|
||||
_, err := s.conn.Ask("mesh-hand-acts.hand-act", args, "")
|
||||
return err
|
||||
}
|
||||
|
||||
var errNoBus = errors.New("no bus connection")
|
||||
@@ -0,0 +1,59 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// viewer is a Source that also holds first views.
|
||||
type viewer struct {
|
||||
busSource
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (v *viewer) Status() []views.Status {
|
||||
return []views.Status{{Owner: "mesh-delivery-assigner", Tool: "assignments", State: "assignments", Revision: 4, Asks: 1}}
|
||||
}
|
||||
|
||||
func (v *viewer) Again(_ context.Context, why string) []error {
|
||||
v.asked = append(v.asked, why)
|
||||
return nil
|
||||
}
|
||||
|
||||
// issue 1966 TLS-6: the console's mesh_first_views shows each first view, mesh_first_view_again asks them again with its
|
||||
// reason, and a source holding none refuses both in words.
|
||||
func TestTheConsoleShowsAndAsksAgainTheRunnersFirstViews(t *testing.T) {
|
||||
text := func(r map[string]any) string {
|
||||
c, _ := json.Marshal(r["content"])
|
||||
return string(c)
|
||||
}
|
||||
v := &viewer{}
|
||||
var recorded []map[string]any
|
||||
s := &Surface{src: v, who: "desk.node-tools", Record: func(a map[string]any) error { recorded = append(recorded, a); return nil }}
|
||||
if got := text(s.discover(context.Background(), "mesh_first_views", nil)); !strings.Contains(got, "mesh-delivery-assigner") || !strings.Contains(got, "assignments") {
|
||||
t.Errorf("mesh_first_views shows %s", got)
|
||||
}
|
||||
got := s.discover(context.Background(), "mesh_first_view_again", map[string]any{"why": "doubted"})
|
||||
if len(v.asked) != 1 || v.asked[0] != "doubted" || !strings.Contains(text(got), "asked again") {
|
||||
t.Errorf("mesh_first_view_again asked %v and answered %s", v.asked, text(got))
|
||||
}
|
||||
if len(recorded) != 1 || recorded[0]["why"] != "doubted" || recorded[0]["cause"] != "first-view-again" {
|
||||
t.Errorf("the hand act recorded is %v", recorded)
|
||||
}
|
||||
// An act that cannot be recorded is not done.
|
||||
refusing := &Surface{src: v, who: "desk.node-tools", Record: func(map[string]any) error { return errors.New("no answer") }}
|
||||
if r := refusing.discover(context.Background(), "mesh_first_view_again", map[string]any{"why": "x"}); r["isError"] != true || len(v.asked) != 1 {
|
||||
t.Errorf("an unrecorded hand act was done: %s (asked %v)", text(r), v.asked)
|
||||
}
|
||||
for _, name := range []string{"mesh_first_views", "mesh_first_view_again"} {
|
||||
none := &Surface{src: busSource{}, who: "desk.node-tools"}
|
||||
if r := none.discover(context.Background(), name, nil); r["isError"] != true || !strings.Contains(text(r), "holds no first views") {
|
||||
t.Errorf("%s without first views answers %s", name, text(r))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/wire"
|
||||
)
|
||||
@@ -45,12 +46,30 @@ func Serve(s *Surface, listen string) (*Listening, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route(w, r, s) })}
|
||||
// Every request's context ends when the console is closed, so a mesh_wait in flight ends at once rather
|
||||
// than holding a stop or a restart for up to its ten minutes (the review of issue 745).
|
||||
base, cancel := context.WithCancel(context.Background())
|
||||
server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route(w, r, s) }),
|
||||
BaseContext: func(net.Listener) context.Context { return base }}
|
||||
go func() { _ = server.Serve(ln) }()
|
||||
bound := ln.Addr().(*net.TCPAddr).Port
|
||||
return &Listening{Address: host + ":" + strconv.Itoa(bound), Close: func() error { return server.Shutdown(context.Background()) }}, nil
|
||||
return &Listening{Address: host + ":" + strconv.Itoa(bound), Close: func() error {
|
||||
cancel()
|
||||
// A backstop: what is still answering after the cancel is cut off, never waited on for long.
|
||||
ctx, done := context.WithTimeout(context.Background(), ShutdownWithin)
|
||||
defer done()
|
||||
err := server.Shutdown(ctx)
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return server.Close()
|
||||
}
|
||||
return err
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// ShutdownWithin is how long closing the console waits for the requests still being answered after their
|
||||
// context was cancelled.
|
||||
var ShutdownWithin = 5 * time.Second
|
||||
|
||||
func route(w http.ResponseWriter, r *http.Request, s *Surface) {
|
||||
switch r.URL.Path {
|
||||
case "/":
|
||||
@@ -89,7 +108,7 @@ func route(w http.ResponseWriter, r *http.Request, s *Surface) {
|
||||
}
|
||||
replies := []*Reply{}
|
||||
for _, req := range batch {
|
||||
if reply := s.Handle(req); reply != nil {
|
||||
if reply := s.HandleContext(r.Context(), req); reply != nil {
|
||||
replies = append(replies, reply)
|
||||
}
|
||||
}
|
||||
@@ -105,7 +124,7 @@ func route(w http.ResponseWriter, r *http.Request, s *Surface) {
|
||||
writeJSON(w, 400, map[string]any{"jsonrpc": "2.0", "id": nil, "error": map[string]any{"code": -32700, "message": "the body is not JSON"}})
|
||||
return
|
||||
}
|
||||
reply := s.Handle(req)
|
||||
reply := s.HandleContext(r.Context(), req)
|
||||
if reply == nil {
|
||||
w.WriteHeader(202)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0367 §5: the tool runner's manifest names the families a session waits on today, one by one,
|
||||
// and the asker's `decided` for the claude-code module alone. Adding one is a reviewed change to this file
|
||||
// (§6), so this list changes with it. The delivery's transitions are waited on under the gate's name alone (ADR 0450 §8, issues 1350
|
||||
// and 1406): today's module is forgotten.
|
||||
func TestTheManifestNamesTheFirstFamiliesOneByOne(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Waits []string `json:"waits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"mesh-knowledge.reviewed", "mesh-delivery-gate.transition", "gitea.forge.status",
|
||||
"mesh-self-check.checked", "operator-channel.decided.claude-code"}
|
||||
if m.Module != "node-tools" || !reflect.DeepEqual(m.Waits, want) {
|
||||
t.Errorf("%s waits on %v, want %v", m.Module, m.Waits, want)
|
||||
}
|
||||
for _, w := range m.Waits {
|
||||
if strings.ContainsAny(w, "*>") {
|
||||
t.Errorf("a family named by a wildcard: %s", w)
|
||||
}
|
||||
}
|
||||
// A wait is no consumer (ADR 0367 §2): what it consumes is the four views families of its first views (ADR 0615 §5, issue
|
||||
// 1966 TLS-5) and the two events of the seat that serves the seat list (ADR 0636 §4), and nothing else.
|
||||
wantConsumes := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.seat-holders",
|
||||
"mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules"}
|
||||
if !reflect.DeepEqual(m.Consumes, wantConsumes) {
|
||||
t.Errorf("the tool runner consumes %v, want %v", m.Consumes, wantConsumes)
|
||||
}
|
||||
}
|
||||
|
||||
// issue 1966 TLS-1: the tool runner declares its own secret `ledgers`, each module's ledger credential, in the directory it
|
||||
// keeps its broker credential in, which is where the runner reads it (runtime.LedgersFileOf).
|
||||
func TestTheManifestDeclaresTheRunnersOwnSecretLedgers(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"broker": "${dir:mesh-state}/broker", "ledgers": "${dir:mesh-state}/ledgers"}
|
||||
if !reflect.DeepEqual(m.OwnSecrets, want) {
|
||||
t.Errorf("own secrets %v, want %v", m.OwnSecrets, want)
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,10 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -44,6 +47,7 @@ type RPCError struct {
|
||||
// Surface answers MCP requests over one bus connection, as one account.
|
||||
type Surface struct {
|
||||
conn *bus.Conn
|
||||
src Source
|
||||
who string
|
||||
mu sync.Mutex
|
||||
known *Listing
|
||||
@@ -52,14 +56,46 @@ type Surface struct {
|
||||
idxAt time.Time
|
||||
// runtimes is every runtime discovery has heard, as it last heard it (mesh_runtimes).
|
||||
runtimes map[string]*runtimeSeen
|
||||
// roundSaid is what the last round logged was missing, so a round is logged once (novox/hq issue 554).
|
||||
roundSaid string
|
||||
// Flat announces the whole catalogue, as the console did before ADR 0195: for a person reading
|
||||
// it or a client that wants it. Off by default; MESH_CONSOLE_FLAT=1 turns it on.
|
||||
Flat bool
|
||||
// Record, when set, records a hand act in place of the seat mesh-hand-acts' verb (a test's).
|
||||
Record func(args map[string]any) error
|
||||
}
|
||||
|
||||
// MethodSeats is the console's JSON-RPC method beside MCP's that answers every seat's verbs and holders, in the shape
|
||||
// the old `tools` answered: `{seats: [{seat, scope, tools: [{name, description, input, replaces}], holders:
|
||||
// [{module, node}]}], holders_why}`.
|
||||
const MethodSeats = "mesh/seats"
|
||||
|
||||
// seatsAnswer is MethodSeats' answer: mesh-sdk's seat definitions, with each seat's holders by assignment.
|
||||
func (s *Surface) seatsAnswer() map[string]any {
|
||||
placed, err := seatHoldersOn(s.src)
|
||||
x := &index{Placed: placed}
|
||||
seats := []map[string]any{}
|
||||
for _, st := range catalogue.Seats() {
|
||||
holders := x.placedHolders(st.Name, "")
|
||||
if holders == nil {
|
||||
holders = []meshmoduleassigner.SeatHoldersHoldersItem{}
|
||||
}
|
||||
tools := st.Serves
|
||||
if tools == nil {
|
||||
tools = []catalogue.Verb{}
|
||||
}
|
||||
seats = append(seats, map[string]any{"seat": st.Name, "scope": st.Scope, "tools": tools, "holders": holders})
|
||||
}
|
||||
out := map[string]any{"seats": seats}
|
||||
if err != nil {
|
||||
out["holders_why"] = "who holds which seat by assignment could not be read: " + err.Error()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NewSurface is the surface over a connection, as `who`.
|
||||
func NewSurface(conn *bus.Conn, who string) *Surface {
|
||||
return &Surface{conn: conn, who: who, Flat: os.Getenv("MESH_CONSOLE_FLAT") == "1"}
|
||||
func NewSurface(conn *bus.Conn, src Source, who string) *Surface {
|
||||
return &Surface{conn: conn, src: src, who: who, Flat: os.Getenv("MESH_CONSOLE_FLAT") == "1"}
|
||||
}
|
||||
|
||||
func (s *Surface) listing() (*Listing, error) {
|
||||
@@ -70,7 +106,7 @@ func (s *Surface) listing() (*Listing, error) {
|
||||
return l, nil
|
||||
}
|
||||
s.mu.Unlock()
|
||||
l, err := toolsOn(s.conn)
|
||||
l, err := toolsOn(s.conn, s.src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -101,7 +137,11 @@ func idOrNull(id json.RawMessage) json.RawMessage {
|
||||
}
|
||||
|
||||
// Handle answers one request; nil for a notification, which expects none.
|
||||
func (s *Surface) Handle(r Request) *Reply {
|
||||
func (s *Surface) Handle(r Request) *Reply { return s.HandleContext(context.Background(), r) }
|
||||
|
||||
// HandleContext answers one request for as long as ctx lasts: a wait whose client hung up or cancelled
|
||||
// ends at once, its subscription closed (the review of issue 745).
|
||||
func (s *Surface) HandleContext(ctx context.Context, r Request) *Reply {
|
||||
notification := isNotification(r.ID)
|
||||
switch r.Method {
|
||||
case "initialize":
|
||||
@@ -118,6 +158,11 @@ func (s *Surface) Handle(r Request) *Reply {
|
||||
return nil
|
||||
}
|
||||
return answer(r.ID, map[string]any{})
|
||||
case MethodSeats:
|
||||
// The seats' contract and who holds each, for a client on this machine that has no seat definitions of its own
|
||||
// (the TypeScript `mesh`): read in-process from mesh-sdk and from the assigner, never asked of anything else
|
||||
// (novox/hq ADR 0571 clauses 4 and 5).
|
||||
return answer(r.ID, s.seatsAnswer())
|
||||
case "tools/list":
|
||||
if !s.Flat {
|
||||
return answer(r.ID, map[string]any{"tools": discovery()})
|
||||
@@ -162,7 +207,7 @@ func (s *Surface) Handle(r Request) *Reply {
|
||||
if p.Arguments == nil {
|
||||
p.Arguments = map[string]any{}
|
||||
}
|
||||
return answer(r.ID, s.discover(p.Name, p.Arguments))
|
||||
return answer(r.ID, s.discover(ctx, p.Name, p.Arguments))
|
||||
}
|
||||
args := map[string]any{}
|
||||
for k, v := range p.Arguments {
|
||||
@@ -296,14 +341,15 @@ func (s *Surface) instructions() string {
|
||||
"that serves it; what may be called was fixed when this account was issued, so a " +
|
||||
"refusal means the account, not the tool. The list is what the running modules " +
|
||||
"answered, plus every role's tools from the mesh's records — the mesh's own verbs " +
|
||||
"(mesh-controller.status, .push, .assign …) among them; a module that did not answer " +
|
||||
"(mesh-delivery.deliveries, mesh-module-assigner.propose …) among them; a module that did not answer " +
|
||||
"is named in the list's _meta and can still be called by <module>.<tool>."
|
||||
}
|
||||
return "The tools of a Novox mesh, reached as " + s.who + ", found by address rather than listed " +
|
||||
"whole (novox/hq ADR 0195). mesh_overview shows the mesh's seats and machines; mesh_machine one " +
|
||||
"machine's seats and modules; mesh_search finds a tool by words; mesh_describe gives one tool's " +
|
||||
"arguments; mesh_call calls it; mesh_runtimes says which machines' runtimes answered discovery, how fast " +
|
||||
"and how much — ask it when an address is said to be missing or on another machine. " + grammar + " What may be called was fixed when this account " +
|
||||
"and how much — ask it when an address is said to be missing or on another machine; mesh_wait waits on the " +
|
||||
"bus for an event instead of polling (novox/hq ADR 0367). " + grammar + " What may be called was fixed when this account " +
|
||||
"was issued, so a refusal means the account, not the tool."
|
||||
}
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@ import (
|
||||
// 2026-10-05 the controller's push was described as taking nothing, and a push naming one machine
|
||||
// ran as a push of every machine behind.
|
||||
func TestAMeshSeatsNodeIsItsOwnArgument(t *testing.T) {
|
||||
push := Tool{Module: "mesh-controller", Name: "push", Seat: true, Scope: "mesh",
|
||||
push := Tool{Module: "mesh-module-assigner", Name: "push", Seat: true, Scope: "mesh",
|
||||
Input: json.RawMessage(`{"type":"object","properties":{"node":{"type":"string"},"behind":{"type":"string"}}}`)}
|
||||
mesh := target{Address: "mesh-controller.push", Tool: push, Seat: true}
|
||||
mesh := target{Address: "mesh-module-assigner.push", Tool: push, Seat: true}
|
||||
|
||||
props := schemaAsPassed(mesh)["properties"].(map[string]any)
|
||||
if _, has := props["node"]; !has {
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// A tool's own error that mentions 503 is the tool's answer, not a routing fault (issue 1529).
|
||||
func TestAToolsOwn503IsNotNothingServes(t *testing.T) {
|
||||
got := whyItFailed("scanner.scan", errors.New("the scanner refused the scan job: 503 Service Unavailable"))
|
||||
if strings.Contains(got, "nothing serves") || !strings.HasPrefix(got, "scanner.scan failed: ") || !strings.Contains(got, "503") {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARealNoRespondersStillSaysNothingServes(t *testing.T) {
|
||||
for _, err := range []error{bus.ErrNoResponders, fmt.Errorf("ask: %w", bus.ErrNoResponders), nats.ErrNoResponders} {
|
||||
if got := whyItFailed("a.b", err); !strings.HasPrefix(got, "nothing serves a.b") {
|
||||
t.Fatalf("%v: got %q", err, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,16 +22,29 @@ func TestARefusalIsSaidOnlyForTheBussOwn(t *testing.T) {
|
||||
}
|
||||
|
||||
// **A timeout says it is not a failure** (novox/hq issue 265): a push that outlasted the wait had
|
||||
// pushed, and the caller read "did not answer in time". And from the controller, where its answer is.
|
||||
// pushed, and the caller read "did not answer in time". No hint names a seat that is gone (novox/hq issue 1126):
|
||||
// the grammar's example, the handshake's instructions and a timeout's remedy named the controller's verbs.
|
||||
func TestATimeoutIsNotAFailure(t *testing.T) {
|
||||
got := whyItFailed("seat:mesh-controller.push", errors.New("timeout"))
|
||||
for _, want := range []string{"not a failure", "may already have done", "mesh-controller.calls"} {
|
||||
got := whyItFailed("seat:mesh-delivery.deliveries", errors.New("timeout"))
|
||||
for _, want := range []string{"not a failure", "may already have done"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("a controller timeout does not say %q: %s", want, got)
|
||||
t.Errorf("a seat's timeout does not say %q: %s", want, got)
|
||||
}
|
||||
}
|
||||
if other := whyItFailed("novox/postgres.postgres_list_databases", errors.New("timeout")); !strings.Contains(other, "not a failure") ||
|
||||
strings.Contains(other, "calls") {
|
||||
t.Errorf("a module's timeout: %s", other)
|
||||
for name, text := range map[string]string{
|
||||
"grammar": grammar,
|
||||
"flat instructions": (&Surface{Flat: true, who: "x"}).instructions(),
|
||||
"deep instructions": (&Surface{who: "x"}).instructions(),
|
||||
"timeout": got,
|
||||
} {
|
||||
if strings.Contains(text, "controller") {
|
||||
t.Errorf("%s names the controller: %s", name, text)
|
||||
}
|
||||
// The seats were renamed (ADR 0634 §9): no hint names the seat that placed modules or enrolled a machine by its old name.
|
||||
for _, old := range []string{"mesh-placement", "mesh-enrolment"} {
|
||||
if strings.Contains(text, old) {
|
||||
t.Errorf("%s names the seat %s, which is now mesh-module-assigner or mesh-node-enroller: %s", name, old, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,9 @@ package console
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
@@ -31,6 +33,11 @@ type runtimeSeen struct {
|
||||
Shortened string
|
||||
Answered bool // in the latest round
|
||||
Why string // why not, when it did not
|
||||
// Reused is true when its PING said its announcement was unchanged, so it was not fetched again.
|
||||
Reused bool
|
||||
// Last is its last whole answer: what a round that does not hear it uses instead, and what an
|
||||
// unchanged digest stands for (novox/hq issue 554).
|
||||
Last announce.Heard
|
||||
}
|
||||
|
||||
func runtimeKey(name, id string) string { return name + "/" + id }
|
||||
@@ -50,7 +57,108 @@ func seenOf(h announce.Heard, at time.Time) *runtimeSeen {
|
||||
}
|
||||
}
|
||||
return &runtimeSeen{Name: h.Info.Name, ID: h.Info.ID, Machine: h.Machine(), LastHeard: at, Took: h.Took,
|
||||
Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true}
|
||||
Bytes: h.Bytes, Modules: len(modules), Tools: len(tools), SeatVerbs: len(verbs), Shortened: h.Shortened, Answered: true,
|
||||
Reused: h.Reused, Last: h}
|
||||
}
|
||||
|
||||
// held is the answer this console holds for an instance at this digest (announce.Known): an instance
|
||||
// whose announcement is unchanged is not asked for it again (novox/hq issue 554).
|
||||
func (s *Surface) held(name, id, digest string) (announce.Heard, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
r := s.runtimes[runtimeKey(name, id)]
|
||||
if r == nil || r.Last.Digest == "" || r.Last.Digest != digest || r.Last.Info.Name == "" {
|
||||
return announce.Heard{}, false
|
||||
}
|
||||
h := r.Last
|
||||
h.Took, h.Reused = 0, false
|
||||
return h, true
|
||||
}
|
||||
|
||||
// before is the last answer of every runtime this round did not hear, heard within RuntimeForgotten
|
||||
// and not since restarted under another instance (novox/hq issue 554).
|
||||
func (s *Surface) before(d announce.Discovery) []earlier {
|
||||
heard := map[string]bool{}
|
||||
running := map[string]bool{}
|
||||
for _, h := range d.Heard {
|
||||
heard[runtimeKey(h.Info.Name, h.Info.ID)] = true
|
||||
if m := h.Machine(); m != "" {
|
||||
running[h.Info.Name+"@"+m] = true
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []earlier
|
||||
for k, r := range s.runtimes {
|
||||
if heard[k] || r.Last.Info.Name == "" || r.LastHeard.IsZero() || time.Since(r.LastHeard) > RuntimeForgotten {
|
||||
continue
|
||||
}
|
||||
if r.Machine != "" && running[r.Name+"@"+r.Machine] {
|
||||
continue // restarted: the new instance answered
|
||||
}
|
||||
out = append(out, earlier{Heard: r.Last, At: r.LastHeard})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return runtimeKey(out[i].Heard.Info.Name, out[i].Heard.Info.ID) < runtimeKey(out[j].Heard.Info.Name, out[j].Heard.Info.ID)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// sayRound logs a round of discovery that did not hear every runtime — once, when what it missed
|
||||
// changes, and once when every runtime is heard again — so the next case leaves evidence in the
|
||||
// journal (novox/hq issue 554). Called with s.mu held, after remember.
|
||||
func (s *Surface) sayRound(x *index) {
|
||||
d := x.Discovery
|
||||
var missed []string
|
||||
for _, u := range x.Unheard {
|
||||
who := u.Runtime
|
||||
if u.Machine != "" {
|
||||
who += " on " + u.Machine
|
||||
}
|
||||
missed = append(missed, who)
|
||||
}
|
||||
sort.Strings(missed)
|
||||
shape := strings.Join(missed, ", ")
|
||||
if d.Empty() {
|
||||
shape = "empty|" + shape
|
||||
}
|
||||
if shape == s.roundSaid {
|
||||
return
|
||||
}
|
||||
s.roundSaid = shape
|
||||
reused := 0
|
||||
for _, h := range d.Heard {
|
||||
if h.Reused {
|
||||
reused++
|
||||
}
|
||||
}
|
||||
logf := log.Printf
|
||||
if s.conn != nil && s.conn.Logf != nil {
|
||||
logf = s.conn.Logf
|
||||
}
|
||||
window := fmt.Sprintf("waited %s (window %s, patience %s)", d.Waited.Round(time.Millisecond), announce.Window, announce.Patience)
|
||||
held := ""
|
||||
if len(x.Earlier) > 0 {
|
||||
held = fmt.Sprintf("; what %d of them announced earlier is used, with its age", len(x.Earlier))
|
||||
}
|
||||
switch {
|
||||
case d.Empty():
|
||||
logf("[mesh-tools] discovery heard nothing at all: no PING arrived; %s; not heard: %s%s; asking again at the next question",
|
||||
window, orNone(missed), held)
|
||||
case len(missed) > 0:
|
||||
logf("[mesh-tools] discovery did not hear every runtime: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s; not heard: %s%s",
|
||||
d.Pinged, len(d.Heard), reused, window, shape, held)
|
||||
default:
|
||||
logf("[mesh-tools] discovery hears every runtime again: %d PING answer(s), %d announcement(s) (%d unchanged, not fetched); %s",
|
||||
d.Pinged, len(d.Heard), reused, window)
|
||||
}
|
||||
}
|
||||
|
||||
func orNone(xs []string) string {
|
||||
if len(xs) == 0 {
|
||||
return "none heard before either"
|
||||
}
|
||||
return strings.Join(xs, ", ")
|
||||
}
|
||||
|
||||
// remember records a round of discovery and adds to the index every runtime heard lately that did not
|
||||
@@ -101,14 +209,25 @@ func (s *Surface) remember(x *index) {
|
||||
continue
|
||||
}
|
||||
r.Answered = false
|
||||
used := ""
|
||||
if at, ok := x.Earlier[k]; ok {
|
||||
used = fmt.Sprintf("; what it announced %s ago is used", time.Since(at).Round(time.Second))
|
||||
}
|
||||
if why, ok := silent[k]; ok {
|
||||
r.Why = why
|
||||
if used != "" {
|
||||
for i := range x.Unheard {
|
||||
if x.Unheard[i].Runtime == r.Name && x.Unheard[i].Machine == r.Machine && x.Unheard[i].Why == why {
|
||||
x.Unheard[i].Why += used
|
||||
}
|
||||
}
|
||||
}
|
||||
continue // already in the index's unheard
|
||||
}
|
||||
r.Why = "it did not answer this round of discovery"
|
||||
if time.Since(r.LastHeard) <= RuntimeForgotten {
|
||||
x.Unheard = append(x.Unheard, unheard{Runtime: r.Name, Machine: r.Machine,
|
||||
Why: fmt.Sprintf("it answered discovery %s ago, not now", time.Since(r.LastHeard).Round(time.Second))})
|
||||
Why: fmt.Sprintf("it answered discovery %s ago, not now%s", time.Since(r.LastHeard).Round(time.Second), used)})
|
||||
}
|
||||
}
|
||||
for _, u := range x.Discovery.Silent {
|
||||
@@ -157,6 +276,9 @@ func (s *Surface) runtimesAnswer() map[string]any {
|
||||
if r.Shortened != "" {
|
||||
entry["shortened, how"] = r.Shortened
|
||||
}
|
||||
if r.Reused && r.Answered {
|
||||
entry["announcement"] = "unchanged (its PING's digest), so not fetched again; took is its PING's"
|
||||
}
|
||||
}
|
||||
if r.Answered {
|
||||
onMachine[r.Machine] = true
|
||||
@@ -184,7 +306,7 @@ func (s *Surface) runtimesAnswer() map[string]any {
|
||||
return answerText(map[string]any{
|
||||
"asked": x.Discovery.At.Format(time.RFC3339),
|
||||
"waited": x.Discovery.Waited.Round(time.Millisecond).String(),
|
||||
"window": fmt.Sprintf("at least %s; up to %s for a runtime that answered PING", announce.Window, announce.Patience),
|
||||
"window": fmt.Sprintf("at least %s; up to %s for a runtime that answered PING, or until one does", announce.Window, announce.Patience),
|
||||
"answered": answered,
|
||||
"not heard": notHeard,
|
||||
})
|
||||
|
||||
@@ -2,14 +2,19 @@ package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
assigner "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
seatassigner "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
@@ -23,18 +28,22 @@ func slowRuntime(t *testing.T, conn *bus.Conn, node string, lag time.Duration, e
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
stop, err = conn.Raw("$SRV.INFO", func(string, []byte) []byte {
|
||||
// Asked of every service, and of this instance by name and id — as discovery asks it (issue 554).
|
||||
info := func(string, []byte) []byte {
|
||||
if lag < 0 {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(lag)
|
||||
body, _ := json.Marshal(announce.Info(announce.Service{Name: id.Name, ID: id.ID, Metadata: id.Metadata}, endpoints))
|
||||
return body
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
for _, subject := range []string{"$SRV.INFO", "$SRV.INFO." + id.Name + "." + id.ID} {
|
||||
stop, err := conn.Raw(subject, info)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
conn.Flush()
|
||||
}
|
||||
|
||||
@@ -48,7 +57,25 @@ func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T)
|
||||
// Kept for the test: one round of discovery answers every question but mesh_runtimes, which asks anew.
|
||||
IndexKept, announce.Patience = time.Minute, 2500*time.Millisecond
|
||||
t.Cleanup(func() { IndexKept, announce.Patience = keptWas, patienceWas })
|
||||
mt.New(t)
|
||||
mesh := mt.New(t)
|
||||
// What the mesh holds, as the indexer and the assigner write it (ADR 0571): never the controller's records.
|
||||
mesh.Placed(t, map[string][]string{"slack": {"slack_check"}, "nap": {"doze"}, "nextcloud-client": {"nextcloud_check"}},
|
||||
map[string][]string{"laptop": {"slack", "nextcloud-client"}, "sleeper": {"nap", "nextcloud-client"},
|
||||
"desktop": {"nextcloud-client"}, "bench": nil})
|
||||
// Who holds which seat, by assignment (ADR 0571 clause 4), as the seat mesh-module-assigner's holder composes it.
|
||||
placedKV := mesh.Bucket(t, meshtest.SeatHoldersBucket)
|
||||
holdersKey, err := assigner.SeatHoldersKey("mesh-wants", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(seatassigner.SeatHolders{V: assigner.Version, Seat: "mesh-wants", Scope: "mesh",
|
||||
Holders: []seatassigner.SeatHoldersHoldersItem{{Module: "mesh-delivery-assigner", Node: "bench"}}})
|
||||
if _, err := placedKV.Put(holdersKey, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consoleMembership := mt.MembershipOf("node-tools", "laptop-console", false, nil)
|
||||
consoleMembership.State = mt.ConsoleReads()
|
||||
mesh.Issue(t, consoleMembership)
|
||||
|
||||
laptop := connect(t, "node-tools", "laptop")
|
||||
stopTool, err := laptop.HandleSubject("mesh.mod.slack.tool.slack_check.laptop", func(json.RawMessage) (any, error) {
|
||||
@@ -72,14 +99,17 @@ func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T)
|
||||
Subject: "mesh.mod.nextcloud-client.tool.nextcloud_check.desktop"},
|
||||
})
|
||||
|
||||
controller := connect(t, "mesh-controller", "bench")
|
||||
controller := connect(t, "mesh-module-assigner", "bench")
|
||||
var toolsAsked atomic.Int32
|
||||
for verb, output := range map[string]string{
|
||||
"nodes": `[{"name":"laptop"},{"name":"sleeper"},{"name":"desktop"},{"name":"bench"}]`,
|
||||
"modules": `[{"module":"slack","on":["laptop"],"tools":true},{"module":"nap","on":["sleeper"],"tools":true},` +
|
||||
`{"module":"nextcloud-client","on":["laptop","desktop","sleeper"],"tools":true}]`,
|
||||
// The controller's records are served here only to see that none is asked (ADR 0571 clauses 4 and 5).
|
||||
"nodes": `[{"name":"laptop"}]`,
|
||||
"modules": `[]`,
|
||||
"tools": `{"seats":[]}`,
|
||||
} {
|
||||
output := output
|
||||
stop, err := controller.HandleSubject("mesh.seat.mesh-controller.tool."+verb, func(json.RawMessage) (any, error) {
|
||||
output, verb := output, verb
|
||||
stop, err := controller.HandleSubject("mesh.seat.mesh-module-assigner.tool."+verb, func(json.RawMessage) (any, error) {
|
||||
toolsAsked.Add(1)
|
||||
return map[string]any{"output": output, "ok": true}, nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -90,7 +120,14 @@ func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T)
|
||||
controller.Flush()
|
||||
|
||||
console := connect(t, "node-tools", "laptop-console")
|
||||
up, err := Serve(NewSurface(console, "laptop.node-tools"), "127.0.0.1:0")
|
||||
console.Follow("node-tools")
|
||||
mt.Until(t, func() error {
|
||||
if console.Membership("node-tools") == nil {
|
||||
return fmt.Errorf("the console's membership is not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
up, err := Serve(NewSurface(console, busSource{console}, "laptop.node-tools"), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -109,8 +146,20 @@ func TestARuntimeThatAnswersLateIsFoundAndOneThatNeverDoesIsNamed(t *testing.T)
|
||||
t.Errorf("the late runtime's machine lists nothing: %s", machine)
|
||||
}
|
||||
|
||||
// The seats' contract and holders, for a client on this machine with no seat definitions (the TypeScript `mesh`).
|
||||
seats := post(t, endpoint, `{"jsonrpc":"2.0","id":9,"method":"mesh/seats"}`)
|
||||
if raw, _ := json.Marshal(seats["result"]); !strings.Contains(string(raw), `"holders":[{"module":"mesh-delivery-assigner","node":"bench"}],"scope":"mesh","seat":"mesh-wants"`) ||
|
||||
!strings.Contains(string(raw), `"replaces":["journalctl"]`) {
|
||||
t.Errorf("mesh/seats answered %s", raw)
|
||||
}
|
||||
if n := toolsAsked.Load(); n != 0 {
|
||||
t.Errorf("the console asked the controller's records %d times", n)
|
||||
}
|
||||
|
||||
// The silent one is named, never called missing.
|
||||
for address, want := range map[string]string{
|
||||
// A seat whose holder announced nothing is named with its holder by assignment, never called missing.
|
||||
"mesh-wants.read": "the seat mesh-wants is held by mesh-delivery-assigner on bench by assignment",
|
||||
"sleeper/nap.doze": "nap is assigned to sleeper in the mesh's records, but nothing that answered discovery serves it: discovery did not hear in full from node-tools on sleeper (it answered PING",
|
||||
"sleeper/nextcloud-client.nextcloud_check": "nextcloud-client on sleeper did not answer discovery: discovery did not hear in full from node-tools on sleeper",
|
||||
} {
|
||||
@@ -175,7 +224,7 @@ func TestARuntimeHeardBeforeAndNotNowIsNamed(t *testing.T) {
|
||||
s.remember(before)
|
||||
now := &index{Modules: map[string]*moduleInfo{"nextcloud-client": {Module: "nextcloud-client", On: []string{"desktop"},
|
||||
Tools: []Tool{{Module: "nextcloud-client", Name: "nextcloud_check"}}}}, Recorded: map[string][]string{},
|
||||
Discovery: announce.Discovery{At: time.Now().UTC()}}
|
||||
Discovery: announce.Discovery{At: time.Now().UTC(), Pinged: 1}} // desktop's runtime answered
|
||||
s.remember(now)
|
||||
if len(now.Unheard) != 1 || now.Unheard[0].Machine != "laptop" {
|
||||
t.Fatalf("unheard: %+v", now.Unheard)
|
||||
@@ -190,7 +239,7 @@ func TestARuntimeHeardBeforeAndNotNowIsNamed(t *testing.T) {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
// Nobody unheard: the plain answers stand.
|
||||
quiet := &index{Modules: now.Modules, Recorded: map[string][]string{}}
|
||||
quiet := &index{Modules: now.Modules, Recorded: map[string][]string{}, Discovery: now.Discovery}
|
||||
if _, err := resolve(quiet, "laptop/nextcloud-client.nextcloud_check"); err == nil || !strings.Contains(err.Error(), "does not run on laptop; it runs on desktop") {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
|
||||
@@ -19,7 +19,8 @@ package console
|
||||
// Ties go to a seat — the place tools belong — then to the shorter name, then by address.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -44,6 +45,31 @@ var synonyms = map[string][]string{
|
||||
"hosts": {"hostname"},
|
||||
}
|
||||
|
||||
// filler are the function words a question carries and no tool is named for: a query's words must all
|
||||
// match, so "did our weekly upgrade run on all our nodes" would otherwise find nothing for "did", "our",
|
||||
// "on" and "all". Kept short and here only; a word that names something (`every`, `list`, `not`) is not
|
||||
// in it. A query of nothing but these matches nothing.
|
||||
var filler = map[string]bool{}
|
||||
|
||||
func init() {
|
||||
for _, w := range strings.Fields("a an the did do does our my your we i is are was on in at of for to all any and or with it its there what which how has have been") {
|
||||
filler[w] = true
|
||||
}
|
||||
}
|
||||
|
||||
// queryWords are a query's words to match: split at spaces, the punctuation of a sentence trimmed from
|
||||
// each end (`nodes?` is `nodes`), filler left out.
|
||||
func queryWords(query string) []string {
|
||||
var out []string
|
||||
for _, w := range strings.Fields(query) {
|
||||
w = strings.Trim(w, "?!.,;:\"'()")
|
||||
if w != "" && !filler[w] {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hit is one match as mesh_search answers it.
|
||||
type hit struct {
|
||||
Address string `json:"address"`
|
||||
@@ -169,8 +195,11 @@ func scoreOf(query string, qwords []string, name, description string, replaces [
|
||||
// search is mesh_search's answer: every match, best first.
|
||||
func (x *index) search(query string) []hit {
|
||||
query = strings.ToLower(strings.TrimSpace(query))
|
||||
qwords := strings.Fields(query)
|
||||
qwords := queryWords(query)
|
||||
var hits []hit
|
||||
if len(qwords) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, st := range x.Seats {
|
||||
for _, v := range st.Verbs {
|
||||
replaces := x.Replaces["seat:"+st.Seat+"."+v.Name]
|
||||
@@ -237,27 +266,16 @@ func (x *index) search(query string) []hit {
|
||||
return hits
|
||||
}
|
||||
|
||||
// replacesOf is what every verb and tool replaces, from the controller's records (novox/hq ADR 0245):
|
||||
// a seat's verbs from its `tools` answer, keyed `seat:<seat>.<verb>`, and a module's own tools from
|
||||
// its `modules` answer, keyed `<module>.<tool>`. A controller older than the field answers neither,
|
||||
// and search matches names and descriptions as before.
|
||||
func replacesOf(toolsAnswer json.RawMessage, modules []recordedModule) map[string][]string {
|
||||
// replacesOf is what every verb and tool replaces (novox/hq ADR 0245): a seat's verbs from the seat's contract in
|
||||
// mesh-sdk, keyed `seat:<seat>.<verb>`, read in-process and asked of no part (ADR 0571 clause 5: no part keeps a
|
||||
// central record of tools); and a module's own tools from its records, keyed `<module>.<tool>`. A verb or a tool with
|
||||
// no `replaces` is matched by its name and description as before.
|
||||
func replacesOf(seats []catalogue.Seat, modules []recordedModule) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
var seats struct {
|
||||
Seats []struct {
|
||||
Seat string `json:"seat"`
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
Replaces []string `json:"replaces"`
|
||||
} `json:"tools"`
|
||||
} `json:"seats"`
|
||||
}
|
||||
if json.Unmarshal(toolsAnswer, &seats) == nil {
|
||||
for _, s := range seats.Seats {
|
||||
for _, t := range s.Tools {
|
||||
if len(t.Replaces) > 0 {
|
||||
out["seat:"+s.Seat+"."+t.Name] = t.Replaces
|
||||
}
|
||||
for _, s := range seats {
|
||||
for _, v := range s.Serves {
|
||||
if len(v.Replaces) > 0 {
|
||||
out["seat:"+s.Name+"."+v.Name] = append([]string(nil), v.Replaces...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -19,6 +20,7 @@ func meshToSearch(t *testing.T) *index {
|
||||
"status": "One unit as the service manager sees it now: its states, whether it starts at boot.",
|
||||
"restart": "Restart one unit.",
|
||||
"journal": "The last lines of one unit's journal (at most 2000).",
|
||||
"failed": "The units that failed.",
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
for verb, does := range verbs {
|
||||
@@ -31,8 +33,8 @@ func meshToSearch(t *testing.T) *index {
|
||||
"node": "What one machine reported it can do, what it is assigned, and why.",
|
||||
"nodes": "Every machine the mesh knows, with whether it is converged or adopted.",
|
||||
} {
|
||||
endpoints = append(endpoints, announce.Endpoint{Kind: announce.KindSeat, Module: "mesh-controller", Seat: "mesh-controller",
|
||||
Tool: verb, Scope: "mesh", Description: does, Subject: "mesh.seat.mesh-controller.tool." + verb})
|
||||
endpoints = append(endpoints, announce.Endpoint{Kind: announce.KindSeat, Module: "mesh-module-assigner", Seat: "mesh-module-assigner",
|
||||
Tool: verb, Scope: "mesh", Description: does, Subject: "mesh.seat.mesh-module-assigner.tool." + verb})
|
||||
}
|
||||
for tool, does := range map[string]string{
|
||||
"docker_list": "Every container on this machine: name, image, state, ports.",
|
||||
@@ -44,13 +46,7 @@ func meshToSearch(t *testing.T) *index {
|
||||
Description: does, Subject: "mesh.mod.docker.tool." + tool + ".anchor"})
|
||||
}
|
||||
x, _, _ := indexOfEndpoints(endpoints)
|
||||
tools := json.RawMessage(`{"seats":[{"seat":"node-service-manager","scope":"node","tools":[
|
||||
{"name":"units","replaces":["systemctl list-units","systemctl --failed"]},
|
||||
{"name":"status","replaces":["systemctl status","systemctl is-active"]},
|
||||
{"name":"restart","replaces":["systemctl restart"]},
|
||||
{"name":"journal","replaces":["journalctl"]}]},
|
||||
{"seat":"mesh-controller","scope":"mesh","tools":[{"name":"node","replaces":["hostnamectl","uptime"]},{"name":"nodes"}]}]}`)
|
||||
x.Replaces = replacesOf(tools, []recordedModule{{Module: "docker", On: []string{"anchor"}, Tools: true,
|
||||
x.Replaces = replacesOf(catalogue.Seats(), []recordedModule{{Module: "docker", On: []string{"anchor"}, Tools: true,
|
||||
Replaces: map[string][]string{"docker_list": {"docker ps"}, "docker_logs": {"docker logs"}, "docker_restart": {"docker restart"}}}})
|
||||
return x
|
||||
}
|
||||
@@ -60,22 +56,21 @@ func meshToSearch(t *testing.T) *index {
|
||||
func TestSearchFindsTheToolThatReplacesACommandFirst(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
for query, want := range map[string]string{
|
||||
"journalctl": "anchor/node-service-manager.journal",
|
||||
"journalctl -u mesh-controller -n 50": "anchor/node-service-manager.journal",
|
||||
"sudo journalctl -fu sshd": "anchor/node-service-manager.journal",
|
||||
"/usr/bin/journalctl": "anchor/node-service-manager.journal",
|
||||
"logs": "anchor/node-service-manager.journal",
|
||||
"systemctl status": "anchor/node-service-manager.status",
|
||||
"systemctl status nats.service": "anchor/node-service-manager.status",
|
||||
"systemctl restart mesh-controller": "anchor/node-service-manager.restart",
|
||||
"systemctl --failed": "anchor/node-service-manager.units",
|
||||
"docker ps": "anchor/docker.docker_list",
|
||||
"docker ps -a": "anchor/docker.docker_list",
|
||||
"docker logs": "anchor/docker.docker_logs",
|
||||
"docker logs --tail 100 nats": "anchor/docker.docker_logs",
|
||||
"docker restart nats": "anchor/docker.docker_restart",
|
||||
"container logs": "anchor/docker.docker_logs",
|
||||
"uptime": "mesh-controller.node",
|
||||
"journalctl": "anchor/node-service-manager.journal",
|
||||
"journalctl -u mesh-module-assigner -n 50": "anchor/node-service-manager.journal",
|
||||
"sudo journalctl -fu sshd": "anchor/node-service-manager.journal",
|
||||
"/usr/bin/journalctl": "anchor/node-service-manager.journal",
|
||||
"logs": "anchor/node-service-manager.journal",
|
||||
"systemctl status": "anchor/node-service-manager.status",
|
||||
"systemctl status nats.service": "anchor/node-service-manager.status",
|
||||
"systemctl restart mesh-module-assigner": "anchor/node-service-manager.restart",
|
||||
"systemctl --failed": "anchor/node-service-manager.failed",
|
||||
"docker ps": "anchor/docker.docker_list",
|
||||
"docker ps -a": "anchor/docker.docker_list",
|
||||
"docker logs": "anchor/docker.docker_logs",
|
||||
"docker logs --tail 100 nats": "anchor/docker.docker_logs",
|
||||
"docker restart nats": "anchor/docker.docker_restart",
|
||||
"container logs": "anchor/docker.docker_logs",
|
||||
} {
|
||||
hits := x.search(query)
|
||||
if len(hits) == 0 {
|
||||
@@ -116,7 +111,7 @@ func TestASearchByCommandSaysWhatItReplaces(t *testing.T) {
|
||||
// A synonym is a word, never part of one: `logs` matches through `log`, and finds no `login`.
|
||||
func TestSearchStillMatchesEveryWord(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
if hits := x.search("machine converged"); len(hits) != 1 || hits[0].Address != "mesh-controller.nodes" {
|
||||
if hits := x.search("machine converged"); len(hits) != 1 || hits[0].Address != "mesh-module-assigner.nodes" {
|
||||
t.Errorf("words in a description: %+v", hits)
|
||||
}
|
||||
if hits := x.search("journalctl kubernetes"); len(hits) != 1 || hits[0].Address != "anchor/node-service-manager.journal" {
|
||||
@@ -134,10 +129,10 @@ func TestSearchStillMatchesEveryWord(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A controller older than `replaces` answers without it: search matches names and descriptions as before.
|
||||
// A seat's contract and a module's records without `replaces`: search matches names and descriptions as before.
|
||||
func TestSearchWithoutReplacesMatchesAsBefore(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
x.Replaces = replacesOf(json.RawMessage(`{"seats":[{"seat":"node-service-manager","tools":[{"name":"journal"}]}]}`),
|
||||
x.Replaces = replacesOf([]catalogue.Seat{{Name: "node-service-manager", Serves: []catalogue.Verb{{Name: "journal"}}}},
|
||||
[]recordedModule{{Module: "docker"}})
|
||||
if len(x.Replaces) != 0 {
|
||||
t.Fatalf("replaces from records that have none: %v", x.Replaces)
|
||||
@@ -149,3 +144,31 @@ func TestSearchWithoutReplacesMatchesAsBefore(t *testing.T) {
|
||||
t.Errorf("a tool's own name: %+v", hits)
|
||||
}
|
||||
}
|
||||
|
||||
// A question's filler words do not have to match (every other word still does): the operator's own words
|
||||
// find the tool whose description says it in other words; a query of filler alone finds nothing.
|
||||
func TestSearchIgnoresFillerWords(t *testing.T) {
|
||||
x := meshToSearch(t)
|
||||
x.Modules["pacman"] = &moduleInfo{Module: "pacman", On: []string{"anchor"}, Tools: []Tool{{Module: "pacman", Name: "pacman_round",
|
||||
Description: "Whether this week's system upgrade ran on every node (the weekly upgrade round, last run of all nodes)."}}}
|
||||
for _, c := range []struct {
|
||||
query string
|
||||
want string // the first hit's address, "" for none
|
||||
}{
|
||||
{"did our weekly system upgrade run on all our nodes?", "anchor/pacman.pacman_round"},
|
||||
{"the weekly upgrade", "anchor/pacman.pacman_round"},
|
||||
{"did our", ""},
|
||||
{"what is the", ""},
|
||||
{"", ""},
|
||||
{"did our weekly kubernetes upgrade run?", ""},
|
||||
} {
|
||||
hits := x.search(c.query)
|
||||
got := ""
|
||||
if len(hits) > 0 {
|
||||
got = hits[0].Address
|
||||
}
|
||||
if got != c.want || (c.want == "" && len(hits) != 0) {
|
||||
t.Errorf("%q: first hit %q, want %q (%+v)", c.query, got, c.want, hits)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
|
||||
)
|
||||
|
||||
// The seats the tests describe are given to catalogue as the tool runner gives the ones it read from the bus
|
||||
// (novox/hq ADR 0635 §1); catalogue compiles none.
|
||||
func init() { catalogue.UseSeats(catalogue.SeatsFromDefinitions(seatlisttest.Definitions())) }
|
||||
@@ -0,0 +1,39 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// busSource is a Source over a bus connection's own states, which a test's fixture mesh fills: the console's tests stand a
|
||||
// membership that reads the four states beside it, where the tool runner itself holds first views (firstview.Views).
|
||||
type busSource struct{ conn *bus.Conn }
|
||||
|
||||
func (b busSource) State(name string) (map[string][]byte, error) {
|
||||
var mu sync.Mutex
|
||||
keys := map[string][]byte{}
|
||||
stop, err := b.conn.StateWatch(b.conn.Credential().Module, name, "", func(c bus.StateChange) error {
|
||||
if c.Current && c.Op == "put" {
|
||||
mu.Lock()
|
||||
keys[c.Key] = c.Value
|
||||
mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stop()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (b busSource) Entry(name, key string) ([]byte, error) {
|
||||
entry, err := b.conn.StateGet(b.conn.Credential().Module, name, key)
|
||||
if err != nil || entry == nil {
|
||||
return nil, err
|
||||
}
|
||||
return entry.Value, nil
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package console
|
||||
|
||||
// mesh_wait: a session waits on the bus for one event instead of polling (novox/hq ADR 0367).
|
||||
//
|
||||
// **Only the families the tool runner was issued** (§2, §6): its membership names them, composed by the
|
||||
// mesh from the tool runner's manifest together with the grant that lets this account subscribe them.
|
||||
// A name not listed there is refused here, before the bus would refuse it; no argument or setting adds one.
|
||||
// **An event only wakes the session** (§4): what it says is read again from the verb that owns it.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WaitDefault is how long a wait lasts when no timeout is given, and WaitLongest the most it may last
|
||||
// (ADR 0367 §1): four minutes and ten.
|
||||
var (
|
||||
WaitDefault = 4 * time.Minute
|
||||
WaitLongest = 10 * time.Minute
|
||||
)
|
||||
|
||||
func waitTool() map[string]any {
|
||||
return map[string]any{"name": verbWait, "inputSchema": map[string]any{"type": "object", "properties": map[string]any{
|
||||
"subjects": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "one or more event families to wait on, by name, e.g. `mesh-knowledge.reviewed`; only " +
|
||||
"those the mesh granted this machine's tool runner — a refusal names them"},
|
||||
"filter": map[string]any{"type": "object",
|
||||
"description": "fields of the event's body that must equal the given values, e.g. {\"kind\": \"decision\", \"number\": 367} (optional)"},
|
||||
"timeout": map[string]any{"type": "number",
|
||||
"description": fmt.Sprintf("seconds to wait: %d when absent, %d at most", int(WaitDefault.Seconds()), int(WaitLongest.Seconds()))},
|
||||
}, "required": []string{"subjects"}},
|
||||
"description": "Wait for the first event of the named families that matches the filter, and answer it with its " +
|
||||
"family, body and time — or say it timed out. Call it in a loop from a background command, and after every " +
|
||||
"answer read the state from the verb that owns it: an event only wakes you, and one published between two " +
|
||||
"waits is not heard (novox/hq ADR 0367)."}
|
||||
}
|
||||
|
||||
// waitAnswer answers one mesh_wait call.
|
||||
func (s *Surface) waitAnswer(ctx context.Context, args map[string]any) map[string]any {
|
||||
names, err := waitNames(args["subjects"])
|
||||
if err != nil {
|
||||
return failure(err.Error())
|
||||
}
|
||||
timeout, err := waitTimeout(args["timeout"])
|
||||
if err != nil {
|
||||
return failure(err.Error())
|
||||
}
|
||||
filter, _ := args["filter"].(map[string]any)
|
||||
if args["filter"] != nil && filter == nil {
|
||||
return failure("mesh_wait's filter is an object of body fields to the values they must equal")
|
||||
}
|
||||
|
||||
issued := map[string]string{} // name → subject
|
||||
if m := s.conn.Membership(s.conn.Module()); m != nil {
|
||||
for _, w := range m.Waits {
|
||||
issued[w.Event] = w.Subject
|
||||
}
|
||||
}
|
||||
if len(issued) == 0 {
|
||||
return failure("this machine's tool runner was issued no event families to wait on: the mesh grants them from " +
|
||||
"the tool runner's manifest (novox/hq ADR 0367 §2), and its membership names none yet")
|
||||
}
|
||||
byName := make([]string, 0, len(issued))
|
||||
for name := range issued {
|
||||
byName = append(byName, name)
|
||||
}
|
||||
sort.Strings(byName)
|
||||
family := map[string]string{} // subject → name
|
||||
var subjects []string
|
||||
for _, name := range names {
|
||||
subject, granted := issued[name]
|
||||
if !granted {
|
||||
return failure(fmt.Sprintf("mesh_wait may not wait on %q: this machine's tool runner is granted %s alone. "+
|
||||
"A family is added only by a reviewed change to the tool runner's manifest (node-tools' module.json, "+
|
||||
"novox/hq ADR 0367 §6); no argument, call or setting adds one", name, strings.Join(byName, ", ")))
|
||||
}
|
||||
if _, twice := family[subject]; !twice {
|
||||
subjects = append(subjects, subject)
|
||||
}
|
||||
family[subject] = name
|
||||
}
|
||||
|
||||
heard, ok, err := s.conn.HearFirst(ctx, subjects, timeout, func(body json.RawMessage) bool { return bodyMatches(body, filter) })
|
||||
if err != nil {
|
||||
return failure("mesh_wait could not wait: " + err.Error())
|
||||
}
|
||||
if !ok {
|
||||
return answerText(map[string]any{"timed-out": true, "waited": timeout.String(), "subjects": names,
|
||||
"read": "nothing matching was heard; read the state once from the verb that owns it, then wait again"})
|
||||
}
|
||||
return answerText(map[string]any{
|
||||
"heard": map[string]any{"subject": family[heard.Subject], "body": heard.Body, "at": heard.At.Format(time.RFC3339Nano)},
|
||||
"read": "an event only wakes you: read the state from the verb that owns it before acting (novox/hq ADR 0367 §4)",
|
||||
})
|
||||
}
|
||||
|
||||
// waitNames is the families a call names: a list of names, or one name.
|
||||
func waitNames(given any) ([]string, error) {
|
||||
var names []string
|
||||
switch v := given.(type) {
|
||||
case string:
|
||||
names = []string{v}
|
||||
case []any:
|
||||
for _, x := range v {
|
||||
name, ok := x.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("mesh_wait's subjects are event names, such as mesh-knowledge.reviewed; %v is not one", x)
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
for i := range names {
|
||||
names[i] = strings.TrimSpace(names[i])
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return nil, fmt.Errorf("mesh_wait names at least one event family in `subjects`, such as mesh-knowledge.reviewed")
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// waitTimeout is the call's timeout: WaitDefault when absent, refused beyond WaitLongest or below a second.
|
||||
func waitTimeout(given any) (time.Duration, error) {
|
||||
if given == nil {
|
||||
return WaitDefault, nil
|
||||
}
|
||||
longest := int(WaitLongest.Seconds())
|
||||
seconds, ok := given.(float64)
|
||||
if !ok {
|
||||
if n, isInt := given.(int); isInt {
|
||||
seconds, ok = float64(n), true
|
||||
}
|
||||
}
|
||||
if !ok || math.IsNaN(seconds) || seconds < 1 || seconds > float64(longest) {
|
||||
return 0, fmt.Errorf("mesh_wait's timeout is a number of seconds from 1 to %d (novox/hq ADR 0367 §1), not %v", longest, given)
|
||||
}
|
||||
return time.Duration(seconds * float64(time.Second)), nil
|
||||
}
|
||||
|
||||
// bodyMatches says whether every field the filter names equals, in the event's body, the value given.
|
||||
func bodyMatches(body json.RawMessage, filter map[string]any) bool {
|
||||
if len(filter) == 0 {
|
||||
return true
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(body, &fields) != nil {
|
||||
return false
|
||||
}
|
||||
for k, want := range filter {
|
||||
got, has := fields[k]
|
||||
if !has {
|
||||
return false
|
||||
}
|
||||
wanted, _ := json.Marshal(want)
|
||||
if !sameJSON(got, wanted) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// sameJSON compares two JSON values as values: 367 and 367.0 are one number, key order does not matter.
|
||||
func sameJSON(a, b []byte) bool {
|
||||
var x, y any
|
||||
da, db := json.NewDecoder(bytes.NewReader(a)), json.NewDecoder(bytes.NewReader(b))
|
||||
if da.Decode(&x) != nil || db.Decode(&y) != nil {
|
||||
return false
|
||||
}
|
||||
ja, _ := json.Marshal(x)
|
||||
jb, _ := json.Marshal(y)
|
||||
return bytes.Equal(ja, jb)
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// waitingRuntime is the tool runner on a machine, issued the families ADR 0367 §5 names, as the controller
|
||||
// composes them from its manifest: the surface over its connection, and the mesh to publish on.
|
||||
func waitingRuntime(t *testing.T) (*Surface, *mt.Mesh) {
|
||||
t.Helper()
|
||||
mesh := mt.New(t)
|
||||
m := mt.MembershipOf("node-tools", "desk", false, nil)
|
||||
m.Waits = []bus.WaitIssued{
|
||||
{Event: "mesh-knowledge.reviewed", Subject: "mesh.mod.mesh-knowledge.event.reviewed"},
|
||||
{Event: "operator-channel.decided.claude-code", Subject: "mesh.seat.operator-channel.event.decided.claude-code"},
|
||||
}
|
||||
mesh.Issue(t, m)
|
||||
conn := connect(t, "node-tools", "desk")
|
||||
mt.Until(t, func() error {
|
||||
if got := conn.Membership("node-tools"); got == nil || len(got.Waits) == 0 {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return NewSurface(conn, busSource{conn}, "desk.node-tools"), mesh
|
||||
}
|
||||
|
||||
// callWait is one tools/call of mesh_wait, answered as an agent host reads it.
|
||||
func callWait(s *Surface, args map[string]any) (text string, isError bool) {
|
||||
raw, _ := json.Marshal(map[string]any{"name": verbWait, "arguments": args})
|
||||
r := s.Handle(Request{JSONRPC: "2.0", ID: json.RawMessage("1"), Method: "tools/call", Params: raw})
|
||||
result, _ := r.Result.(map[string]any)
|
||||
content, _ := result["content"].([]map[string]any)
|
||||
var parts []string
|
||||
for _, c := range content {
|
||||
parts = append(parts, c["text"].(string))
|
||||
}
|
||||
isErr, _ := result["isError"].(bool)
|
||||
return strings.Join(parts, "\n"), isErr
|
||||
}
|
||||
|
||||
// novox/hq ADR 0367 §1: mesh_wait answers the first event that matches — by its family and its filter — with
|
||||
// its subject, body and time, within a second of its publication, and ignores the others.
|
||||
func TestMeshWaitAnswersTheFirstMatchingEvent(t *testing.T) {
|
||||
s, mesh := waitingRuntime(t)
|
||||
type answer struct {
|
||||
text string
|
||||
isErr bool
|
||||
at time.Time
|
||||
}
|
||||
done := make(chan answer, 1)
|
||||
go func() {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"},
|
||||
"filter": map[string]any{"kind": "decision", "number": 367}, "timeout": 20})
|
||||
done <- answer{text, isErr, time.Now()}
|
||||
}()
|
||||
// Once it listens: another record's verdict, another family's event, then the one it waits for.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
mesh.Emit(t, "mesh.mod.mesh-knowledge.event.reviewed", map[string]any{"kind": "decision", "number": 366, "what": "approved"})
|
||||
mesh.Emit(t, "mesh.mod.mesh-knowledge.event.proposed", map[string]any{"kind": "decision", "number": 367})
|
||||
published := time.Now()
|
||||
mesh.Emit(t, "mesh.mod.mesh-knowledge.event.reviewed", map[string]any{"kind": "decision", "number": 367, "what": "approved"})
|
||||
var got answer
|
||||
select {
|
||||
case got = <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("mesh_wait did not answer the matching event")
|
||||
}
|
||||
if got.isErr {
|
||||
t.Fatalf("mesh_wait failed: %s", got.text)
|
||||
}
|
||||
if got.at.Sub(published) > time.Second {
|
||||
t.Errorf("answered %s after the event was published, want within a second", got.at.Sub(published))
|
||||
}
|
||||
var heard struct {
|
||||
Heard struct {
|
||||
Subject string `json:"subject"`
|
||||
Body map[string]any `json:"body"`
|
||||
At time.Time `json:"at"`
|
||||
} `json:"heard"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(got.text), &heard); err != nil {
|
||||
t.Fatalf("not an answer: %s", got.text)
|
||||
}
|
||||
if heard.Heard.Subject != "mesh-knowledge.reviewed" || heard.Heard.Body["number"] != float64(367) || heard.Heard.At.IsZero() {
|
||||
t.Errorf("answered %+v, want record 367's verdict with its time", heard.Heard)
|
||||
}
|
||||
// And the subject it is granted on the asker's own `decided`, by the name a session asks for.
|
||||
go func() {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": "operator-channel.decided.claude-code", "timeout": 20})
|
||||
done <- answer{text, isErr, time.Now()}
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
// A seat's event, said by the router on core NATS here: no stream of the test's bus carries the seat.
|
||||
if err := connect(t, "messenger", "").Say("mesh.seat.operator-channel.event.decided.claude-code", []byte(`{"ask":"a1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case got = <-done:
|
||||
if got.isErr || !strings.Contains(got.text, `"subject": "operator-channel.decided.claude-code"`) {
|
||||
t.Errorf("the asker's decided: %s", got.text)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("mesh_wait did not answer the asker's decided")
|
||||
}
|
||||
}
|
||||
|
||||
// It says it timed out at its timeout, and leaves no subscription behind (ADR 0367 §1, §2).
|
||||
func TestMeshWaitSaysItTimedOutAndKeepsNothing(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
before := s.conn.Subscriptions()
|
||||
start := time.Now()
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"}, "timeout": 1})
|
||||
if took := time.Since(start); took < time.Second || took > 3*time.Second {
|
||||
t.Errorf("a one-second wait took %s", took)
|
||||
}
|
||||
if isErr || !strings.Contains(text, `"timed-out": true`) {
|
||||
t.Errorf("a wait that heard nothing: %s", text)
|
||||
}
|
||||
if after := s.conn.Subscriptions(); after != before {
|
||||
t.Errorf("%d subscriptions before the wait, %d after it", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// Its timeout is four minutes when none is given and ten at most: a longer one, or none at all, is refused
|
||||
// with the bound, never shortened in silence.
|
||||
func TestMeshWaitsTimeoutIsBounded(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
if WaitDefault != 4*time.Minute || WaitLongest != 10*time.Minute {
|
||||
t.Errorf("the bounds are %s and %s, want 4m and 10m (ADR 0367 §1)", WaitDefault, WaitLongest)
|
||||
}
|
||||
for _, timeout := range []any{601, 0, -5, "soon"} {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"}, "timeout": timeout})
|
||||
if !isErr || !strings.Contains(text, "600") {
|
||||
t.Errorf("a timeout of %v: %s", timeout, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0367 §2 and §6: an event outside the granted families is refused, naming the families and how
|
||||
// one is added; no argument — a wildcard, another asker's `decided`, a subject spelled out — widens it.
|
||||
func TestMeshWaitRefusesAnEventOutsideTheGrantedFamilies(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
for _, subjects := range []any{
|
||||
[]any{"mesh-knowledge.proposed"},
|
||||
[]any{"mesh-knowledge.reviewed", "gitea.forge.push"},
|
||||
[]any{"mesh-knowledge.*"},
|
||||
[]any{"operator-channel.decided.shop"},
|
||||
[]any{"operator-channel.decided.*"},
|
||||
[]any{"mesh.mod.mesh-knowledge.event.reviewed"},
|
||||
[]any{">"},
|
||||
[]any{},
|
||||
} {
|
||||
text, isErr := callWait(s, map[string]any{"subjects": subjects, "timeout": 1})
|
||||
if !isErr {
|
||||
t.Errorf("%v was waited on: %s", subjects, text)
|
||||
continue
|
||||
}
|
||||
if len(subjects.([]any)) > 0 && (!strings.Contains(text, "mesh-knowledge.reviewed") || !strings.Contains(text, "manifest")) {
|
||||
t.Errorf("%v refused without naming the families and how one is added: %s", subjects, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A tool runner issued no families waits on nothing, and says so.
|
||||
func TestMeshWaitWithNoFamiliesIssuedWaitsOnNothing(t *testing.T) {
|
||||
mt.New(t)
|
||||
s := NewSurface(connect(t, "node-tools", "desk"), nil, "desk.node-tools")
|
||||
text, isErr := callWait(s, map[string]any{"subjects": []any{"mesh-knowledge.reviewed"}, "timeout": 1})
|
||||
if !isErr || !strings.Contains(text, "no event families") {
|
||||
t.Errorf("a runtime issued no families: %s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// The mesh MCP server announces seven tools (ADR 0367 v13): ADR 0195's five, mesh_runtimes and mesh_wait, its timeout bounded.
|
||||
func TestMeshWaitIsAnnouncedAmongSeven(t *testing.T) {
|
||||
var names []string
|
||||
var wait map[string]any
|
||||
for _, tool := range discovery() {
|
||||
names = append(names, tool["name"].(string))
|
||||
if tool["name"] == verbWait {
|
||||
wait = tool
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
want := []string{"mesh_call", "mesh_describe", "mesh_first_view_again", "mesh_first_views", "mesh_machine", "mesh_overview", "mesh_runtimes", "mesh_search", "mesh_wait"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("announced %v, want %v", names, want)
|
||||
}
|
||||
if wait == nil || !isDiscovery(verbWait) {
|
||||
t.Fatal("mesh_wait is not announced")
|
||||
}
|
||||
schema, _ := json.Marshal(wait["inputSchema"])
|
||||
for _, word := range []string{`"subjects"`, `"filter"`, `"timeout"`} {
|
||||
if !strings.Contains(string(schema), word) {
|
||||
t.Errorf("mesh_wait's schema has no %s: %s", word, schema)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A client that hangs up or cancels ends its wait at once: the subscription is closed and nothing is left
|
||||
// waiting for the timeout (the review of issue 745). Over the console's own HTTP, as an agent host calls it.
|
||||
func TestMeshWaitEndsWhenItsClientHangsUp(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
up, err := Serve(s, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = up.Close() })
|
||||
before := s.conn.Subscriptions()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
body := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"mesh_wait",` +
|
||||
`"arguments":{"subjects":["mesh-knowledge.reviewed"],"timeout":600}}}`
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "http://"+up.Address+"/mcp", strings.NewReader(body))
|
||||
req.Header.Set("content-type", "application/json")
|
||||
go func() { _, _ = http.DefaultClient.Do(req) }()
|
||||
mt.Until(t, func() error {
|
||||
if s.conn.Subscriptions() <= before {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
cancel()
|
||||
start := time.Now()
|
||||
for s.conn.Subscriptions() > before {
|
||||
if time.Since(start) > 3*time.Second {
|
||||
t.Fatalf("%d subscriptions 3s after the client hung up, %d before the wait", s.conn.Subscriptions(), before)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// The tool runner stopping ends every wait in flight at once (the second review of issue 745): Close cancels
|
||||
// the requests' context rather than waiting up to ten minutes for a wait to time out.
|
||||
func TestMeshWaitEndsWhenTheConsoleCloses(t *testing.T) {
|
||||
s, _ := waitingRuntime(t)
|
||||
up, err := Serve(s, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := s.conn.Subscriptions()
|
||||
body := `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"mesh_wait",` +
|
||||
`"arguments":{"subjects":["mesh-knowledge.reviewed"],"timeout":600}}}`
|
||||
go func() { _, _ = http.Post("http://"+up.Address+"/mcp", "application/json", strings.NewReader(body)) }()
|
||||
mt.Until(t, func() error {
|
||||
if s.conn.Subscriptions() <= before {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
closed := make(chan error, 1)
|
||||
start := time.Now()
|
||||
go func() { closed <- up.Close() }()
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("closing the console waited on a mesh_wait in flight")
|
||||
}
|
||||
if took := time.Since(start); took > 3*time.Second {
|
||||
t.Errorf("closing took %s", took)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
if s.conn.Subscriptions() > before {
|
||||
return os.ErrExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// Package firstview is the tool runner's picture of the four states it reads of the indexer and the assigner: the assigner's
|
||||
// assignments and seat-holders, the indexer's manifests and modules (novox/hq ADR 0615 §1, §3; ADR 0621 §2; issue 1966 TLS-5).
|
||||
//
|
||||
// **It never reads the owners' state.** At start it asks each owner's first-view tool until answered (the owner answers in a
|
||||
// runtime of its own, never through this one: ADR 0621 §1), then follows the owner's views family, one event per changed key;
|
||||
// a gap in the revisions, or an owner whose state was made again, asks the first view once more, said in the log. Past two
|
||||
// minutes without an answer it writes a finding in its own state `findings`, which the condition keeper raises and clears.
|
||||
package firstview
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
conditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// The owners and their first-view tools (the tools' addresses are `<owner>.<tool>`).
|
||||
const (
|
||||
Indexer = "mesh-delivery-indexer"
|
||||
Assigner = "mesh-delivery-assigner"
|
||||
)
|
||||
|
||||
// The names the console reads the four states by (`<owner>.<state>`), as the manifest's `consumes` names their views
|
||||
// families (no seat's client names a holder, novox/hq ADR 0625 §4; TestTheFirstViewsAreTheManifestsViews).
|
||||
const (
|
||||
AssignmentsRead = Assigner + ".assignments"
|
||||
SeatHoldersRead = Assigner + ".seat-holders"
|
||||
ModulesRead = Indexer + ".modules"
|
||||
ManifestsRead = Indexer + ".manifests"
|
||||
)
|
||||
|
||||
// Spec is one first view the runner holds.
|
||||
type Spec struct{ Owner, Tool, State string }
|
||||
|
||||
// Specs are the four.
|
||||
var Specs = []Spec{
|
||||
{Assigner, "assignments", "assignments"},
|
||||
{Assigner, "seat-holders", "seat-holders"},
|
||||
{Indexer, "manifests", "manifests"},
|
||||
{Indexer, "modules", "modules"},
|
||||
}
|
||||
|
||||
// Name is the name a state is read by, `<owner>.<state>`: AssignmentsRead, SeatHoldersRead and the indexer's.
|
||||
func (s Spec) Name() string { return s.Owner + "." + s.State }
|
||||
|
||||
// Views is the four followers.
|
||||
type Views struct {
|
||||
logf func(string, ...any)
|
||||
followers map[string]*views.Follower // by Spec.Name()
|
||||
byState map[string]*views.Follower // by the state an event names
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
deps Deps
|
||||
}
|
||||
|
||||
// FindingsState is the runner's own state the findings are written in.
|
||||
const FindingsState = "findings"
|
||||
|
||||
// Deps is what the views reach the bus through: Start makes them from the runner's connection, a test from fakes.
|
||||
type Deps struct {
|
||||
// Module is the runner's own name, the observer of its findings.
|
||||
Module string
|
||||
// Ask calls an owner's tool, `<owner>.<tool>`.
|
||||
Ask func(address string, body any) (json.RawMessage, error)
|
||||
// Consume hands each event of the owners' views families to deliver, until the stop is called.
|
||||
Consume func(deliver func(bus.Envelope) error) (stop func(), err error)
|
||||
// PutFinding and DeleteFinding keep and remove a finding in the runner's own state.
|
||||
PutFinding func(key string, value json.RawMessage) error
|
||||
DeleteFinding func(key string) error
|
||||
// Bound and Wait are the follower's (zero: two minutes, five seconds).
|
||||
Bound, Wait time.Duration
|
||||
}
|
||||
|
||||
// Start asks the four first views, each until answered, and follows the owners' views families on the runner's own
|
||||
// consumers. It returns at once: the picture fills as the owners answer.
|
||||
func Start(parent context.Context, conn *bus.Conn, logf func(string, ...any)) (*Views, func()) {
|
||||
return StartWith(parent, DepsOf(conn), logf)
|
||||
}
|
||||
|
||||
// DepsOf is the views' reach through the runner's own connection: its tool calls, its consumer and its findings state.
|
||||
func DepsOf(conn *bus.Conn) Deps {
|
||||
me := conn.Module()
|
||||
return Deps{Module: me,
|
||||
Ask: func(address string, body any) (json.RawMessage, error) {
|
||||
got, err := conn.Ask(address, body, "")
|
||||
return got.Result, err
|
||||
},
|
||||
Consume: func(deliver func(bus.Envelope) error) (func(), error) { return conn.ConsumeAs(me, deliver) },
|
||||
PutFinding: func(key string, value json.RawMessage) error {
|
||||
_, err := conn.StatePut(me, FindingsState, key, value)
|
||||
return err
|
||||
},
|
||||
DeleteFinding: func(key string) error { return conn.StateDelete(me, FindingsState, key) },
|
||||
}
|
||||
}
|
||||
|
||||
// StartWith is Start over deps.
|
||||
func StartWith(parent context.Context, d Deps, logf func(string, ...any)) (*Views, func()) {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
v := &Views{logf: logf, ctx: ctx, cancel: cancel, followers: map[string]*views.Follower{}, byState: map[string]*views.Follower{}, deps: d}
|
||||
for _, s := range Specs {
|
||||
s := s
|
||||
call := func(tool string, body any) (json.RawMessage, error) { return d.Ask(s.Owner+"."+tool, body) }
|
||||
f := &views.Follower{Owner: s.Owner, Tool: s.Tool, State: s.State, Ask: views.AskVia(call, s.Tool), Logf: logf,
|
||||
Bound: d.Bound, Wait: d.Wait,
|
||||
Raise: func(st views.Status) { v.finding(st, true) },
|
||||
Clear: func(st views.Status) { v.finding(st, false) }}
|
||||
v.followers[s.Name()] = f
|
||||
v.byState[s.State] = f
|
||||
v.wg.Add(1)
|
||||
go func() { defer v.wg.Done(); _ = f.Start(ctx) }()
|
||||
}
|
||||
stop, err := d.Consume(v.hear)
|
||||
if err != nil {
|
||||
logf("[mesh-tools] the owners' views are not followed (%v): the pictures are the first views alone until the runner starts again", err)
|
||||
stop = func() {}
|
||||
}
|
||||
return v, func() { stop(); cancel(); v.wg.Wait() }
|
||||
}
|
||||
|
||||
// hear applies one views event; a gap asks the first view once more.
|
||||
func (v *Views) hear(env bus.Envelope) error {
|
||||
ev, err := views.DecodeEvent(env.Body)
|
||||
if err != nil {
|
||||
return nil // not a views event: nothing to hold, nothing to retry
|
||||
}
|
||||
f := v.byState[ev.State]
|
||||
if f == nil || f.Owner != ev.Owner {
|
||||
return nil
|
||||
}
|
||||
if gap, why := f.Apply(ev); gap {
|
||||
v.logf("[mesh-tools] a gap in %s's %s (%s): its first view %s is asked once more", f.Owner, f.State, why, f.Tool)
|
||||
go func() { _ = f.Again(v.ctx, "a gap: "+why) }()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// finding writes or removes the runner's finding of a first view unanswered past its bound, in its own state.
|
||||
func (v *Views) finding(st views.Status, raised bool) {
|
||||
key, value, err := conditionOf(views.UnansweredFinding(v.deps.Module, st, time.Now()))
|
||||
if err != nil {
|
||||
v.logf("[mesh-tools] the finding of the first view %s is not made: %v", st.Tool, err)
|
||||
return
|
||||
}
|
||||
if raised {
|
||||
err = v.deps.PutFinding(key, value)
|
||||
} else {
|
||||
err = v.deps.DeleteFinding(key)
|
||||
}
|
||||
if err != nil {
|
||||
v.logf("[mesh-tools] the finding of the first view %s is not kept: %v", st.Tool, err)
|
||||
}
|
||||
}
|
||||
|
||||
// State is the picture of the state spec names (`<owner>.<state>`), every current value by key; an error says the owner has
|
||||
// not answered yet, so nothing is claimed of it.
|
||||
func (v *Views) State(name string) (map[string][]byte, error) {
|
||||
f := v.followers[name]
|
||||
if f == nil {
|
||||
return nil, fmt.Errorf("the tool runner holds no first view of %s", name)
|
||||
}
|
||||
select {
|
||||
case <-f.Ready():
|
||||
default:
|
||||
return nil, fmt.Errorf("%s's first view %s has not been answered yet", f.Owner, f.Tool)
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for k, raw := range f.Values() {
|
||||
out[k] = raw
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Entry is one value of the picture, nil when the owner holds none.
|
||||
func (v *Views) Entry(name, key string) ([]byte, error) {
|
||||
all, err := v.State(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return all[key], nil
|
||||
}
|
||||
|
||||
// Status is each first view as it stands, by `<owner>.<state>`, sorted.
|
||||
func (v *Views) Status() []views.Status {
|
||||
var out []views.Status
|
||||
for _, f := range v.followers {
|
||||
out = append(out, f.Status())
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Owner+out[i].State < out[j].Owner+out[j].State })
|
||||
return out
|
||||
}
|
||||
|
||||
// Again asks every first view once more, by hand, for why.
|
||||
func (v *Views) Again(ctx context.Context, why string) []error {
|
||||
var errs []error
|
||||
for _, f := range v.followers {
|
||||
if err := f.Again(ctx, why); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// conditionOf is a finding in the layout of the seat mesh-conditions' state findings (its contract, ADR 0634 §1), about a
|
||||
// module, a warning.
|
||||
func conditionOf(f views.Finding) (string, []byte, error) {
|
||||
key := conditions.Key(scopeModule, f.ID, f.Kind)
|
||||
v := condseat.FindingsByKeyValue{V: 1, Key: key, Kind: f.Kind, Observer: f.Observer, Severity: conditions.Warning,
|
||||
Subject: condseat.Subject{Scope: scopeModule, ID: f.ID}, Summary: f.Summary, Headline: f.Headline,
|
||||
Explanation: f.Explanation, Source: f.Observer + "." + f.Kind, At: f.At.UTC(),
|
||||
Evidence: []condseat.Evidence{{At: f.At.UTC(), Said: f.Summary}}}
|
||||
if err := conditions.CheckFinding(v); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
raw, err := json.Marshal(v)
|
||||
return key, raw, err
|
||||
}
|
||||
|
||||
// scopeModule is the scope of a finding about a module.
|
||||
const scopeModule = "module"
|
||||
@@ -0,0 +1,158 @@
|
||||
//go:build integration
|
||||
|
||||
package firstview
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// issue 1966 TLS-5 [i] (ADR 0615 §2, §4, §6): against the real bus, with the owners down and then up, the runner's own
|
||||
// consumers on the owners' own views streams carry the manifest's four `consumes`: the first views are asked until answered,
|
||||
// the unanswered ones are findings that the answer clears, and each owner's next event reaches its picture.
|
||||
func TestTheRunnersConsumerCarriesTheOwnersViewsFamiliesWhenTheOwnersAreDownThenUp(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
url := mt.URL(t)
|
||||
|
||||
// The owners publish their views on their own streams (views.Emit: `mesh.stream.<owner>.event.views.<state>`, on
|
||||
// assigner-views and indexer-views), never on EVENTS. The consumers on them are the mesh's, one filter per family the
|
||||
// manifest's `consumes` names, and the runner's membership names them as the mesh issues it (ADR 0363 §8).
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var manifest struct {
|
||||
Consumes []string `json:"consumes"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
streamOf := map[string]string{Assigner: catalogue.StreamBusName("assigner-views"), Indexer: catalogue.StreamBusName("indexer-views")}
|
||||
filters := map[string][]string{}
|
||||
families := 0
|
||||
for _, c := range manifest.Consumes {
|
||||
emitter, event, isView := strings.Cut(c, ".views.")
|
||||
if !isView { // the seat mesh-catalog's events: the seat list's, on EVENTS (cmd/node-tools/seatlist.go)
|
||||
continue
|
||||
}
|
||||
stream := streamOf[emitter]
|
||||
if stream == "" {
|
||||
t.Fatalf("the manifest consumes %s, whose views stream the test does not know", c)
|
||||
}
|
||||
filters[stream] = append(filters[stream], process.ModuleStreamSubject(emitter, views.EventPrefix+event))
|
||||
families++
|
||||
}
|
||||
if families != len(Specs) {
|
||||
t.Fatalf("the manifest consumes %d families, the runner holds %d first views", families, len(Specs))
|
||||
}
|
||||
runner := mt.MembershipOf("node-tools", "anchor", false, nil)
|
||||
for _, owner := range []string{Assigner, Indexer} {
|
||||
stream := streamOf[owner]
|
||||
mesh.ModuleStream(t, owner, stream)
|
||||
mesh.ConsumerOn(t, stream, "anchor", "node-tools", filters[stream], 2*time.Second)
|
||||
runner.Hears = append(runner.Hears, bus.HeardIssued{Stream: stream, Consumer: "anchor_node-tools"})
|
||||
}
|
||||
mesh.Consumer(t, "anchor", "node-tools", []string{"mesh.mod.mesh-knowledge.event.reviewed"}, 2*time.Second) // its `waits`, on EVENTS
|
||||
mesh.Issue(t, runner)
|
||||
|
||||
conn, err := bus.Connect(bus.Credential{URL: url, Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("node-tools")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("node-tools") == nil {
|
||||
return errFindings(-1)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
var mu sync.Mutex
|
||||
findings := map[string]bool{}
|
||||
deps := DepsOf(conn)
|
||||
deps.Bound, deps.Wait = 100*time.Millisecond, 20*time.Millisecond
|
||||
deps.PutFinding = func(key string, _ json.RawMessage) error { mu.Lock(); findings[key] = true; mu.Unlock(); return nil }
|
||||
deps.DeleteFinding = func(key string) error { mu.Lock(); delete(findings, key); mu.Unlock(); return nil }
|
||||
v, stop := StartWith(context.Background(), deps, func(string, ...any) {})
|
||||
t.Cleanup(stop)
|
||||
|
||||
// The owners are down: no tool answers, nothing is claimed, and past the bound each view is a finding.
|
||||
if _, err := v.State(Assigner + ".assignments"); err == nil {
|
||||
t.Fatal("a state its owner has not answered is claimed")
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(findings) != len(Specs) {
|
||||
return errFindings(len(findings))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// The owners come up and answer their first views.
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
for _, s := range Specs {
|
||||
s := s
|
||||
if _, err := nc.Subscribe("mesh.mod."+s.Owner+".tool."+s.Tool, func(m *nats.Msg) {
|
||||
answer, _ := json.Marshal(views.Answer{V: 1, Owner: s.Owner, State: s.State, Revision: 4, Created: time.Unix(1000, 0).UTC(),
|
||||
Values: map[string]json.RawMessage{"first": json.RawMessage(`{"n":1}`)}})
|
||||
body, _ := json.Marshal(map[string]json.RawMessage{"result": answer})
|
||||
_ = m.Respond(body)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_ = nc.Flush()
|
||||
for _, s := range Specs {
|
||||
name := s.Name()
|
||||
mt.Until(t, func() error { _, err := v.Entry(name, "first"); return err })
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(findings) != 0 {
|
||||
return errFindings(len(findings))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Each owner's next event arrives on the runner's consumer: all four families are bound.
|
||||
for _, s := range Specs {
|
||||
ev, _ := json.Marshal(views.Event{V: 1, Owner: s.Owner, State: s.State, Key: "next", Revision: 5, Created: time.Unix(1000, 0).UTC(),
|
||||
Value: json.RawMessage(`{"n":2}`)})
|
||||
mesh.Emit(t, process.ModuleStreamSubject(s.Owner, views.EventPrefix+s.State), json.RawMessage(ev))
|
||||
}
|
||||
for _, s := range Specs {
|
||||
name := s.Name()
|
||||
mt.Until(t, func() error {
|
||||
got, err := v.Entry(name, "next")
|
||||
if err != nil || string(got) != `{"n":2}` {
|
||||
return errFindings(-1)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type errFindings int
|
||||
|
||||
func (e errFindings) Error() string { return "not yet" }
|
||||
@@ -0,0 +1,197 @@
|
||||
package firstview
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// owners is the four owners' first-view tools as a test answers them: a state's values at a revision, or nothing while down.
|
||||
type owners struct {
|
||||
mu sync.Mutex
|
||||
down bool
|
||||
state map[string]map[string]string // tool → key → value
|
||||
rev map[string]uint64
|
||||
asked map[string]int
|
||||
deliver func(bus.Envelope) error
|
||||
finds map[string]bool
|
||||
}
|
||||
|
||||
func newOwners() *owners {
|
||||
return &owners{state: map[string]map[string]string{"assignments": {"anchor": `{"p":1}`}, "seat-holders": {"seat": `{"h":1}`},
|
||||
"manifests": {"d1": `{"m":1}`}, "modules": {"web": `{"w":1}`}},
|
||||
rev: map[string]uint64{"assignments": 4, "seat-holders": 2, "manifests": 7, "modules": 3}, asked: map[string]int{}, finds: map[string]bool{}}
|
||||
}
|
||||
|
||||
func (o *owners) ask(address string, _ any) (json.RawMessage, error) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
tool := address[strings.LastIndex(address, ".")+1:]
|
||||
o.asked[tool]++
|
||||
if o.down {
|
||||
return nil, errors.New("503 no responders")
|
||||
}
|
||||
vals := map[string]json.RawMessage{}
|
||||
for k, v := range o.state[tool] {
|
||||
vals[k] = json.RawMessage(v)
|
||||
}
|
||||
return json.Marshal(views.Answer{V: 1, Owner: "o", State: tool, Revision: o.rev[tool], Created: time.Unix(1000, 0).UTC(), Values: vals})
|
||||
}
|
||||
|
||||
func (o *owners) say(t *testing.T, state, key, value string, revision uint64, gap bool) {
|
||||
t.Helper()
|
||||
ev := views.Event{V: 1, Owner: owner(state), State: state, Key: key, Revision: revision, Created: time.Unix(1000, 0).UTC(), Gap: gap}
|
||||
if value != "" {
|
||||
ev.Value = json.RawMessage(value)
|
||||
}
|
||||
raw, _ := json.Marshal(ev)
|
||||
if err := o.deliver(bus.Envelope{Key: ev.Owner + ".views." + state, Body: raw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func owner(state string) string {
|
||||
for _, s := range Specs {
|
||||
if s.State == state {
|
||||
return s.Owner
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func eventually(t *testing.T, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
for end := time.Now().Add(3 * time.Second); !ok(); time.Sleep(5 * time.Millisecond) {
|
||||
if time.Now().After(end) {
|
||||
t.Fatal(what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// issue 1966 TLS-5 (ADR 0615 §2, §4, §5, §6): the runner holds the four states from the owners' first views and their
|
||||
// events alone; nothing is claimed of a state not yet answered; a gap asks the first view once more; an owner that stays down
|
||||
// past the bound is a finding, cleared by the answer; the status and the hand ask show them.
|
||||
func TestTheRunnerHoldsTheFourStatesFromTheOwnersFirstViewsAndTheirEvents(t *testing.T) {
|
||||
o := newOwners()
|
||||
o.down = true
|
||||
var logs []string
|
||||
var lmu sync.Mutex
|
||||
deps := Deps{Module: "node-tools", Ask: o.ask, Bound: 30 * time.Millisecond, Wait: 10 * time.Millisecond,
|
||||
Consume: func(deliver func(bus.Envelope) error) (func(), error) { o.deliver = deliver; return func() {}, nil },
|
||||
PutFinding: func(key string, _ json.RawMessage) error {
|
||||
o.mu.Lock()
|
||||
o.finds[key] = true
|
||||
o.mu.Unlock()
|
||||
return nil
|
||||
},
|
||||
DeleteFinding: func(key string) error {
|
||||
o.mu.Lock()
|
||||
delete(o.finds, key)
|
||||
o.mu.Unlock()
|
||||
return nil
|
||||
}}
|
||||
v, stop := StartWith(context.Background(), deps, func(f string, a ...any) { lmu.Lock(); logs = append(logs, f); lmu.Unlock() })
|
||||
defer stop()
|
||||
|
||||
// Down: nothing is claimed, and past the bound each unanswered view is a finding.
|
||||
if _, err := v.State("mesh-delivery-assigner.assignments"); err == nil || !strings.Contains(err.Error(), "not been answered") {
|
||||
t.Fatalf("a state its owner has not answered is claimed: %v", err)
|
||||
}
|
||||
eventually(t, "no finding past the bound", func() bool { o.mu.Lock(); defer o.mu.Unlock(); return len(o.finds) == 4 })
|
||||
|
||||
// Up: answered, the findings are cleared, and the pictures hold the owners' values.
|
||||
o.mu.Lock()
|
||||
o.down = false
|
||||
o.mu.Unlock()
|
||||
eventually(t, "not answered", func() bool { _, err := v.State("mesh-delivery-indexer.manifests"); return err == nil })
|
||||
eventually(t, "findings not cleared", func() bool { o.mu.Lock(); defer o.mu.Unlock(); return len(o.finds) == 0 })
|
||||
if got, _ := v.Entry("mesh-delivery-indexer.modules", "web"); string(got) != `{"w":1}` {
|
||||
t.Errorf("the modules picture holds %q", got)
|
||||
}
|
||||
if got, _ := v.Entry("mesh-delivery-indexer.modules", "none"); got != nil {
|
||||
t.Errorf("an absent key is %q", got)
|
||||
}
|
||||
|
||||
// Events: the next revision is applied; an old one passed over; a deletion removes; none of it asks again.
|
||||
asked := func(tool string) int { o.mu.Lock(); defer o.mu.Unlock(); return o.asked[tool] }
|
||||
before := asked("assignments")
|
||||
o.say(t, "assignments", "laptop", `{"p":2}`, 5, false)
|
||||
o.say(t, "assignments", "laptop", `{"p":0}`, 5, false)
|
||||
o.say(t, "assignments", "anchor", "", 6, false)
|
||||
// (a deleted mark carries no value: Deleted is the encoding's, set below)
|
||||
if got, _ := v.Entry("mesh-delivery-assigner.assignments", "laptop"); string(got) != `{"p":2}` {
|
||||
t.Errorf("the next revision is not applied: %q", got)
|
||||
}
|
||||
if n := asked("assignments"); n != before {
|
||||
t.Errorf("an event in order asked the first view again (%d asks, was %d)", n, before)
|
||||
}
|
||||
|
||||
// A gap (revision 9 where 7 was next) asks the first view once more, said in the log.
|
||||
o.say(t, "assignments", "x", `{"p":9}`, 9, false)
|
||||
eventually(t, "a gap did not ask the first view again", func() bool { return asked("assignments") > before })
|
||||
|
||||
// The status says owner, tool, revision and the asks; the hand ask asks every view again.
|
||||
var assignments views.Status
|
||||
for _, s := range v.Status() {
|
||||
if s.State == "assignments" {
|
||||
assignments = s
|
||||
}
|
||||
}
|
||||
if assignments.Owner != "mesh-delivery-assigner" || assignments.Tool != "assignments" || assignments.Asks < 2 || assignments.Answered.IsZero() {
|
||||
t.Errorf("the status of assignments is %+v", assignments)
|
||||
}
|
||||
manifests := asked("manifests")
|
||||
if errs := v.Again(context.Background(), "doubted"); len(errs) != 0 {
|
||||
t.Fatalf("the hand ask failed: %v", errs)
|
||||
}
|
||||
if asked("manifests") != manifests+1 {
|
||||
t.Errorf("the hand ask did not ask the manifests again")
|
||||
}
|
||||
for _, st := range v.Status() {
|
||||
if st.State == "manifests" && st.ByHand != 1 {
|
||||
t.Errorf("the hand ask is not counted: %+v", st)
|
||||
}
|
||||
}
|
||||
lmu.Lock()
|
||||
defer lmu.Unlock()
|
||||
if !strings.Contains(strings.Join(logs, "\n"), "a gap in") {
|
||||
t.Errorf("the gap was not said in the log: %v", logs)
|
||||
}
|
||||
}
|
||||
|
||||
// The four first views are the manifest's `consumes` families, and the console reads each by the name here.
|
||||
func TestTheFirstViewsAreTheManifestsViews(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Consumes []string `json:"consumes"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]bool{AssignmentsRead: true, SeatHoldersRead: true, ModulesRead: true, ManifestsRead: true}
|
||||
for _, s := range Specs {
|
||||
if !want[s.Name()] {
|
||||
t.Errorf("the first view %s is read by no name here", s.Name())
|
||||
}
|
||||
delete(want, s.Name())
|
||||
if view := s.Owner + ".views." + s.Tool; !slices.Contains(m.Consumes, view) {
|
||||
t.Errorf("the first view %s has no family in the manifest's consumes %v", view, m.Consumes)
|
||||
}
|
||||
}
|
||||
if len(want) > 0 {
|
||||
t.Errorf("no first view reads %v", want)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,12 @@ package launch
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type lostWrite struct{}
|
||||
@@ -28,3 +33,84 @@ func TestALostWriteIsAnsweredWithItsOwnCodeAndNothingElseIs(t *testing.T) {
|
||||
t.Errorf("a plain refusal is answered %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A bundle that answers the handshake and an empty tool list, and notes each start and its pid in $OUT.
|
||||
const notingBundle = `#!/bin/sh
|
||||
printf 'started %s\n' "$$" >> "$OUT"
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id" ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
// The service manager stops the runtime by signalling its whole group, every bundle with it (issue 1126, ace's
|
||||
// reboot of 2026-10-28: every bundle "exited (SIGTERM); started again in 500ms" for the 15 s the runtime took
|
||||
// to stop, the power bundle among them dialling the system bus again, which restarted the bus during the
|
||||
// shutdown and held it 29 minutes). A bundle that exits while it serves is started again; once the runtime
|
||||
// itself is stopping, none is, by a restart or by a call.
|
||||
func TestABundleIsStartedAgainUnlessTheRuntimeIsStopping(t *testing.T) {
|
||||
saved := RestartFirst
|
||||
RestartFirst = 10 * time.Millisecond
|
||||
t.Cleanup(func() { RestartFirst = saved; shuttingDown.Store(false) })
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
stopping bool
|
||||
starts int
|
||||
}{
|
||||
{"a bundle that dies while the runtime serves", false, 2},
|
||||
{"a bundle signalled with the stopping runtime", true, 1},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
shuttingDown.Store(false)
|
||||
dir := t.TempDir()
|
||||
entry, out := filepath.Join(dir, "bundle"), filepath.Join(dir, "out")
|
||||
if err := os.WriteFile(entry, []byte(notingBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l, err := Start("noting", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
said, _ := os.ReadFile(out)
|
||||
var pid int
|
||||
if _, err := fmt.Sscanf(string(said), "started %d", &pid); err != nil {
|
||||
t.Fatalf("the bundle noted %q", said)
|
||||
}
|
||||
if row.stopping {
|
||||
ShutDown()
|
||||
}
|
||||
if err := syscall.Kill(pid, syscall.SIGTERM); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The restart row ends at its second start; the stopping row watches a window many restart pauses long.
|
||||
deadline := time.Now().Add(250 * time.Millisecond)
|
||||
for {
|
||||
said, _ = os.ReadFile(out)
|
||||
n := strings.Count(string(said), "started")
|
||||
if n > row.starts {
|
||||
t.Fatalf("started %d times, want %d", n, row.starts)
|
||||
}
|
||||
if n == row.starts && !row.stopping {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
if n != row.starts {
|
||||
t.Fatalf("started %d times, want %d", n, row.starts)
|
||||
}
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if row.stopping {
|
||||
// And a bundle first loaded now (a reload) is not started either.
|
||||
if _, err := Start("late", entry, append(os.Environ(), "OUT="+out), &subscribing{}, t.Logf); err == nil {
|
||||
t.Error("a bundle was started after the runtime began stopping")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,9 @@ while IFS= read -r line; do
|
||||
*'"method":"tools/list"'*)
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
||||
printf '{"jsonrpc":"2.0","id":"s1","method":"mesh/subscribe","params":{"pattern":"#"}}\n' ;;
|
||||
*'"method":"mesh/heartbeat"'*)
|
||||
params=$(printf '%s' "$line" | sed -n 's/.*"params":\(.*\)}$/\1/p')
|
||||
printf '{"jsonrpc":"2.0","id":"h1","method":"mesh/publish","params":%s}\n' "$params" ;;
|
||||
*'"method":"mesh/event"'*)
|
||||
case "$ANSWER" in
|
||||
error) printf '{"jsonrpc":"2.0","id":%s,"error":{"code":-32000,"message":"Unexpected end of JSON input"}}\n' "$id" ;;
|
||||
@@ -30,23 +33,41 @@ done
|
||||
`
|
||||
|
||||
type subscribing struct {
|
||||
mu sync.Mutex
|
||||
deliver func(json.RawMessage) error
|
||||
mu sync.Mutex
|
||||
deliver func(json.RawMessage) error
|
||||
beat func(json.RawMessage)
|
||||
published []json.RawMessage
|
||||
}
|
||||
|
||||
func (b *subscribing) Publish(json.RawMessage) error { return nil }
|
||||
func (b *subscribing) Publish(p json.RawMessage) error {
|
||||
b.mu.Lock()
|
||||
b.published = append(b.published, p)
|
||||
b.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
func (b *subscribing) Ask(json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) State(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) Log(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) Watch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
func (b *subscribing) Seat(string, json.RawMessage, func(string, any) (json.RawMessage, error)) (json.RawMessage, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (b *subscribing) Subscribe(d func(json.RawMessage) error) error {
|
||||
func (b *subscribing) Provision(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) ProvisionWatch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
func (b *subscribing) BusState(string, json.RawMessage) (json.RawMessage, error) { return nil, nil }
|
||||
func (b *subscribing) BusStateWatch(string, json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
func (b *subscribing) Begin(string) (string, func()) { return "", func() {} }
|
||||
func (b *subscribing) Subscribe(d func(json.RawMessage) error, beat func(json.RawMessage)) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.deliver = d
|
||||
b.beat = beat
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -90,3 +111,49 @@ func TestAnEventIsTakenByAnyAnswerThatIsNotAnError(t *testing.T) {
|
||||
t.Fatalf("an error answer was not the bundle's refusal of mesh/event: %#v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *subscribing) LogWatch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
// The runtime's word on the module's consumer between events reaches the child that subscribed as `mesh/heartbeat`,
|
||||
// unchanged (stdio.OnHeartbeat reads it in the bundle).
|
||||
func TestTheWordOnTheConsumerBetweenEventsReachesTheBundleThatSubscribed(t *testing.T) {
|
||||
entry := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.WriteFile(entry, []byte(answeringBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := &subscribing{}
|
||||
l, err := Start("plex", entry, append(os.Environ(), "ANSWER=result"), b, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
var beat func(json.RawMessage)
|
||||
for i := 0; beat == nil; i++ {
|
||||
b.mu.Lock()
|
||||
beat = b.beat
|
||||
b.mu.Unlock()
|
||||
if i > 100 {
|
||||
t.Fatal("the bundle never subscribed")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
word := json.RawMessage(`{"at":"2026-11-04T13:00:00Z","missed":true,"why":"the bus ended it"}`)
|
||||
beat(word)
|
||||
for i := 0; ; i++ {
|
||||
b.mu.Lock()
|
||||
got := b.published
|
||||
b.mu.Unlock()
|
||||
if len(got) == 1 {
|
||||
if string(got[0]) != string(word) {
|
||||
t.Fatalf("the bundle was told %s, not %s", got[0], word)
|
||||
}
|
||||
return
|
||||
}
|
||||
if i > 100 {
|
||||
t.Fatal("the bundle was told nothing")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,9 +15,11 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/wire"
|
||||
@@ -37,7 +39,36 @@ type Tool struct {
|
||||
Name string
|
||||
Description string
|
||||
Input json.RawMessage
|
||||
Run func(args json.RawMessage) (json.RawMessage, error)
|
||||
// Effect is what the tool does, reads, acts or dangerous (novox/hq ADR 0387 §1), where the runtime knows it: its own
|
||||
// tools say it; a bundle's tool is marked by its manifest.
|
||||
Effect string
|
||||
// Run calls the tool for caller: the bus user the call's subject named, or "" when it named none. A
|
||||
// caller is handed to the bundle in the call's `_meta` (MetaCaller), never among its arguments.
|
||||
Run func(caller string, args json.RawMessage) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// MetaCaller is where in a `tools/call`'s `_meta` the runtime names the caller it took from the subject the
|
||||
// call arrived on (novox/hq issue 365): mesh-sdk's stdio.MetaCaller, which hands it to the tool as
|
||||
// stdio.CallerOf, and says Unproven when it is absent.
|
||||
const MetaCaller = "mesh/caller"
|
||||
|
||||
// callParams is a `tools/call`'s params: the tool, its arguments as the caller sent them, and in `_meta`, which the
|
||||
// runtime writes and no caller can, the caller — only when the subject named one — and the id of the call in flight
|
||||
// (mesh-sdk buscalls.MetaCall), which a ledger's apply names so the runtime stamps the caller it proved (novox/hq ADR
|
||||
// 0597 §7).
|
||||
func callParams(name, caller, call string, args json.RawMessage) map[string]any {
|
||||
params := map[string]any{"name": name, "arguments": args}
|
||||
meta := map[string]any{}
|
||||
if caller != "" {
|
||||
meta[MetaCaller] = caller
|
||||
}
|
||||
if call != "" {
|
||||
meta[buscalls.MetaCall] = call
|
||||
}
|
||||
if len(meta) > 0 {
|
||||
params["_meta"] = meta
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
// Registration is the tools a bundle listed under one name: its module's, or a seat's.
|
||||
@@ -49,7 +80,9 @@ type Registration struct {
|
||||
// Bus is what a launched bundle reaches the mesh through (novox/hq ADR 0193, ADR 0198): the runtime
|
||||
// publishes, asks and subscribes on the module's behalf. Subscribe is asked each time the module's
|
||||
// code subscribes; the runtime binds the module's consumer once and calls deliver for every event,
|
||||
// acknowledging it on the bus when deliver returns nil.
|
||||
// acknowledging it on the bus when deliver returns nil. beat is the runtime's word on that consumer between
|
||||
// events (a bus.Beat as JSON), which the launch hands the child that subscribed as `mesh/heartbeat`, so a bundle
|
||||
// can tell a silent source from a quiet one (stdio.OnHeartbeat).
|
||||
//
|
||||
// State and Watch reach the module's state (novox/hq ADR 0201): State answers `get`, `put`, `delete`
|
||||
// and `keys`; Watch hands deliver the current values and then every change, and returns once the
|
||||
@@ -57,15 +90,45 @@ type Registration struct {
|
||||
type Bus interface {
|
||||
Publish(params json.RawMessage) error
|
||||
Ask(params json.RawMessage) (json.RawMessage, error)
|
||||
Subscribe(deliver func(envelope json.RawMessage) error) error
|
||||
Subscribe(deliver func(envelope json.RawMessage) error, beat func(heartbeat json.RawMessage)) error
|
||||
State(verb string, params json.RawMessage) (json.RawMessage, error)
|
||||
Watch(params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error)
|
||||
// Log answers a bundle's `append`, `read`, `last` and `info` on its module's log (novox/hq ADR 0297).
|
||||
Log(verb string, params json.RawMessage) (json.RawMessage, error)
|
||||
// LogWatch hands deliver each entry of a log from where the bundle asked, one at a time (novox/hq ADR 0571).
|
||||
LogWatch(params json.RawMessage, deliver func(entry json.RawMessage) error) (stop func(), err error)
|
||||
// Seat answers a bundle's seat traffic (novox/hq ADR 0259 §3): `publish`, `prove` and `record` at once;
|
||||
// `take` and `answer` bind once and hand each message or proof on through handOn, to whichever child
|
||||
// of the module asked last — a restarted child asks again as its code runs again.
|
||||
Seat(verb string, params json.RawMessage, handOn func(method string, params any) (json.RawMessage, error)) (json.RawMessage, error)
|
||||
// Provision answers a bundle's `get` of one of its module's values at runtime, and ProvisionWatch hands deliver
|
||||
// that value now and each change after, as the module whose bundle asks and never a module the bundle names
|
||||
// (novox/hq ADR 0349 §14, ADR 0370 §4): its config, and its bindings' values that are not secret.
|
||||
Provision(verb string, params json.RawMessage) (json.RawMessage, error)
|
||||
ProvisionWatch(params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error)
|
||||
// BusState answers a bundle's calls on its module's bus state in the new form (mesh-sdk buscalls; novox/hq ADR 0595
|
||||
// §9, design 25 § 12.5): its objects, a stream's read-back and fill, a work queue's size and dead letters, and its
|
||||
// ledgers, by the method the bundle asked, from its own module's bindings alone.
|
||||
BusState(method string, params json.RawMessage) (json.RawMessage, error)
|
||||
// BusStateWatch hands deliver each change of an objects or a ledger watch, the current ones first.
|
||||
BusStateWatch(method string, params json.RawMessage, deliver func(change json.RawMessage) error) (stop func(), err error)
|
||||
// Begin says a tool call of the module is in flight for the caller the runtime proved, and answers its id, which the
|
||||
// bundle is handed in the call's `_meta`, and the end to call once the call is answered (novox/hq ADR 0597 §7).
|
||||
Begin(caller string) (id string, end func())
|
||||
}
|
||||
|
||||
// busStateCalls are the methods of mesh-sdk buscalls a bundle asks and BusState answers.
|
||||
var busStateCalls = map[string]bool{
|
||||
buscalls.ObjectsPut: true, buscalls.ObjectsGet: true, buscalls.ObjectsList: true, buscalls.ObjectsDelete: true,
|
||||
buscalls.ObjectsInfo: true, buscalls.StreamRead: true, buscalls.StreamFill: true, buscalls.WorkQueueSize: true,
|
||||
buscalls.WorkQueueDead: true, buscalls.LedgerApply: true, buscalls.LedgerGet: true, buscalls.LedgerHistory: true,
|
||||
buscalls.LedgerQuery: true, buscalls.LedgerRebuild: true, buscalls.LedgerReindex: true, buscalls.LedgerSnapshot: true,
|
||||
buscalls.LedgerLag: true,
|
||||
}
|
||||
|
||||
// busStateWatches are the watches of mesh-sdk buscalls, each with the method its changes are handed to the bundle by.
|
||||
var busStateWatches = map[string]string{buscalls.ObjectsWatch: buscalls.ObjectsWatched, buscalls.LedgerWatch: buscalls.LedgerWatched}
|
||||
|
||||
// EventTimeout bounds how long a bundle has to handle one event before it is offered again.
|
||||
var EventTimeout = 2 * time.Minute
|
||||
|
||||
@@ -88,6 +151,21 @@ var (
|
||||
RestartSettle = time.Minute
|
||||
)
|
||||
|
||||
// shuttingDown is set once the runtime itself is stopping (ShutDown): from then on no bundle is started again.
|
||||
var shuttingDown atomic.Bool
|
||||
|
||||
// ShutDown says the runtime itself is stopping, before it stops its bundles one by one. The service manager stops
|
||||
// the runtime by signalling its whole group, so every bundle receives SIGTERM with it, and a bundle's exit must not
|
||||
// be taken for a fault: started again, it would run on through the machine's shutdown with nothing left to stop it
|
||||
// (issue 1126: on ace's reboot of 2026-10-28 the restarted bundles dialled the system bus again and started it
|
||||
// during the shutdown, which then waited 29 minutes). After ShutDown no bundle is started again, by a restart or a
|
||||
// call.
|
||||
func ShutDown() { shuttingDown.Store(true) }
|
||||
|
||||
// StopWait bounds how long Launched.Stop waits for its bundle's exit to be handled; a bundle that
|
||||
// ignores SIGTERM longer than this is left to exit on its own (novox/hq issue 476).
|
||||
var StopWait = 5 * time.Second
|
||||
|
||||
// Launched is a running bundle: what it registered, and how to stop it.
|
||||
type Launched struct {
|
||||
Registrations []Registration
|
||||
@@ -124,6 +202,9 @@ type child struct {
|
||||
pending map[int64]chan message
|
||||
next int64
|
||||
dead chan struct{}
|
||||
// handled is closed once the child's exit has been handled whole — its state cleared and its
|
||||
// restart decided — which Stop waits for (novox/hq issue 476).
|
||||
handled chan struct{}
|
||||
why error
|
||||
// watches are the state watches this child asked for, stopped when it exits (ADR 0201): a child
|
||||
// that starts again watches again, as its code runs again.
|
||||
@@ -143,9 +224,17 @@ func (c *child) write(m any) error {
|
||||
|
||||
var stackLine = regexp.MustCompile(`^\s+at\s`)
|
||||
|
||||
// colour is a terminal's colour or style sequence, which node writes into a stack trace when its
|
||||
// environment asks for colour (FORCE_COLOR): taken out before a line is judged or kept as a bundle's
|
||||
// reason, so the reason reads the same in every environment.
|
||||
var colour = regexp.MustCompile("\x1b\\[[0-9;]*m")
|
||||
|
||||
// Start launches one bundle and learns its tools. It fails when the child cannot be started or does
|
||||
// not complete the handshake. A child that exits later is started again on its next call.
|
||||
func Start(module, entry string, env []string, mesh Bus, logf func(string, ...any)) (*Launched, error) {
|
||||
if shuttingDown.Load() {
|
||||
return nil, fmt.Errorf("%s's bundle is not started: the runtime is stopping", module)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var current *child
|
||||
// The child that last subscribed is the one events are handed to: a restarted child subscribes
|
||||
@@ -174,9 +263,23 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
_, err := c.ask(module, "mesh/event", map[string]any{"envelope": envelope}, EventTimeout)
|
||||
return err
|
||||
}
|
||||
// The heartbeat of the module's consumer goes to the child that last subscribed, as a notification: the
|
||||
// bundle answers nothing, and a child not running misses it, as it misses the events it did not take.
|
||||
beat := func(heartbeat json.RawMessage) {
|
||||
mu.Lock()
|
||||
c := subscriber
|
||||
mu.Unlock()
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
_ = c.write(map[string]any{"jsonrpc": "2.0", "method": "mesh/heartbeat", "params": heartbeat})
|
||||
}
|
||||
var restart func(after time.Duration)
|
||||
var startedAt time.Time
|
||||
pause := RestartFirst
|
||||
// The restart pauses are read once, here: the exit handling runs on its own goroutine and must not
|
||||
// read the package's settings while a caller (a test) changes them (novox/hq issue 476).
|
||||
pauseFirst, pauseMost, settle := RestartFirst, RestartMost, RestartSettle
|
||||
pause := pauseFirst
|
||||
|
||||
start := func() (*child, error) {
|
||||
cmd := exec.Command(entry)
|
||||
@@ -196,7 +299,8 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &child{cmd: cmd, stdin: stdin, pending: map[int64]chan message{}, next: 1, dead: make(chan struct{})}
|
||||
c := &child{cmd: cmd, stdin: stdin, pending: map[int64]chan message{}, next: 1, dead: make(chan struct{}),
|
||||
handled: make(chan struct{})}
|
||||
var lastSaid string
|
||||
var saidMu sync.Mutex
|
||||
stderrDone := make(chan struct{})
|
||||
@@ -210,14 +314,16 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
continue
|
||||
}
|
||||
logf("[%s] %s", module, line)
|
||||
if !stackLine.MatchString(line) && !strings.HasPrefix(line, "Node.js v") {
|
||||
plain := colour.ReplaceAllString(line, "")
|
||||
if !stackLine.MatchString(plain) && !strings.HasPrefix(plain, "Node.js v") {
|
||||
saidMu.Lock()
|
||||
lastSaid = strings.TrimSpace(line)
|
||||
lastSaid = strings.TrimSpace(plain)
|
||||
saidMu.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer close(c.handled)
|
||||
scan := bufio.NewScanner(stdout)
|
||||
scan.Buffer(make([]byte, 64*1024), 16<<20)
|
||||
for scan.Scan() {
|
||||
@@ -251,8 +357,8 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
mu.Lock()
|
||||
subscriber = c
|
||||
mu.Unlock()
|
||||
err = mesh.Subscribe(deliver)
|
||||
case "mesh/seat.publish", "mesh/seat.prove", "mesh/seat.record", "mesh/seat.kinds":
|
||||
err = mesh.Subscribe(deliver, beat)
|
||||
case "mesh/seat.publish", "mesh/seat.prove", "mesh/seat.record", "mesh/seat.kinds", "mesh/seat.pull", "mesh/seat.settle":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Seat(strings.TrimPrefix(m.Method, "mesh/seat."), m.Params, nil); err == nil {
|
||||
result = answered
|
||||
@@ -266,11 +372,59 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
result = answered
|
||||
}
|
||||
case "mesh/state.get", "mesh/state.put", "mesh/state.delete", "mesh/state.keys",
|
||||
"mesh/state.create", "mesh/state.update":
|
||||
"mesh/state.create", "mesh/state.update", "mesh/state.history", "mesh/state.info":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.State(strings.TrimPrefix(m.Method, "mesh/state."), m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/log.append", "mesh/log.read", "mesh/log.newest", "mesh/log.last", "mesh/log.info":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Log(strings.TrimPrefix(m.Method, "mesh/log."), m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/provision.get":
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.Provision("get", m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
case "mesh/provision.watch":
|
||||
// As a state's watch: each change asked of this child in order, naming the watch by the
|
||||
// id the child asked it with, and answered once the current value is handed over.
|
||||
var stop func()
|
||||
watch := m.ID
|
||||
stop, err = mesh.ProvisionWatch(m.Params, func(change json.RawMessage) error {
|
||||
_, err := c.ask(module, "mesh/provision", map[string]any{"watch": watch, "change": change}, EventTimeout)
|
||||
return err
|
||||
})
|
||||
if err == nil {
|
||||
c.mu.Lock()
|
||||
if c.why != nil {
|
||||
c.mu.Unlock()
|
||||
stop()
|
||||
} else {
|
||||
c.watches = append(c.watches, stop)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
case "mesh/log.watch":
|
||||
// As a state's watch: each entry asked of this child in order, `mesh/log {watch, entry}`,
|
||||
// naming the watch by the id the child asked it with (novox/hq ADR 0571).
|
||||
var stop func()
|
||||
watch := m.ID
|
||||
stop, err = mesh.LogWatch(m.Params, func(entry json.RawMessage) error {
|
||||
_, err := c.ask(module, "mesh/log", map[string]any{"watch": watch, "entry": entry}, EventTimeout)
|
||||
return err
|
||||
})
|
||||
if err == nil {
|
||||
c.mu.Lock()
|
||||
if c.why != nil {
|
||||
c.mu.Unlock()
|
||||
stop()
|
||||
} else {
|
||||
c.watches = append(c.watches, stop)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
case "mesh/state.watch":
|
||||
// Each change is asked of this child, in order, naming the watch it is for by
|
||||
// the id the child asked it with — two watches of one state are two handlers;
|
||||
@@ -292,7 +446,33 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
case buscalls.ObjectsWatch, buscalls.LedgerWatch:
|
||||
// As a state's watch: each change asked of this child in order, by the method of its
|
||||
// family; the change names the watch by the id the bundle gave in its request.
|
||||
var stop func()
|
||||
told := busStateWatches[m.Method]
|
||||
stop, err = mesh.BusStateWatch(m.Method, m.Params, func(change json.RawMessage) error {
|
||||
_, err := c.ask(module, told, change, EventTimeout)
|
||||
return err
|
||||
})
|
||||
if err == nil {
|
||||
c.mu.Lock()
|
||||
if c.why != nil {
|
||||
c.mu.Unlock()
|
||||
stop()
|
||||
} else {
|
||||
c.watches = append(c.watches, stop)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
}
|
||||
default:
|
||||
if busStateCalls[m.Method] {
|
||||
var answered json.RawMessage
|
||||
if answered, err = mesh.BusState(m.Method, m.Params); err == nil {
|
||||
result = answered
|
||||
}
|
||||
break
|
||||
}
|
||||
if len(m.ID) > 0 {
|
||||
_ = c.write(map[string]any{"jsonrpc": "2.0", "id": m.ID,
|
||||
"error": map[string]any{"code": -32601, "message": "the runtime answers no " + m.Method + " from a bundle"}})
|
||||
@@ -363,17 +543,20 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
}
|
||||
wasStopped := stopped
|
||||
// Started again at once if it ran a while; after a growing pause while it keeps exiting.
|
||||
wait := RestartFirst
|
||||
if time.Since(startedAt) < RestartSettle {
|
||||
wait := pauseFirst
|
||||
if time.Since(startedAt) < settle {
|
||||
wait = pause
|
||||
if pause *= 2; pause > RestartMost {
|
||||
pause = RestartMost
|
||||
if pause *= 2; pause > pauseMost {
|
||||
pause = pauseMost
|
||||
}
|
||||
} else {
|
||||
pause = RestartFirst
|
||||
pause = pauseFirst
|
||||
}
|
||||
mu.Unlock()
|
||||
if !wasStopped && wasServing {
|
||||
if wasServing && !wasStopped && shuttingDown.Load() {
|
||||
logf("[mesh-tools] %s; not started again: the runtime is stopping", why)
|
||||
}
|
||||
if !wasStopped && wasServing && !shuttingDown.Load() {
|
||||
// Every bundle stays up (ADR 0198): code that runs long — a handler, a provisioner —
|
||||
// is not waiting for a call to bring it back, and a tool bundle back early costs nothing.
|
||||
logf("[mesh-tools] %s; started again in %s", why, wait.Round(100*time.Millisecond))
|
||||
@@ -391,14 +574,27 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
|
||||
asking := func(method string, params any, timeout time.Duration) (json.RawMessage, error) {
|
||||
mu.Lock()
|
||||
c := current
|
||||
c, gone := current, stopped
|
||||
mu.Unlock()
|
||||
if gone {
|
||||
// A stopped bundle is never started again by a call: a process started now would be one
|
||||
// nothing stops (issue 1945).
|
||||
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
|
||||
}
|
||||
if c == nil {
|
||||
if shuttingDown.Load() {
|
||||
return nil, fmt.Errorf("%s's bundle is not started again: the runtime is stopping", module)
|
||||
}
|
||||
fresh, err := start()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mu.Lock()
|
||||
if stopped {
|
||||
mu.Unlock()
|
||||
_ = fresh.cmd.Process.Signal(syscall.SIGTERM)
|
||||
return nil, fmt.Errorf("%s's bundle is stopped; it answers no call", module)
|
||||
}
|
||||
if current == nil {
|
||||
current = fresh
|
||||
startedAt = time.Now()
|
||||
@@ -416,7 +612,7 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
go func() {
|
||||
time.Sleep(after)
|
||||
mu.Lock()
|
||||
if stopped || current != nil {
|
||||
if stopped || current != nil || shuttingDown.Load() {
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
@@ -425,10 +621,10 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
if !stopped {
|
||||
if !stopped && !shuttingDown.Load() {
|
||||
wait := pause
|
||||
if pause *= 2; pause > RestartMost {
|
||||
pause = RestartMost
|
||||
if pause *= 2; pause > pauseMost {
|
||||
pause = pauseMost
|
||||
}
|
||||
logf("[mesh-tools] %s's bundle did not start again: %v; trying in %s", module, err, wait)
|
||||
go restart(wait)
|
||||
@@ -484,11 +680,13 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
input = json.RawMessage("{}")
|
||||
}
|
||||
tool := Tool{Name: name, Description: t.Description, Input: input,
|
||||
Run: func(args json.RawMessage) (json.RawMessage, error) {
|
||||
Run: func(caller string, args json.RawMessage) (json.RawMessage, error) {
|
||||
if len(args) == 0 || string(args) == "null" {
|
||||
args = json.RawMessage("{}")
|
||||
}
|
||||
res, err := asking("tools/call", map[string]any{"name": full, "arguments": args}, CallTimeout)
|
||||
call, end := mesh.Begin(caller)
|
||||
defer end()
|
||||
res, err := asking("tools/call", callParams(full, caller, call, args), CallTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -533,6 +731,12 @@ func Start(module, entry string, env []string, mesh Bus, logf func(string, ...an
|
||||
mu.Unlock()
|
||||
if c != nil && c.cmd.Process != nil {
|
||||
_ = c.cmd.Process.Signal(syscall.SIGTERM)
|
||||
// Returns once the bundle's exit is handled, so nothing of it runs on after Stop
|
||||
// (novox/hq issue 476); bounded, for a bundle that does not exit on SIGTERM.
|
||||
select {
|
||||
case <-c.handled:
|
||||
case <-time.After(StopWait):
|
||||
}
|
||||
}
|
||||
}}
|
||||
for _, under := range order {
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package launch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A bundle that asks for one of its values and watches it, and writes each change it is handed to $OUT.
|
||||
const provisionBundle = `#!/bin/sh
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*)
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
||||
printf '{"jsonrpc":"2.0","id":"g1","method":"mesh/provision.get","params":{"kind":"config","key":"role"}}\n'
|
||||
printf '{"jsonrpc":"2.0","id":"w1","method":"mesh/provision.watch","params":{"kind":"config","key":"role"}}\n' ;;
|
||||
*'"method":"mesh/provision"'*)
|
||||
printf '%s\n' "$line" >> "$OUT"
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"id":"g1"'*) printf '%s\n' "$line" >> "$OUT" ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
type provisioning struct {
|
||||
subscribing
|
||||
mu sync.Mutex
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (b *provisioning) Provision(verb string, params json.RawMessage) (json.RawMessage, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.asked = append(b.asked, verb+" "+string(params))
|
||||
return json.RawMessage(`"the build machine"`), nil
|
||||
}
|
||||
|
||||
func (b *provisioning) ProvisionWatch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
b.mu.Lock()
|
||||
b.asked = append(b.asked, "watch "+string(params))
|
||||
b.mu.Unlock()
|
||||
return func() {}, deliver(json.RawMessage(`{"kind":"config","key":"role","value":"the build machine"}`))
|
||||
}
|
||||
|
||||
// novox/hq ADR 0349 §14, ADR 0370 §4: a bundle's ask for its values reaches the runtime's bus from its own channel,
|
||||
// answered there, and a watch's change is handed to it naming the watch it asked.
|
||||
func TestABundleAsksForItsValuesAndIsHandedItsWatchsChanges(t *testing.T) {
|
||||
entry := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.WriteFile(entry, []byte(provisionBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := filepath.Join(t.TempDir(), "out")
|
||||
b := &provisioning{}
|
||||
l, err := Start("claude-code", entry, append(os.Environ(), "OUT="+out), b, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
said, _ := os.ReadFile(out)
|
||||
if strings.Count(string(said), "\n") >= 2 {
|
||||
// Each line as the bundle read it, by what it is: the get's answer, and the watch's change.
|
||||
got := map[string]string{}
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(said)), "\n") {
|
||||
var m struct {
|
||||
ID any `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Result json.RawMessage `json:"result"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(line), &m); err != nil {
|
||||
t.Fatalf("the bundle read %s", line)
|
||||
}
|
||||
if m.Method != "" {
|
||||
got[m.Method] = string(m.Params)
|
||||
} else {
|
||||
got["answer"] = string(m.Result)
|
||||
}
|
||||
}
|
||||
if got["answer"] != `"the build machine"` {
|
||||
t.Errorf("the get was answered %s", got["answer"])
|
||||
}
|
||||
if got["mesh/provision"] != `{"change":{"kind":"config","key":"role","value":"the build machine"},"watch":"w1"}` {
|
||||
t.Errorf("the watch was handed %s", got["mesh/provision"])
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the bundle was handed %q", said)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
slices.Sort(b.asked)
|
||||
if len(b.asked) != 2 || b.asked[0] != `get {"kind":"config","key":"role"}` || b.asked[1] != `watch {"kind":"config","key":"role"}` {
|
||||
t.Errorf("the runtime's bus was asked %q", b.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *provisioning) LogWatch(json.RawMessage, func(json.RawMessage) error) (func(), error) {
|
||||
return func() {}, nil
|
||||
}
|
||||
@@ -69,3 +69,45 @@ func TestABundleTakesASeatsWorkThroughItsOwnChannel(t *testing.T) {
|
||||
t.Fatalf("the work was not handed to the bundle: %s %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A bundle that pulls one ask from its worker when it is listed.
|
||||
const pullingBundle = `#!/bin/sh
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9]*\)[,}].*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{}}\n' "$id" ;;
|
||||
*'"method":"tools/list"'*)
|
||||
printf '{"jsonrpc":"2.0","id":%s,"result":{"tools":[]}}\n' "$id"
|
||||
printf '{"jsonrpc":"2.0","id":"p1","method":"mesh/seat.pull","params":{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker","wait_ms":0}}\n' ;;
|
||||
esac
|
||||
done
|
||||
`
|
||||
|
||||
// 1887/A11: a bundle's pull reaches the runtime's bus from its own channel, as its other seat traffic does.
|
||||
func TestABundlePullsASeatsWorkThroughItsOwnChannel(t *testing.T) {
|
||||
entry := filepath.Join(t.TempDir(), "bundle")
|
||||
if err := os.WriteFile(entry, []byte(pullingBundle), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := &seating{}
|
||||
l, err := Start("mesh-build-go", entry, os.Environ(), b, t.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(l.Stop)
|
||||
for i := 0; ; i++ {
|
||||
b.mu.Lock()
|
||||
verb, params := b.verb, b.params
|
||||
b.mu.Unlock()
|
||||
if verb == "pull" {
|
||||
if params != `{"worker":"SEAT_NODE_BUILD_GO_DELIVERY_worker","wait_ms":0}` {
|
||||
t.Fatalf("pulled with %s", params)
|
||||
}
|
||||
return
|
||||
}
|
||||
if i > 100 {
|
||||
t.Fatal("the bundle's pull never reached the runtime")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
// Package ledger is the tool runner's side of a ledger (novox/hq ADR 0597, design 25 § 12.4; issue 1966): a storage type
|
||||
// whose state is derived from its history. `apply` checks an operation against its type, appends it to the instance's
|
||||
// history with the last sequence expected, derives the state by the operation's effect and writes it as an update of the
|
||||
// revision read, and writes the instance's snapshot. What the history, the state and the snapshots are kept in is given
|
||||
// (History, State, Snapshots): the bus in the runtime, fakes in a test.
|
||||
package ledger
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The refusals the stores answer, told by errors.Is.
|
||||
var (
|
||||
// ErrRaced is an append or a snapshot written with an expected sequence that is no longer the last: another writer
|
||||
// wrote first.
|
||||
ErrRaced = errors.New("another writer wrote the instance's history first")
|
||||
// ErrChanged is a state written at a revision that is no longer the key's.
|
||||
ErrChanged = errors.New("the state was written by another writer first")
|
||||
// ErrFull is a state bucket that refuses a write because it is full.
|
||||
ErrFull = errors.New("the state bucket is full")
|
||||
)
|
||||
|
||||
// CodeChanged is the error code a bundle is answered a lost race with, as a lost state update (mesh-sdk stdio.CodeChanged).
|
||||
const CodeChanged = -32010
|
||||
|
||||
// raced is an apply refused because another writer appended to the instance first (ADR 0597 §5).
|
||||
type raced struct{ instance string }
|
||||
|
||||
func (e raced) Error() string {
|
||||
return "the operation on " + e.instance + " is not taken: another writer appended to its history first; read it and apply again (novox/hq ADR 0597 §5)"
|
||||
}
|
||||
func (e raced) Is(target error) bool { return target == ErrRaced }
|
||||
func (e raced) ErrorCode() int { return CodeChanged }
|
||||
|
||||
// Kept is one message a history keeps: its sequence and its bytes.
|
||||
type Kept struct {
|
||||
Sequence uint64
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// History is a ledger's history: a stream with a subject per instance, each instance named by its key.
|
||||
type History interface {
|
||||
// Last is the sequence of the instance's last operation, 0 when it has none.
|
||||
Last(key string) (uint64, error)
|
||||
// Append writes one operation, expecting the instance's last sequence to be expect (0: it has none); ErrRaced when
|
||||
// it is not.
|
||||
Append(key string, op []byte, expect uint64) (uint64, error)
|
||||
// Read is the operations of the instances keys names (a key, with `*` for one name and `>` for the rest) from a
|
||||
// sequence on, oldest first, at most limit of them (every one when limit is 0).
|
||||
Read(keys string, from uint64, limit int) ([]Kept, error)
|
||||
}
|
||||
|
||||
// State is a ledger's state bucket: each instance's record under its key.
|
||||
type State interface {
|
||||
// Get is the key's value and revision; nil and 0 when it has none.
|
||||
Get(key string) ([]byte, uint64, error)
|
||||
// Write writes the key's value at the revision read (0: only when it has none); ErrChanged when the key is no longer
|
||||
// at it, ErrFull when the bucket is full.
|
||||
Write(key string, value []byte, revision uint64) (uint64, error)
|
||||
// Keys is every key the bucket holds.
|
||||
Keys() ([]string, error)
|
||||
}
|
||||
|
||||
// Snapshots are a ledger's snapshots: the latest record of each instance, on a subject of its own, never expiring.
|
||||
type Snapshots interface {
|
||||
// Get is the key's latest snapshot and the sequence it was written at; nil and 0 when it has none.
|
||||
Get(key string) ([]byte, uint64, error)
|
||||
// Put writes the key's snapshot, expecting its last to be at expect (0: none); ErrRaced when it is not.
|
||||
Put(key string, value []byte, expect uint64) error
|
||||
}
|
||||
|
||||
// Ledger is one ledger of one module: its types and where it is kept.
|
||||
type Ledger struct {
|
||||
Module string
|
||||
Name string
|
||||
// Account is the owning module's authenticated account, which every operation it applies is stamped with.
|
||||
Account string
|
||||
Types []catalogue.LedgerType
|
||||
History History
|
||||
State State
|
||||
Snapshots Snapshots
|
||||
// Indexes are its declared indexes' buckets, by index name.
|
||||
Indexes map[string]Index
|
||||
// Now is the clock an operation is stamped by; time.Now when nil.
|
||||
Now func() time.Time
|
||||
// Logf says what an apply could not do after its operation was taken; nothing is said when nil.
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
func (l *Ledger) say(format string, args ...any) {
|
||||
if l.Logf != nil {
|
||||
l.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Record is what the state and a snapshot keep of an instance: its state and the sequence of the last operation in it.
|
||||
type Record struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
State json.RawMessage `json:"state"`
|
||||
}
|
||||
|
||||
// segment is one part of an instance's path.
|
||||
var segment = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
|
||||
// KeyOf is the key an instance is kept under, in the state and as its history's and snapshot's subject token: its path's
|
||||
// parts joined by dots (`decision/0290` is `decision.0290`).
|
||||
func KeyOf(instance string) (string, error) {
|
||||
parts := strings.Split(instance, "/")
|
||||
if len(parts) < 2 {
|
||||
return "", fmt.Errorf("%q is not an instance: an instance is named by a path whose first part is its type's name, "+
|
||||
"such as decision/0290 (novox/hq ADR 0597 §1)", instance)
|
||||
}
|
||||
for _, p := range parts {
|
||||
if !segment.MatchString(p) {
|
||||
return "", fmt.Errorf("%q is not an instance: each part of its path holds letters, digits, - and _ alone", instance)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "."), nil
|
||||
}
|
||||
|
||||
// InstanceOf is the instance a key names.
|
||||
func InstanceOf(key string) string { return strings.ReplaceAll(key, ".", "/") }
|
||||
|
||||
func (l *Ledger) now() time.Time {
|
||||
if l.Now != nil {
|
||||
return l.Now().UTC()
|
||||
}
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
// typeOf is the instance's type, by its path's first part.
|
||||
func (l *Ledger) typeOf(instance string) (catalogue.LedgerType, error) {
|
||||
name, _, _ := strings.Cut(instance, "/")
|
||||
for _, t := range l.Types {
|
||||
if t.Name == name {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return catalogue.LedgerType{}, fmt.Errorf("%s's ledger %s has no type %q, which the instance %s names", l.Module, l.Name, name, instance)
|
||||
}
|
||||
|
||||
// operationOf is the operation a type allows by its name.
|
||||
func operationOf(t catalogue.LedgerType, name string) (catalogue.LedgerOperation, error) {
|
||||
for _, op := range t.Operations {
|
||||
if op.Name == name {
|
||||
return op, nil
|
||||
}
|
||||
}
|
||||
return catalogue.LedgerOperation{}, fmt.Errorf("the type %s allows no operation %q", t.Name, name)
|
||||
}
|
||||
|
||||
// Digest is the digest a type pairs a text field with (ADR 0597 §2): sha256 of the text, as the knowledge base writes it.
|
||||
func Digest(text string) string {
|
||||
sum := sha256.Sum256([]byte(text))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// kinds is the JSON kind each schema type name is held to; a name not here is the judge's to know.
|
||||
var kinds = map[string]func(any) bool{
|
||||
"string": func(v any) bool { _, ok := v.(string); return ok },
|
||||
"number": func(v any) bool { _, ok := v.(float64); return ok },
|
||||
"integer": func(v any) bool { f, ok := v.(float64); return ok && f == float64(int64(f)) },
|
||||
"boolean": func(v any) bool { _, ok := v.(bool); return ok },
|
||||
"object": func(v any) bool { _, ok := v.(map[string]any); return ok },
|
||||
"array": func(v any) bool { _, ok := v.([]any); return ok },
|
||||
}
|
||||
|
||||
// checked is an operation as apply checked it: its instance's key, its operation and its body.
|
||||
type checked struct {
|
||||
key string
|
||||
op catalogue.LedgerOperation
|
||||
body map[string]any
|
||||
}
|
||||
|
||||
// check holds an operation to its type (ADR 0597 §2): a type the instance names, an operation it allows, a body that is an
|
||||
// object whose fields fit the schema, the key its effect names, and every digest matching its text.
|
||||
func (l *Ledger) check(r buscalls.ApplyRequest) (checked, error) {
|
||||
key, err := KeyOf(r.Instance)
|
||||
if err != nil {
|
||||
return checked{}, err
|
||||
}
|
||||
t, err := l.typeOf(r.Instance)
|
||||
if err != nil {
|
||||
return checked{}, err
|
||||
}
|
||||
op, err := operationOf(t, r.Operation)
|
||||
if err != nil {
|
||||
return checked{}, err
|
||||
}
|
||||
body := map[string]any{}
|
||||
if len(r.Body) > 0 && string(r.Body) != "null" {
|
||||
if err := json.Unmarshal(r.Body, &body); err != nil {
|
||||
return checked{}, fmt.Errorf("the operation %s's body is a JSON object: %w", r.Operation, err)
|
||||
}
|
||||
}
|
||||
for name, field := range t.Schema {
|
||||
v, has := body[name]
|
||||
if !has {
|
||||
continue
|
||||
}
|
||||
if fits, known := kinds[field.Type]; known && !fits(v) {
|
||||
return checked{}, fmt.Errorf("the operation %s's field %s is a %s in the type %s", r.Operation, name, field.Type, t.Name)
|
||||
}
|
||||
}
|
||||
for name, field := range t.Schema {
|
||||
if field.DigestOf == "" {
|
||||
continue
|
||||
}
|
||||
digest, hasDigest := body[name]
|
||||
text, hasText := body[field.DigestOf]
|
||||
if !hasDigest && !hasText {
|
||||
continue
|
||||
}
|
||||
s, isText := text.(string)
|
||||
if !hasDigest || !hasText || !isText || digest != Digest(s) {
|
||||
return checked{}, fmt.Errorf("the operation %s's %s is not the digest of its %s, so it is refused (novox/hq ADR 0597 §2)",
|
||||
r.Operation, name, field.DigestOf)
|
||||
}
|
||||
}
|
||||
switch op.Effect {
|
||||
case catalogue.EffectSetFields, catalogue.EffectAppend:
|
||||
case catalogue.EffectSetEntry, catalogue.EffectRemove:
|
||||
if op.Effect == catalogue.EffectRemove && op.Into == "" {
|
||||
break
|
||||
}
|
||||
if _, ok := body[op.Key].(string); !ok {
|
||||
return checked{}, fmt.Errorf("the operation %s names its entry by its field %s, a text, which its body lacks", r.Operation, op.Key)
|
||||
}
|
||||
default:
|
||||
return checked{}, fmt.Errorf("the operation %s's effect %q is not one the mesh offers (novox/hq ADR 0597 §2)", r.Operation, op.Effect)
|
||||
}
|
||||
return checked{key: key, op: op, body: body}, nil
|
||||
}
|
||||
|
||||
// effect is an operation's effect on a state (ADR 0597 §2): it copies only the fields the operation names, and the
|
||||
// operation's sequence and proven author where it says so; null is a removed instance.
|
||||
func effect(state json.RawMessage, op catalogue.LedgerOperation, body map[string]any, seq uint64, author buscalls.Author) (json.RawMessage, error) {
|
||||
s := map[string]any{}
|
||||
if len(state) > 0 && string(state) != "null" {
|
||||
if err := json.Unmarshal(state, &s); err != nil {
|
||||
return nil, fmt.Errorf("the state held is not an object: %w", err)
|
||||
}
|
||||
}
|
||||
copied := map[string]any{}
|
||||
for _, f := range op.Fields {
|
||||
if v, ok := body[f]; ok {
|
||||
copied[f] = v
|
||||
}
|
||||
}
|
||||
if op.Sequence {
|
||||
copied["sequence"] = seq
|
||||
}
|
||||
// Written last, so a body's own `author` never stands for the proven one (ADR 0597 §7).
|
||||
if op.Author {
|
||||
copied["author"] = author
|
||||
}
|
||||
switch op.Effect {
|
||||
case catalogue.EffectSetFields:
|
||||
for k, v := range copied {
|
||||
s[k] = v
|
||||
}
|
||||
case catalogue.EffectSetEntry:
|
||||
m, _ := s[op.Into].(map[string]any)
|
||||
if m == nil {
|
||||
m = map[string]any{}
|
||||
}
|
||||
m[body[op.Key].(string)] = copied
|
||||
s[op.Into] = m
|
||||
case catalogue.EffectAppend:
|
||||
list, _ := s[op.Into].([]any)
|
||||
s[op.Into] = append(list, copied)
|
||||
case catalogue.EffectRemove:
|
||||
if op.Into == "" {
|
||||
return json.RawMessage("null"), nil
|
||||
}
|
||||
if m, ok := s[op.Into].(map[string]any); ok {
|
||||
delete(m, body[op.Key].(string))
|
||||
}
|
||||
}
|
||||
return json.Marshal(s)
|
||||
}
|
||||
|
||||
// fold applies one kept operation to a record, as apply derived it.
|
||||
func (l *Ledger) fold(rec Record, k Kept) (Record, error) {
|
||||
var o buscalls.Operation
|
||||
if err := json.Unmarshal(k.Data, &o); err != nil {
|
||||
return rec, fmt.Errorf("the history's operation %d is not one: %w", k.Sequence, err)
|
||||
}
|
||||
t, err := l.typeOf(o.Instance)
|
||||
if err != nil {
|
||||
return rec, err
|
||||
}
|
||||
op, err := operationOf(t, o.Operation)
|
||||
if err != nil {
|
||||
return rec, err
|
||||
}
|
||||
body := map[string]any{}
|
||||
if len(o.Body) > 0 && string(o.Body) != "null" {
|
||||
if err := json.Unmarshal(o.Body, &body); err != nil {
|
||||
return rec, err
|
||||
}
|
||||
}
|
||||
if (op.Effect == catalogue.EffectSetEntry || (op.Effect == catalogue.EffectRemove && op.Into != "")) && !isText(body[op.Key]) {
|
||||
return rec, fmt.Errorf("the history's operation %d names no entry", k.Sequence)
|
||||
}
|
||||
state, err := effect(rec.State, op, body, k.Sequence, o.Author)
|
||||
if err != nil {
|
||||
return rec, err
|
||||
}
|
||||
return Record{Sequence: k.Sequence, State: state}, nil
|
||||
}
|
||||
|
||||
func isText(v any) bool { _, ok := v.(string); return ok }
|
||||
|
||||
// replay folds the instance's operations after rec's sequence, up to upto (every one when 0), onto rec.
|
||||
func (l *Ledger) replay(key string, rec Record, upto uint64) (Record, error) {
|
||||
kept, err := l.History.Read(key, rec.Sequence+1, 0)
|
||||
if err != nil {
|
||||
return rec, err
|
||||
}
|
||||
for _, k := range kept {
|
||||
if upto != 0 && k.Sequence > upto {
|
||||
break
|
||||
}
|
||||
if k.Sequence <= rec.Sequence {
|
||||
continue
|
||||
}
|
||||
if rec, err = l.fold(rec, k); err != nil {
|
||||
return rec, err
|
||||
}
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// readState is the instance's record in the state and its revision; an empty record when it has none.
|
||||
func (l *Ledger) readState(key string) (Record, uint64, error) {
|
||||
raw, rev, err := l.State.Get(key)
|
||||
if err != nil || raw == nil {
|
||||
return Record{}, rev, err
|
||||
}
|
||||
var rec Record
|
||||
if err := json.Unmarshal(raw, &rec); err != nil {
|
||||
return Record{}, rev, fmt.Errorf("the state of %s is not a record: %w", key, err)
|
||||
}
|
||||
return rec, rev, nil
|
||||
}
|
||||
|
||||
// readSnapshot is the instance's snapshot and the sequence it was written at; an empty record when it has none.
|
||||
func (l *Ledger) readSnapshot(key string) (Record, uint64, error) {
|
||||
raw, at, err := l.Snapshots.Get(key)
|
||||
if err != nil || raw == nil {
|
||||
return Record{}, at, err
|
||||
}
|
||||
var rec Record
|
||||
if err := json.Unmarshal(raw, &rec); err != nil {
|
||||
return Record{}, at, fmt.Errorf("the snapshot of %s is not a record: %w", key, err)
|
||||
}
|
||||
return rec, at, nil
|
||||
}
|
||||
|
||||
// derive is the instance's record caught up with its history up to last: from the state, or from the snapshot when it
|
||||
// is further on (an instance whose state was lost, or whose operations expired, ADR 0597 §8).
|
||||
func (l *Ledger) derive(key string, base Record, last uint64) (Record, error) {
|
||||
if base.Sequence >= last {
|
||||
return base, nil
|
||||
}
|
||||
snap, _, err := l.readSnapshot(key)
|
||||
if err != nil {
|
||||
return base, err
|
||||
}
|
||||
if snap.Sequence > base.Sequence {
|
||||
base = snap
|
||||
}
|
||||
return l.replay(key, base, last)
|
||||
}
|
||||
|
||||
// settle writes want into the state at the revision read (ADR 0597 §5): a lost update re-reads, and catches the state
|
||||
// read up to want's sequence from the history, never appending again; a state already further on is left as it is. It
|
||||
// answers what the state holds after it, and whether it is behind because the bucket is full (ADR 0597 §6).
|
||||
func (l *Ledger) settle(key string, want Record, rev uint64) (Record, bool, error) {
|
||||
for tries := 0; tries < 16; tries++ {
|
||||
raw, err := json.Marshal(want)
|
||||
if err != nil {
|
||||
return want, false, err
|
||||
}
|
||||
_, err = l.State.Write(key, raw, rev)
|
||||
switch {
|
||||
case err == nil:
|
||||
return want, false, nil
|
||||
case errors.Is(err, ErrFull):
|
||||
return want, true, nil
|
||||
case !errors.Is(err, ErrChanged):
|
||||
return want, false, err
|
||||
}
|
||||
cur, curRev, err := l.readState(key)
|
||||
if err != nil {
|
||||
return want, false, err
|
||||
}
|
||||
if cur.Sequence >= want.Sequence {
|
||||
return cur, false, nil
|
||||
}
|
||||
if want, err = l.replay(key, cur, want.Sequence); err != nil {
|
||||
return want, false, err
|
||||
}
|
||||
rev = curRev
|
||||
}
|
||||
return want, false, fmt.Errorf("the state of %s kept being written by other writers; the operation is in its history and the next apply catches it up", key)
|
||||
}
|
||||
|
||||
// writeSnapshot writes rec as the instance's snapshot unless the snapshot is already as far on (ADR 0597 §8).
|
||||
func (l *Ledger) writeSnapshot(key string, rec Record) error {
|
||||
for tries := 0; tries < 16; tries++ {
|
||||
snap, at, err := l.readSnapshot(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if at != 0 && snap.Sequence >= rec.Sequence {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(rec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = l.Snapshots.Put(key, raw, at); err == nil || !errors.Is(err, ErrRaced) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the snapshot of %s kept being written by other writers", key)
|
||||
}
|
||||
|
||||
// Apply applies one operation to one instance, stamped with author (ADR 0597 §3, §5, §6, §8; design 25 § 12.4 steps 1-5):
|
||||
// checked against its type, appended with the instance's last sequence expected, the state derived by its effect and
|
||||
// written at the revision read, and the snapshot written. A racing writer's append is refused; a full state bucket
|
||||
// answers taken with the state behind.
|
||||
func (l *Ledger) Apply(r buscalls.ApplyRequest, author buscalls.Author) (buscalls.Applied, error) {
|
||||
c, err := l.check(r)
|
||||
if err != nil {
|
||||
return buscalls.Applied{}, err
|
||||
}
|
||||
last, err := l.History.Last(c.key)
|
||||
if err != nil {
|
||||
return buscalls.Applied{}, err
|
||||
}
|
||||
held, rev, err := l.readState(c.key)
|
||||
if err != nil {
|
||||
return buscalls.Applied{}, err
|
||||
}
|
||||
base, err := l.derive(c.key, held, last)
|
||||
if err != nil {
|
||||
return buscalls.Applied{}, err
|
||||
}
|
||||
body := r.Body
|
||||
if len(body) == 0 {
|
||||
body = json.RawMessage("{}")
|
||||
}
|
||||
op, err := json.Marshal(buscalls.Operation{Instance: r.Instance, Operation: r.Operation, Body: body, Author: author, At: l.now()})
|
||||
if err != nil {
|
||||
return buscalls.Applied{}, err
|
||||
}
|
||||
seq, err := l.History.Append(c.key, op, last)
|
||||
if errors.Is(err, ErrRaced) {
|
||||
return buscalls.Applied{}, raced{r.Instance}
|
||||
}
|
||||
if err != nil {
|
||||
return buscalls.Applied{}, err
|
||||
}
|
||||
state, err := effect(base.State, c.op, c.body, seq, author)
|
||||
if err != nil {
|
||||
return buscalls.Applied{}, err
|
||||
}
|
||||
// From here the operation is taken: it is in the history, and a fault below is the state's to catch up, said here
|
||||
// and answered as the state behind, never as a refusal of what later takes effect (ADR 0597 §6).
|
||||
now, behind, err := l.settle(c.key, Record{Sequence: seq, State: state}, rev)
|
||||
if err != nil {
|
||||
l.say("[mesh-tools] %s's ledger %s took operation %d on %s and its state could not follow: %v; the next apply "+
|
||||
"or a rebuild catches it up", l.Module, l.Name, seq, r.Instance, err)
|
||||
behind, now = true, Record{Sequence: seq, State: state}
|
||||
}
|
||||
if behind {
|
||||
now = Record{Sequence: seq, State: state}
|
||||
}
|
||||
if err := l.writeSnapshot(c.key, now); err != nil {
|
||||
l.say("[mesh-tools] %s's ledger %s took operation %d on %s and its snapshot could not be written: %v; the next "+
|
||||
"apply or a snapshot repair writes it", l.Module, l.Name, seq, r.Instance, err)
|
||||
}
|
||||
return buscalls.Applied{Taken: true, Sequence: seq, StateBehind: behind}, nil
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package ledger
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
var owner = buscalls.Author{Account: "ledger.anchor.m"}
|
||||
|
||||
// 1966/AC13, 1966/AC14 and the checks of an operation against its type (ADR 0597 §2, §5, §6, §8; design 25 § 12.4).
|
||||
func TestApplyAppendsDerivesAndSnapshots(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
run func(t *testing.T, l *Ledger, f fakes)
|
||||
}{
|
||||
{"1966/AC13 an instance at sequence 4 is taken with 5, its state and snapshot the effect's result", func(t *testing.T, l *Ledger, f fakes) {
|
||||
for i := 1; i <= 4; i++ {
|
||||
applied(t, l, "decision/0290", "note", fmt.Sprintf(`{"says":"n%d"}`, i), owner)
|
||||
}
|
||||
if last, _ := f.history.Last("decision.0290"); last != 4 {
|
||||
t.Fatalf("the instance is at %d, want 4", last)
|
||||
}
|
||||
got := applied(t, l, "decision/0290", "title", `{"title":"A ledger","ignored":"x"}`, owner)
|
||||
if !got.Taken || got.Sequence != 5 || got.StateBehind {
|
||||
t.Fatalf("answered %+v, want taken with 5", got)
|
||||
}
|
||||
raw, _, _ := f.state.Get("decision.0290")
|
||||
state := record(t, raw)
|
||||
want := `{"author":{"account":"ledger.anchor.m"},"notes":[{"says":"n1"},{"says":"n2"},{"says":"n3"},{"says":"n4"}],"sequence":5,"title":"A ledger"}`
|
||||
if state.Sequence != 5 || string(state.State) != want {
|
||||
t.Fatalf("the state is %d %s, want 5 %s", state.Sequence, state.State, want)
|
||||
}
|
||||
snap, _, _ := f.snapshots.Get("decision.0290")
|
||||
if string(snap) != string(raw) {
|
||||
t.Fatalf("the snapshot is %s, the state %s", snap, raw)
|
||||
}
|
||||
}},
|
||||
{"1966/AC14 a full state bucket answers taken and state behind, and the next apply after room catches up", func(t *testing.T, l *Ledger, f fakes) {
|
||||
applied(t, l, "decision/0290", "note", `{"says":"first"}`, owner)
|
||||
f.state.Full = true
|
||||
got := applied(t, l, "decision/0290", "note", `{"says":"while full"}`, owner)
|
||||
if !got.Taken || got.Sequence != 2 || !got.StateBehind {
|
||||
t.Fatalf("answered %+v, want taken with 2 and state behind", got)
|
||||
}
|
||||
if raw, _, _ := f.state.Get("decision.0290"); record(t, raw).Sequence != 1 {
|
||||
t.Fatalf("the full state moved: %s", raw)
|
||||
}
|
||||
f.state.Full = false
|
||||
got = applied(t, l, "decision/0290", "note", `{"says":"after room"}`, owner)
|
||||
raw, _, _ := f.state.Get("decision.0290")
|
||||
state := record(t, raw)
|
||||
if got.StateBehind || state.Sequence != 3 || string(state.State) != `{"notes":[{"says":"first"},{"says":"while full"},{"says":"after room"}]}` {
|
||||
t.Fatalf("after room: %+v, the state %d %s", got, state.Sequence, state.State)
|
||||
}
|
||||
}},
|
||||
{"each effect: set an entry, remove it, remove the instance", func(t *testing.T, l *Ledger, f fakes) {
|
||||
text := "the section's text"
|
||||
applied(t, l, "decision/1", "section", fmt.Sprintf(`{"id":"a","text":%q,"digest":%q}`, text, Digest(text)), owner)
|
||||
applied(t, l, "decision/1", "section", fmt.Sprintf(`{"id":"b","text":"b","digest":%q}`, Digest("b")), owner)
|
||||
applied(t, l, "decision/1", "drop-section", `{"id":"a"}`, owner)
|
||||
raw, _, _ := f.state.Get("decision.1")
|
||||
if got := string(record(t, raw).State); got != fmt.Sprintf(`{"sections":{"b":{"digest":%q}}}`, Digest("b")) {
|
||||
t.Fatalf("the state is %s", got)
|
||||
}
|
||||
applied(t, l, "decision/1", "withdraw", `{}`, owner)
|
||||
raw, _, _ = f.state.Get("decision.1")
|
||||
if got := record(t, raw); got.Sequence != 4 || string(got.State) != "null" {
|
||||
t.Fatalf("the removed instance is %d %s", got.Sequence, got.State)
|
||||
}
|
||||
}},
|
||||
{"an operation outside its type, a body off its schema and a digest off its text are refused", func(t *testing.T, l *Ledger, f fakes) {
|
||||
for _, c := range []struct{ instance, operation, body, refusal string }{
|
||||
{"listing/a17", "title", `{}`, `no type "listing"`},
|
||||
{"decision/0290", "accept", `{}`, `no operation "accept"`},
|
||||
{"decision/0290", "title", `{"title":7}`, "field title is a string"},
|
||||
{"decision/0290", "section", `{"id":"a","text":"x","digest":"sha256:00"}`, "is not the digest of its text"},
|
||||
{"decision/0290", "section", `{"text":"x"}`, "is not the digest"},
|
||||
{"decision/0290", "drop-section", `{}`, "names its entry by its field id"},
|
||||
{"decision", "title", `{}`, "is not an instance"},
|
||||
{"decision/a.b", "title", `{}`, "is not an instance"},
|
||||
} {
|
||||
_, err := l.Apply(buscalls.ApplyRequest{Instance: c.instance, Operation: c.operation, Body: json.RawMessage(c.body)}, owner)
|
||||
if err == nil || !strings.Contains(err.Error(), c.refusal) {
|
||||
t.Errorf("%s %s %s: %v, want %q", c.instance, c.operation, c.body, err, c.refusal)
|
||||
}
|
||||
}
|
||||
if last, _ := f.history.Last("decision.0290"); last != 0 {
|
||||
t.Fatalf("a refused operation was appended: %d", last)
|
||||
}
|
||||
}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
l, f := newLedger()
|
||||
c.run(t, l, f)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package ledger
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// The memory stores keep what the bus keeps, as the bus answers it, for a test of a ledger without a bus: a stream whose
|
||||
// sequence counts across every instance, a bucket with revisions that may be full, and snapshots written with an expected
|
||||
// sequence.
|
||||
|
||||
// MemoryHistory is a history in memory.
|
||||
type MemoryHistory struct {
|
||||
mu sync.Mutex
|
||||
seq uint64
|
||||
kept []struct {
|
||||
key string
|
||||
Kept
|
||||
}
|
||||
}
|
||||
|
||||
// Matches is whether a key pattern (`*` one name, `>` the rest) names a key.
|
||||
func Matches(pattern, key string) bool {
|
||||
p, k := strings.Split(pattern, "."), strings.Split(key, ".")
|
||||
for i, part := range p {
|
||||
if part == ">" {
|
||||
return len(k) > i
|
||||
}
|
||||
if i >= len(k) || (part != "*" && part != k[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(k)
|
||||
}
|
||||
|
||||
func (h *MemoryHistory) Last(key string) (uint64, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
last := uint64(0)
|
||||
for _, k := range h.kept {
|
||||
if k.key == key {
|
||||
last = k.Sequence
|
||||
}
|
||||
}
|
||||
return last, nil
|
||||
}
|
||||
|
||||
func (h *MemoryHistory) Append(key string, op []byte, expect uint64) (uint64, error) {
|
||||
last, _ := h.Last(key)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if last != expect {
|
||||
return 0, ErrRaced
|
||||
}
|
||||
h.seq++
|
||||
h.kept = append(h.kept, struct {
|
||||
key string
|
||||
Kept
|
||||
}{key, Kept{Sequence: h.seq, Data: op}})
|
||||
return h.seq, nil
|
||||
}
|
||||
|
||||
func (h *MemoryHistory) Read(keys string, from uint64, limit int) ([]Kept, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
var out []Kept
|
||||
for _, k := range h.kept {
|
||||
if k.Sequence >= from && Matches(keys, k.key) && (limit == 0 || len(out) < limit) {
|
||||
out = append(out, k.Kept)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Expire drops every operation of the history, as a ledger's keeping days end them.
|
||||
func (h *MemoryHistory) Expire() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.kept = nil
|
||||
}
|
||||
|
||||
type memoryValue struct {
|
||||
value []byte
|
||||
rev uint64
|
||||
}
|
||||
|
||||
// MemoryState is a state bucket in memory.
|
||||
type MemoryState struct {
|
||||
mu sync.Mutex
|
||||
rev uint64
|
||||
// Full makes every write refused as a full bucket's.
|
||||
Full bool
|
||||
kv map[string]memoryValue
|
||||
}
|
||||
|
||||
func (s *MemoryState) Get(key string) ([]byte, uint64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v := s.kv[key]
|
||||
return v.value, v.rev, nil
|
||||
}
|
||||
|
||||
func (s *MemoryState) Write(key string, value []byte, revision uint64) (uint64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.Full {
|
||||
return 0, ErrFull
|
||||
}
|
||||
if s.kv[key].rev != revision {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
if s.kv == nil {
|
||||
s.kv = map[string]memoryValue{}
|
||||
}
|
||||
s.rev++
|
||||
s.kv[key] = memoryValue{value, s.rev}
|
||||
return s.rev, nil
|
||||
}
|
||||
|
||||
func (s *MemoryState) Keys() ([]string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []string
|
||||
for k := range s.kv {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// MemorySnapshots are snapshots in memory.
|
||||
type MemorySnapshots struct {
|
||||
mu sync.Mutex
|
||||
at uint64
|
||||
kv map[string]memoryValue
|
||||
}
|
||||
|
||||
func (s *MemorySnapshots) Get(key string) ([]byte, uint64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v := s.kv[key]
|
||||
return v.value, v.rev, nil
|
||||
}
|
||||
|
||||
func (s *MemorySnapshots) Put(key string, value []byte, expect uint64) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.kv[key].rev != expect {
|
||||
return ErrRaced
|
||||
}
|
||||
if s.kv == nil {
|
||||
s.kv = map[string]memoryValue{}
|
||||
}
|
||||
s.at++
|
||||
s.kv[key] = memoryValue{value, s.at}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MemoryIndex is an index's bucket in memory.
|
||||
type MemoryIndex struct {
|
||||
mu sync.Mutex
|
||||
kv map[string][]byte
|
||||
}
|
||||
|
||||
func (x *MemoryIndex) Entries(pattern string, limit int) ([]Indexed, error) {
|
||||
x.mu.Lock()
|
||||
defer x.mu.Unlock()
|
||||
if pattern == "" {
|
||||
pattern = ">"
|
||||
}
|
||||
var keys []string
|
||||
for k := range x.kv {
|
||||
if Matches(pattern, k) {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var out []Indexed
|
||||
for _, k := range keys {
|
||||
if limit > 0 && len(out) >= limit {
|
||||
break
|
||||
}
|
||||
out = append(out, Indexed{Key: k, Value: x.kv[k]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (x *MemoryIndex) Put(key string, value []byte) error {
|
||||
x.mu.Lock()
|
||||
defer x.mu.Unlock()
|
||||
if x.kv == nil {
|
||||
x.kv = map[string][]byte{}
|
||||
}
|
||||
x.kv[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *MemoryIndex) Delete(key string) error {
|
||||
x.mu.Lock()
|
||||
defer x.mu.Unlock()
|
||||
delete(x.kv, key)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package ledger
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// Reading a ledger (ADR 0597 §10, design 25 § 12.5; issue 1966 T3): an instance's state, its history, a watch of its
|
||||
// states, and a declared index. Only the owning module reads its ledger: the runtime opens a ledger for the module whose
|
||||
// bundle asks, never one the request names.
|
||||
|
||||
// Index is one declared index's bucket, kept by a watcher in the owning module's process and rebuilt by reindex (ADR
|
||||
// 0597 §4).
|
||||
type Index interface {
|
||||
// Entries is the entries whose key a pattern names (`*` one name, `>` the rest; empty is every key), at most limit
|
||||
// (every one when 0), each as the bucket holds it.
|
||||
Entries(pattern string, limit int) ([]Indexed, error)
|
||||
// Put writes one entry; Delete removes one.
|
||||
Put(key string, value []byte) error
|
||||
Delete(key string) error
|
||||
}
|
||||
|
||||
// Indexed is one key of an index's bucket and its value, a buscalls.IndexEntry as JSON.
|
||||
type Indexed struct {
|
||||
Key string
|
||||
Value []byte
|
||||
}
|
||||
|
||||
// Watcher is a state bucket that hands each instance's record as it is and as it changes, under a key pattern.
|
||||
type Watcher interface {
|
||||
Watch(pattern string, deliver func(key string, value []byte, current bool) error) (func(), error)
|
||||
}
|
||||
|
||||
// HistoryMost is the most operations one history read answers.
|
||||
const HistoryMost = 1000
|
||||
|
||||
// Get is an instance's state as the state holds it, and the sequence of its last operation in it; null and 0 for an
|
||||
// instance with none.
|
||||
func (l *Ledger) Get(r buscalls.InstanceRequest) (buscalls.InstanceState, error) {
|
||||
key, err := KeyOf(r.Instance)
|
||||
if err != nil {
|
||||
return buscalls.InstanceState{}, err
|
||||
}
|
||||
rec, _, err := l.readState(key)
|
||||
if err != nil {
|
||||
return buscalls.InstanceState{}, err
|
||||
}
|
||||
state := rec.State
|
||||
if len(state) == 0 {
|
||||
state = json.RawMessage("null")
|
||||
}
|
||||
return buscalls.InstanceState{Instance: r.Instance, State: state, Sequence: rec.Sequence}, nil
|
||||
}
|
||||
|
||||
// operationsOf reads kept operations as the history answers them, each with its sequence.
|
||||
func operationsOf(kept []Kept) ([]buscalls.Operation, error) {
|
||||
out := make([]buscalls.Operation, 0, len(kept))
|
||||
for _, k := range kept {
|
||||
var o buscalls.Operation
|
||||
if err := json.Unmarshal(k.Data, &o); err != nil {
|
||||
return nil, fmt.Errorf("the history's operation %d is not one: %w", k.Sequence, err)
|
||||
}
|
||||
o.Sequence = k.Sequence
|
||||
out = append(out, o)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ReadHistory is an instance's operations from a sequence on, oldest first, each with its sequence and its proven author.
|
||||
func (l *Ledger) ReadHistory(r buscalls.HistoryRequest) (buscalls.HistoryPage, error) {
|
||||
key, err := KeyOf(r.Instance)
|
||||
if err != nil {
|
||||
return buscalls.HistoryPage{}, err
|
||||
}
|
||||
limit := r.Limit
|
||||
if limit <= 0 || limit > HistoryMost {
|
||||
limit = HistoryMost
|
||||
}
|
||||
last, err := l.History.Last(key)
|
||||
if err != nil {
|
||||
return buscalls.HistoryPage{}, err
|
||||
}
|
||||
kept, err := l.History.Read(key, r.From, limit+1)
|
||||
if err != nil {
|
||||
return buscalls.HistoryPage{}, err
|
||||
}
|
||||
page := buscalls.HistoryPage{Last: last}
|
||||
if len(kept) > limit {
|
||||
page.Next = kept[limit].Sequence
|
||||
kept = kept[:limit]
|
||||
}
|
||||
if page.Operations, err = operationsOf(kept); err != nil {
|
||||
return buscalls.HistoryPage{}, err
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// Watch hands deliver the state of every instance Instances names (`*` one name, `>` the rest, in an instance's path),
|
||||
// the current ones first, then each change.
|
||||
func (l *Ledger) Watch(r buscalls.LedgerWatchRequest, deliver func(buscalls.InstanceChange) error) (func(), error) {
|
||||
w, ok := l.State.(Watcher)
|
||||
if !ok {
|
||||
return nil, errors.New("this ledger's state cannot be watched")
|
||||
}
|
||||
pattern := ">"
|
||||
if r.Instances != "" {
|
||||
pattern = keyPattern(r.Instances)
|
||||
}
|
||||
return w.Watch(pattern, func(key string, value []byte, current bool) error {
|
||||
var rec Record
|
||||
if value != nil {
|
||||
if err := json.Unmarshal(value, &rec); err != nil {
|
||||
return fmt.Errorf("the state of %s is not a record: %w", key, err)
|
||||
}
|
||||
}
|
||||
state := rec.State
|
||||
if len(state) == 0 {
|
||||
state = json.RawMessage("null")
|
||||
}
|
||||
return deliver(buscalls.InstanceChange{Watch: r.Watch, Instance: InstanceOf(key), State: state, Sequence: rec.Sequence,
|
||||
Current: current})
|
||||
})
|
||||
}
|
||||
|
||||
// keyPattern is an instance pattern as a key pattern: its path's parts joined by dots.
|
||||
func keyPattern(instances string) string {
|
||||
out := []byte(instances)
|
||||
for i, c := range out {
|
||||
if c == '/' {
|
||||
out[i] = '.'
|
||||
}
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// index is a declared index by its name, or the reason there is none.
|
||||
func (l *Ledger) index(name string) (Index, error) {
|
||||
if ix, ok := l.Indexes[name]; ok {
|
||||
return ix, nil
|
||||
}
|
||||
return nil, fmt.Errorf("%s's ledger %s declares no index %q", l.Module, l.Name, name)
|
||||
}
|
||||
|
||||
// Query is a declared index's entries whose key Key names.
|
||||
func (l *Ledger) Query(r buscalls.QueryRequest) (buscalls.QueryAnswer, error) {
|
||||
ix, err := l.index(r.Index)
|
||||
if err != nil {
|
||||
return buscalls.QueryAnswer{}, err
|
||||
}
|
||||
held, err := ix.Entries(r.Key, r.Limit)
|
||||
if err != nil {
|
||||
return buscalls.QueryAnswer{}, err
|
||||
}
|
||||
entries := make([]buscalls.IndexEntry, 0, len(held))
|
||||
for _, h := range held {
|
||||
var e buscalls.IndexEntry
|
||||
if err := json.Unmarshal(h.Value, &e); err != nil {
|
||||
return buscalls.QueryAnswer{}, fmt.Errorf("the index %s's entry %s is not one: %w", r.Index, h.Key, err)
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
return buscalls.QueryAnswer{Entries: entries}, nil
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package ledger
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// 1966/AC15, the other reads of a ledger (get, query) and its repairs (ADR 0597 §8, §10; design 25 § 12.5).
|
||||
func TestReadsAndRepairsOfALedger(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
run func(t *testing.T, l *Ledger, f fakes)
|
||||
}{
|
||||
{"1966/AC15 history answers an instance's operations oldest first with sequence and author", func(t *testing.T, l *Ledger, f fakes) {
|
||||
op := buscalls.Author{Account: "ledger.anchor.m", Caller: "op@shanks"}
|
||||
applied(t, l, "decision/1", "note", `{"says":"one"}`, owner)
|
||||
applied(t, l, "decision/2", "note", `{"says":"elsewhere"}`, owner)
|
||||
applied(t, l, "decision/1", "title", `{"title":"Two"}`, op)
|
||||
applied(t, l, "decision/1", "note", `{"says":"three"}`, owner)
|
||||
page, err := l.ReadHistory(buscalls.HistoryRequest{Instance: "decision/1", Limit: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, o := range page.Operations {
|
||||
got = append(got, fmt.Sprintf("%s@%d %s %s", o.Operation, o.Sequence, o.Author.Account, o.Author.Caller))
|
||||
}
|
||||
if strings.Join(got, "; ") != "note@1 ledger.anchor.m ; title@3 ledger.anchor.m op@shanks" || page.Next != 4 || page.Last != 4 {
|
||||
t.Fatalf("the history is %q, next %d, last %d", got, page.Next, page.Last)
|
||||
}
|
||||
rest, _ := l.ReadHistory(buscalls.HistoryRequest{Instance: "decision/1", From: page.Next})
|
||||
if len(rest.Operations) != 1 || rest.Operations[0].Sequence != 4 || rest.Next != 0 {
|
||||
t.Fatalf("read on: %+v", rest)
|
||||
}
|
||||
}},
|
||||
{"get answers the state and its sequence, null for an instance with none", func(t *testing.T, l *Ledger, f fakes) {
|
||||
applied(t, l, "decision/1", "title", `{"title":"One"}`, owner)
|
||||
got, err := l.Get(buscalls.InstanceRequest{Instance: "decision/1"})
|
||||
if err != nil || got.Sequence != 1 || !strings.Contains(string(got.State), `"title":"One"`) {
|
||||
t.Fatalf("got %+v, %v", got, err)
|
||||
}
|
||||
if none, err := l.Get(buscalls.InstanceRequest{Instance: "decision/9"}); err != nil || string(none.State) != "null" {
|
||||
t.Fatalf("none is %+v, %v", none, err)
|
||||
}
|
||||
}},
|
||||
{"query reads a declared index by key, and an undeclared one is refused", func(t *testing.T, l *Ledger, f fakes) {
|
||||
_ = f.index.Put("One.decision.1", []byte(`{"instance":"decision/1","sequence":1}`))
|
||||
_ = f.index.Put("Two.decision.2", []byte(`{"instance":"decision/2","sequence":2}`))
|
||||
got, err := l.Query(buscalls.QueryRequest{Index: "by-title", Key: "One.>"})
|
||||
if err != nil || len(got.Entries) != 1 || got.Entries[0] != (buscalls.IndexEntry{Instance: "decision/1", Sequence: 1}) {
|
||||
t.Fatalf("queried %+v, %v", got, err)
|
||||
}
|
||||
if _, err := l.Query(buscalls.QueryRequest{Index: "by-date"}); err == nil || !strings.Contains(err.Error(), `no index "by-date"`) {
|
||||
t.Fatalf("an undeclared index: %v", err)
|
||||
}
|
||||
}},
|
||||
{"a lost state rebuilds from the history, an expired one from its snapshot, and lag says what is behind", func(t *testing.T, l *Ledger, f fakes) {
|
||||
applied(t, l, "decision/1", "title", `{"title":"One"}`, owner)
|
||||
applied(t, l, "decision/1", "note", `{"says":"kept"}`, owner)
|
||||
want, _, _ := f.state.Get("decision.1")
|
||||
f.state.kv = nil
|
||||
lag, _ := l.Lag(buscalls.LedgerRequest{})
|
||||
if len(lag.Instances) != 1 || lag.Instances[0].History != 2 || lag.Instances[0].State != 0 || lag.Instances[0].Snapshot != 2 {
|
||||
t.Fatalf("lag says %+v", lag)
|
||||
}
|
||||
if got, err := l.Rebuild(buscalls.RebuildRequest{}); err != nil || got.Written != 1 {
|
||||
t.Fatalf("rebuilt %+v, %v", got, err)
|
||||
}
|
||||
if got, _, _ := f.state.Get("decision.1"); string(record(t, got).State) != string(record(t, want).State) {
|
||||
t.Fatalf("rebuilt from the history %s, want %s", got, want)
|
||||
}
|
||||
f.state.kv = nil
|
||||
f.history.Expire()
|
||||
if got, err := l.Rebuild(buscalls.RebuildRequest{Instance: "decision/1", From: buscalls.RebuildFromSnapshot}); err != nil || got.Written != 1 {
|
||||
t.Fatalf("rebuilt %+v, %v", got, err)
|
||||
}
|
||||
if got, _, _ := f.state.Get("decision.1"); string(got) != string(want) {
|
||||
t.Fatalf("rebuilt from the snapshot %s, want %s", got, want)
|
||||
}
|
||||
}},
|
||||
{"reindex builds a declared index from the states, and lag says an index behind until it does", func(t *testing.T, l *Ledger, f fakes) {
|
||||
applied(t, l, "decision/1", "title", `{"title":"One"}`, owner)
|
||||
applied(t, l, "decision/2", "note", `{"says":"untitled"}`, owner)
|
||||
_ = f.index.Put("stale.decision.9", []byte(`{}`))
|
||||
if lag, _ := l.Lag(buscalls.LedgerRequest{}); len(lag.Indexes) != 1 || lag.Indexes[0].Behind != 1 {
|
||||
t.Fatalf("lag says %+v", lag)
|
||||
}
|
||||
if got, err := l.Reindex(buscalls.ReindexRequest{Index: "by-title"}); err != nil || got.Written != 1 {
|
||||
t.Fatalf("reindexed %+v, %v", got, err)
|
||||
}
|
||||
entries, _ := f.index.Entries("", 0)
|
||||
if len(entries) != 1 || entries[0].Key != buscalls.IndexKey([]string{"One"}, "decision/1") || string(entries[0].Value) != `{"instance":"decision/1","sequence":1}` {
|
||||
t.Fatalf("the index holds %+v", entries)
|
||||
}
|
||||
if lag, _ := l.Lag(buscalls.LedgerRequest{}); len(lag.Indexes) != 0 || len(lag.Instances) != 0 {
|
||||
t.Fatalf("lag after reindex says %+v", lag)
|
||||
}
|
||||
}},
|
||||
{"snapshot rewrites a snapshot behind its state", func(t *testing.T, l *Ledger, f fakes) {
|
||||
applied(t, l, "decision/1", "title", `{"title":"One"}`, owner)
|
||||
f.snapshots.kv = nil
|
||||
if got, err := l.Snapshot(buscalls.InstanceRequest{}); err != nil || got.Written != 1 {
|
||||
t.Fatalf("snapshot %+v, %v", got, err)
|
||||
}
|
||||
state, _, _ := f.state.Get("decision.1")
|
||||
if snap, _, _ := f.snapshots.Get("decision.1"); string(snap) != string(state) {
|
||||
t.Fatalf("the snapshot %s, the state %s", snap, state)
|
||||
}
|
||||
}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
l, f := newLedger()
|
||||
c.run(t, l, f)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
package ledger
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// The ledger's repairs (ADR 0597 §8, §10; design 25 § 12.5; issue 1966 T4): rebuild an instance's state from its history
|
||||
// or its snapshot, rewrite a snapshot behind its state, build a declared index again, and say what is behind its history.
|
||||
// The tool runner serves them as tools of every module owning a ledger, its own ledgers only (the runtime's repairs.go).
|
||||
|
||||
// instances is every instance the ledger knows, by key: those its state holds and those its history names.
|
||||
func (l *Ledger) instances() ([]string, error) {
|
||||
seen := map[string]bool{}
|
||||
keys, err := l.State.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, k := range keys {
|
||||
seen[k] = true
|
||||
}
|
||||
kept, err := l.History.Read(">", 1, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ops, err := operationsOf(kept)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, o := range ops {
|
||||
if k, err := KeyOf(o.Instance); err == nil {
|
||||
seen[k] = true
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for k := range seen {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// keysOf is the one instance a request names, or every instance when it names none.
|
||||
func (l *Ledger) keysOf(instance string) ([]string, error) {
|
||||
if instance == "" {
|
||||
return l.instances()
|
||||
}
|
||||
k, err := KeyOf(instance)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{k}, nil
|
||||
}
|
||||
|
||||
// Rebuild derives the state of an instance (every instance when it names none) again, from its whole history or from its
|
||||
// snapshot and the operations after it, and writes it, and its snapshot when that is behind. A derivation further back
|
||||
// than the state holds is not written: the state never goes back (ADR 0597 §5).
|
||||
func (l *Ledger) Rebuild(r buscalls.RebuildRequest) (buscalls.Repaired, error) {
|
||||
from := r.From
|
||||
if from == "" {
|
||||
from = buscalls.RebuildFromHistory
|
||||
}
|
||||
if from != buscalls.RebuildFromHistory && from != buscalls.RebuildFromSnapshot {
|
||||
return buscalls.Repaired{}, fmt.Errorf("a rebuild is from %q or %q, not %q", buscalls.RebuildFromHistory, buscalls.RebuildFromSnapshot, from)
|
||||
}
|
||||
keys, err := l.keysOf(r.Instance)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
var out buscalls.Repaired
|
||||
for _, key := range keys {
|
||||
base := Record{}
|
||||
if from == buscalls.RebuildFromSnapshot {
|
||||
if base, _, err = l.readSnapshot(key); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
derived, err := l.replay(key, base, 0)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
held, rev, err := l.readState(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if derived.Sequence < held.Sequence {
|
||||
continue
|
||||
}
|
||||
now, _, err := l.settle(key, derived, rev)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if err := l.writeSnapshot(key, now); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Written++
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Snapshot rewrites the snapshot of an instance (every instance when it names none) that is behind its state.
|
||||
func (l *Ledger) Snapshot(r buscalls.InstanceRequest) (buscalls.Repaired, error) {
|
||||
keys, err := l.keysOf(r.Instance)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
var out buscalls.Repaired
|
||||
for _, key := range keys {
|
||||
held, _, err := l.readState(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
snap, at, err := l.readSnapshot(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if held.Sequence == 0 || (at != 0 && snap.Sequence >= held.Sequence) {
|
||||
continue
|
||||
}
|
||||
if err := l.writeSnapshot(key, held); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Written++
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// indexEntryOf is an instance's entry in a declared index, as reindex writes it and the owner's watcher keeps it
|
||||
// (mesh-sdk buscalls.IndexKey, buscalls.IndexEntry): its key the values of the fields the index is on, in their order,
|
||||
// then the instance; its value the instance and the sequence of the state it was taken from. False when the state holds
|
||||
// none of those fields. A value that is not text is written as its JSON.
|
||||
func indexEntryOf(on []string, key string, rec Record) (string, []byte, bool) {
|
||||
var state map[string]any
|
||||
if json.Unmarshal(rec.State, &state) != nil || state == nil {
|
||||
return "", nil, false
|
||||
}
|
||||
values := make([]string, 0, len(on))
|
||||
found := false
|
||||
for _, f := range on {
|
||||
v, has := state[f]
|
||||
found = found || has
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
raw, _ := json.Marshal(v)
|
||||
s = string(raw)
|
||||
}
|
||||
values = append(values, s)
|
||||
}
|
||||
if !found {
|
||||
return "", nil, false
|
||||
}
|
||||
instance := InstanceOf(key)
|
||||
value, _ := json.Marshal(buscalls.IndexEntry{Instance: instance, Sequence: rec.Sequence})
|
||||
return buscalls.IndexKey(values, instance), value, true
|
||||
}
|
||||
|
||||
// declared is the fields an index is on and the types declaring it.
|
||||
func (l *Ledger) declared(index string) ([]string, map[string]bool, error) {
|
||||
types := map[string]bool{}
|
||||
var on []string
|
||||
for _, t := range l.Types {
|
||||
for _, ix := range t.Indexes {
|
||||
if ix.Name == index {
|
||||
types[t.Name] = true
|
||||
on = ix.On
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(types) == 0 {
|
||||
return nil, nil, fmt.Errorf("%s's ledger %s declares no index %q", l.Module, l.Name, index)
|
||||
}
|
||||
return on, types, nil
|
||||
}
|
||||
|
||||
// wanted is every entry an index should hold, from the states of the types declaring it.
|
||||
func (l *Ledger) wanted(index string) (map[string][]byte, map[string]uint64, error) {
|
||||
on, types, err := l.declared(index)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
keys, err := l.instances()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
want, seqs := map[string][]byte{}, map[string]uint64{}
|
||||
for _, key := range keys {
|
||||
name, _, _ := strings.Cut(key, ".")
|
||||
if !types[name] {
|
||||
continue
|
||||
}
|
||||
held, _, err := l.readState(key)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if k, v, ok := indexEntryOf(on, key, held); ok {
|
||||
want[k], seqs[k] = v, held.Sequence
|
||||
}
|
||||
}
|
||||
return want, seqs, nil
|
||||
}
|
||||
|
||||
// Reindex builds a declared index again from the ledger's states, derived from its history: every entry it should hold
|
||||
// written, every other removed.
|
||||
func (l *Ledger) Reindex(r buscalls.ReindexRequest) (buscalls.Repaired, error) {
|
||||
ix, err := l.index(r.Index)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
want, _, err := l.wanted(r.Index)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
held, err := ix.Entries("", 0)
|
||||
if err != nil {
|
||||
return buscalls.Repaired{}, err
|
||||
}
|
||||
var out buscalls.Repaired
|
||||
for _, e := range held {
|
||||
if _, keep := want[e.Key]; !keep {
|
||||
if err := ix.Delete(e.Key); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(want))
|
||||
for k := range want {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if err := ix.Put(k, want[k]); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Written++
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Lag is every instance whose state or snapshot is behind its history, and how many entries each declared index lacks
|
||||
// or holds from an older state; nothing when nothing is behind.
|
||||
func (l *Ledger) Lag(buscalls.LedgerRequest) (buscalls.Lagged, error) {
|
||||
keys, err := l.instances()
|
||||
if err != nil {
|
||||
return buscalls.Lagged{}, err
|
||||
}
|
||||
var out buscalls.Lagged
|
||||
for _, key := range keys {
|
||||
last, err := l.History.Last(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
held, _, err := l.readState(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
snap, _, err := l.readSnapshot(key)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if held.Sequence < last || snap.Sequence < last {
|
||||
out.Instances = append(out.Instances, buscalls.InstanceLag{Instance: InstanceOf(key), History: last,
|
||||
State: held.Sequence, Snapshot: snap.Sequence})
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(l.Indexes))
|
||||
for name := range l.Indexes {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
if _, _, err := l.declared(name); err != nil {
|
||||
// A bucket bound for an index no type declares has nothing it should hold.
|
||||
continue
|
||||
}
|
||||
want, seqs, err := l.wanted(name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
held, err := l.Indexes[name].Entries("", 0)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
at := map[string]uint64{}
|
||||
for _, e := range held {
|
||||
var v buscalls.IndexEntry
|
||||
_ = json.Unmarshal(e.Value, &v)
|
||||
at[e.Key] = v.Sequence
|
||||
}
|
||||
behind := uint64(0)
|
||||
for k := range want {
|
||||
if s, has := at[k]; !has || s < seqs[k] {
|
||||
behind++
|
||||
}
|
||||
}
|
||||
if behind > 0 {
|
||||
out.Indexes = append(out.Indexes, buscalls.IndexLag{Index: name, Behind: behind})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package ledger
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// records is a ledger type of each effect: a title set, a section set as an entry, a note appended, a section dropped,
|
||||
// and the whole instance removed.
|
||||
var records = catalogue.LedgerType{
|
||||
Name: "decision",
|
||||
Schema: map[string]catalogue.LedgerField{
|
||||
"title": {Type: "string"}, "text": {Type: "string"}, "digest": {Type: "string", DigestOf: "text"},
|
||||
},
|
||||
Operations: []catalogue.LedgerOperation{
|
||||
{Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Sequence: true, Author: true},
|
||||
{Name: "section", Effect: catalogue.EffectSetEntry, Into: "sections", Key: "id", Fields: []string{"digest"}},
|
||||
{Name: "note", Effect: catalogue.EffectAppend, Into: "notes", Fields: []string{"says"}},
|
||||
{Name: "drop-section", Effect: catalogue.EffectRemove, Into: "sections", Key: "id"},
|
||||
{Name: "withdraw", Effect: catalogue.EffectRemove},
|
||||
},
|
||||
Indexes: []catalogue.LedgerIndex{{Name: "by-title", On: []string{"title"}}},
|
||||
}
|
||||
|
||||
type fakes struct {
|
||||
history *MemoryHistory
|
||||
state *MemoryState
|
||||
snapshots *MemorySnapshots
|
||||
index *MemoryIndex
|
||||
}
|
||||
|
||||
func newLedger() (*Ledger, fakes) {
|
||||
f := fakes{&MemoryHistory{}, &MemoryState{}, &MemorySnapshots{}, &MemoryIndex{}}
|
||||
return &Ledger{Module: "m", Name: "records", Types: []catalogue.LedgerType{records}, History: f.history, State: f.state,
|
||||
Snapshots: f.snapshots, Indexes: map[string]Index{"by-title": f.index}}, f
|
||||
}
|
||||
|
||||
// applied applies an operation and fails the test on a refusal.
|
||||
func applied(t *testing.T, l *Ledger, instance, operation, body string, author buscalls.Author) buscalls.Applied {
|
||||
t.Helper()
|
||||
got, err := l.Apply(buscalls.ApplyRequest{Ledger: l.Name, Instance: instance, Operation: operation, Body: json.RawMessage(body)}, author)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", operation, body, err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// record is the instance's record a store holds, read as a Record.
|
||||
func record(t *testing.T, raw []byte) Record {
|
||||
t.Helper()
|
||||
var r Record
|
||||
if raw != nil {
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package meshtest
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// bundlesWhy returns the reason bundles cannot be launched from a checkout rooted at root, or "" when they can: the
|
||||
// TypeScript and Python bundles the runtime and the console launch need node and the package
|
||||
// @novox/mesh-sdk, which comes from the package registry and is installed beside node-tools/package.json
|
||||
// (novox/hq ADR 0529 clause 1: what merge-check.sh said "NOT TESTED HERE" about is a skip with its reason).
|
||||
func bundlesWhy(root string, haveNode bool) string {
|
||||
if !haveNode {
|
||||
return "node is not installed"
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "node_modules", "@novox", "mesh-sdk")); err != nil {
|
||||
return "@novox/mesh-sdk is not installed in " + filepath.Join(root, "node_modules")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// NeedBundles skips the test, saying why, where the bundles it launches cannot run here.
|
||||
func NeedBundles(t *testing.T) {
|
||||
t.Helper()
|
||||
_, err := exec.LookPath("node")
|
||||
if why := bundlesWhy(moduleRoot(), err == nil); why != "" {
|
||||
t.Skip("NOT TESTED HERE: this test launches bundles, and " + why)
|
||||
}
|
||||
}
|
||||
|
||||
// moduleRoot is node-tools/, where package.json and node_modules are.
|
||||
func moduleRoot() string {
|
||||
_, here, _, _ := runtime.Caller(0)
|
||||
return filepath.Join(filepath.Dir(here), "..", "..")
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Package meshtest raises what the controller would, for tests against a real bus: the ASSIGNMENTS
|
||||
// Package meshtest raises what the mesh would, for tests against a real bus: the ASSIGNMENTS
|
||||
// and EVENTS streams, memberships issued by hand, and a fixture's path.
|
||||
//
|
||||
// **Every test has a bus of its own** (novox/hq ADR 0237, the controller's internal/testbus): a server
|
||||
// **Every test has a bus of its own** (novox/hq ADR 0237, the old testbus): a server
|
||||
// linked in at the release go.mod pins — the release the mesh runs, held so by a test beside this one —
|
||||
// started for the one test and gone after it. The packages shared one bus and had to run one at a time:
|
||||
// each test raised the streams afresh, and the console discovers every runtime that announces itself on
|
||||
@@ -11,6 +11,7 @@
|
||||
package meshtest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -20,8 +21,15 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
assigner "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
seatassigner "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
meshcatalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
@@ -67,7 +75,7 @@ func URL(t *testing.T) string {
|
||||
// buses are the running tests' buses, by test.
|
||||
var buses sync.Map
|
||||
|
||||
// Mesh is the controller's job, done by hand.
|
||||
// Mesh is the mesh's job, done by hand.
|
||||
type Mesh struct {
|
||||
nc *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
@@ -81,11 +89,10 @@ func New(t *testing.T) *Mesh {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, _ := nc.JetStream()
|
||||
for _, s := range []string{"ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.DeleteStream(s)
|
||||
}
|
||||
if _, err := js.AddStream(&nats.StreamConfig{Name: "ASSIGNMENTS", Subjects: []string{"mesh.assignment.>"},
|
||||
MaxMsgsPerSubject: 1, AllowDirect: true}); err != nil {
|
||||
_ = js.DeleteStream("EVENTS")
|
||||
// The grants part's state memberships, where a runtime reads what is issued it (ADR 0571 clause 10).
|
||||
_ = js.DeleteKeyValue(membership.GrantsBucket)
|
||||
if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: membership.GrantsBucket, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.AddStream(&nats.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
|
||||
@@ -95,15 +102,131 @@ func New(t *testing.T) *Mesh {
|
||||
return &Mesh{nc: nc, js: js}
|
||||
}
|
||||
|
||||
// Issue publishes a membership.
|
||||
// Issue writes a membership where the grants part keeps it, its state memberships, key `<node>.<module>` (novox/hq ADR
|
||||
// 0571 clause 7 and 10; mesh-sdk membership.GrantsBucket), the one place a runtime reads it. It writes with
|
||||
// mesh-sdk's encoder, the grants part's own, never json of this package's type: a value built here by hand agreed with
|
||||
// the reader's mistake and hid that the part wrote another shape (issue 774).
|
||||
func (m *Mesh) Issue(t *testing.T, mem bus.Membership) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(mem)
|
||||
if _, err := m.js.Publish(bus.MembershipSubject(mem.Node, mem.Module), body); err != nil {
|
||||
m.putGrants(t, mem, membership.EncodeGrantsMembership(sdkMembership(t, mem)))
|
||||
}
|
||||
|
||||
// IssueInGrantsWrapped writes a membership as the grants part wrote it before mesh-sdk's codec (until 2026-10-30, what
|
||||
// the live mesh holds until the part writes again): its pass record, the membership inside, published saying what
|
||||
// became of it ("published", "unchanged", or why it was held back).
|
||||
func (m *Mesh) IssueInGrantsWrapped(t *testing.T, mem bus.Membership, published string) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(map[string]any{"node": mem.Node, "module": mem.Module, "digest": "sha256:0",
|
||||
"membership": sdkMembership(t, mem), "published": published})
|
||||
m.putGrants(t, mem, body)
|
||||
}
|
||||
|
||||
func (m *Mesh) putGrants(t *testing.T, mem bus.Membership, body []byte) {
|
||||
t.Helper()
|
||||
kv, err := m.js.KeyValue(membership.GrantsBucket)
|
||||
if err != nil {
|
||||
if kv, err = m.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: membership.GrantsBucket, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := kv.Put(membership.GrantsKey(mem.Node, mem.Module), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// DropGrants deletes the grants part's state memberships, as a bus that has not made it yet holds none; Issue makes it again.
|
||||
func (m *Mesh) DropGrants(t *testing.T) {
|
||||
t.Helper()
|
||||
if err := m.js.DeleteKeyValue(membership.GrantsBucket); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// sdkMembership is a membership of this package's type as mesh-sdk's, which the grants part composes and writes.
|
||||
func sdkMembership(t *testing.T, mem bus.Membership) membership.Membership {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(mem)
|
||||
var out membership.Membership
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The indexer's two states the console reads, and their buckets (the test bed's own names: the indexer's client names
|
||||
// no holder).
|
||||
const (
|
||||
indexer = "mesh-delivery-indexer"
|
||||
assignerHolder = "mesh-delivery-assigner"
|
||||
// AssignmentsBucket and SeatHoldersBucket are the assigner's states assignments and seat-holders.
|
||||
AssignmentsBucket = assignerHolder + "_assignments"
|
||||
SeatHoldersBucket = assignerHolder + "_seat-holders"
|
||||
ModulesBucket = indexer + "_modules"
|
||||
ManifestsBucket = indexer + "_manifests"
|
||||
)
|
||||
|
||||
// ConsoleReads are the states a console's runtime reads to say what the mesh holds (novox/hq ADR 0571 clauses 4, 5):
|
||||
// assignments, the indexer's modules and manifests, and the seat holders.
|
||||
func ConsoleReads() []bus.StateIssued {
|
||||
return []bus.StateIssued{
|
||||
{Name: assignerHolder + ".assignments", Bucket: AssignmentsBucket},
|
||||
{Name: indexer + ".modules", Bucket: ModulesBucket},
|
||||
{Name: indexer + ".manifests", Bucket: ManifestsBucket},
|
||||
{Name: assignerHolder + ".seat-holders", Bucket: SeatHoldersBucket},
|
||||
}
|
||||
}
|
||||
|
||||
// Placed writes what the mesh holds as its parts write it, with mesh-sdk's encoders: each module the indexer holds with
|
||||
// its manifest (tools, the tools it declares; none for a module without), and each node's assignment of modules.
|
||||
func (m *Mesh) Placed(t *testing.T, tools map[string][]string, on map[string][]string) {
|
||||
t.Helper()
|
||||
kv := func(bucket string) nats.KeyValue {
|
||||
_ = m.js.DeleteKeyValue(bucket)
|
||||
b, err := m.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 1, MaxValueSize: 256 * 1024})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
modules, manifests, assigned := kv(ModulesBucket), kv(ManifestsBucket), kv(AssignmentsBucket)
|
||||
kv(SeatHoldersBucket)
|
||||
for module, declared := range tools {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": module, "tools": declared})
|
||||
digest := meshcatalogclient.ManifestDigest(manifest)
|
||||
raw, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: module, Manifest: string(manifest), V: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := manifests.Put(digest, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err = meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{None: "not built here", Manifest: digest, Repository: "novox/" + module,
|
||||
Commit: strings.Repeat("a", 40), V: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := modules.Put(module, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for node, placed := range on {
|
||||
p := seatassigner.Assignments{V: assigner.Version, Node: node}
|
||||
for _, module := range placed {
|
||||
p.Modules = append(p.Modules, seatassigner.PlacedModule{Module: module})
|
||||
}
|
||||
// As the seat writes it: its node key and a key per module (research 072 row 37).
|
||||
values, err := assigner.AssignmentsValues(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
if _, err := assigned.Put(key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NextEvent is the subject the next event under a pattern lands on.
|
||||
func (m *Mesh) NextEvent(t *testing.T, pattern string) <-chan string {
|
||||
t.Helper()
|
||||
@@ -122,7 +245,7 @@ func (m *Mesh) NextEvent(t *testing.T, pattern string) <-chan string {
|
||||
return ch
|
||||
}
|
||||
|
||||
// MembershipOf is a membership as the controller issues one on a machine.
|
||||
// MembershipOf is a membership as the mesh issues one on a machine.
|
||||
func MembershipOf(module, node string, plain bool, seats map[string][]string) bus.Membership {
|
||||
own := "mesh.mod." + module
|
||||
serves := []bus.Served{{Subject: own + ".tool.{tool}." + node}}
|
||||
@@ -183,7 +306,7 @@ func (l *Logs) Has(fragments ...string) bool {
|
||||
// All is every line.
|
||||
func (l *Logs) All() string { return strings.Join(l.lines, "\n") }
|
||||
|
||||
// Consumer makes a module's durable consumer on a machine as the controller does: pull, on the
|
||||
// Consumer makes a module's durable consumer on a machine as the mesh does: pull, on the
|
||||
// EVENTS stream, filtered to what the module consumes, with a short ack wait so a test sees a
|
||||
// redelivery in seconds rather than the mesh's minutes.
|
||||
func (m *Mesh) Consumer(t *testing.T, node, module string, filters []string, ackWait time.Duration) {
|
||||
@@ -209,6 +332,18 @@ func (m *Mesh) Emit(t *testing.T, subject string, body any) {
|
||||
}
|
||||
}
|
||||
|
||||
// Publish says body on a plain subject, as a core publish with no stream behind it.
|
||||
func (m *Mesh) Publish(t *testing.T, subject string, body any) {
|
||||
t.Helper()
|
||||
data, _ := json.Marshal(body)
|
||||
if err := m.nc.Publish(subject, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.nc.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Pending is what a module's consumer still holds: delivered and not acknowledged, and not yet
|
||||
// delivered.
|
||||
func (m *Mesh) Pending(t *testing.T, node, module string) (ackPending, notDelivered uint64) {
|
||||
@@ -220,7 +355,18 @@ func (m *Mesh) Pending(t *testing.T, node, module string) (ackPending, notDelive
|
||||
return uint64(info.NumAckPending), info.NumPending
|
||||
}
|
||||
|
||||
// Bucket makes a module's state afresh as the controller does from the catalogue (novox/hq ADR 0201),
|
||||
// BucketKeeping makes a module's state afresh as Bucket does, keeping up to history values a key.
|
||||
func (m *Mesh) BucketKeeping(t *testing.T, name string, history uint8) nats.KeyValue {
|
||||
t.Helper()
|
||||
_ = m.js.DeleteKeyValue(name)
|
||||
kv, err := m.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: name, History: history, MaxValueSize: 256 * 1024})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// Bucket makes a module's state afresh as the mesh does from the catalogue (novox/hq ADR 0201),
|
||||
// and answers it for writing what is there before a bundle starts.
|
||||
func (m *Mesh) Bucket(t *testing.T, name string) nats.KeyValue {
|
||||
t.Helper()
|
||||
@@ -231,3 +377,104 @@ func (m *Mesh) Bucket(t *testing.T, name string) nats.KeyValue {
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// BucketKeyTTL makes a module's state afresh as the mesh does for one declared `key-ttl` (mesh-sdk buscompose:
|
||||
// a time to live per key allowed, its end leaving a marker kept a minute; issue 1952).
|
||||
func (m *Mesh) BucketKeyTTL(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
_ = m.js.DeleteKeyValue(name)
|
||||
js, err := jetstream.New(m.nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1, MaxValueSize: 256 * 1024,
|
||||
LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Log makes a module's log afresh as the mesh does from the catalogue (novox/hq ADR 0297): a file
|
||||
// stream named LOG_<module>_<name> on mesh.log.<module>.<name>.>, limits retention, no age, no cap per
|
||||
// key, the declared cap, 260 KiB a message (a 256 KiB value and its headers), refusing new entries when full, direct get, no delete or
|
||||
// purge. It answers the membership entry the mesh issues for it.
|
||||
func (m *Mesh) Log(t *testing.T, module, name string, maxBytes int64) bus.LogIssued {
|
||||
t.Helper()
|
||||
stream := "LOG_" + module + "_" + name
|
||||
subject := "mesh.log." + module + "." + name
|
||||
_ = m.js.DeleteStream(stream)
|
||||
if _, err := m.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{subject + ".>"},
|
||||
Storage: nats.FileStorage, Retention: nats.LimitsPolicy, MaxAge: 0, MaxMsgsPerSubject: -1,
|
||||
MaxBytes: maxBytes, MaxMsgSize: 260 * 1024, Discard: nats.DiscardNew, AllowDirect: true,
|
||||
DenyDelete: true, DenyPurge: true, Replicas: 1,
|
||||
Description: module + "'s log " + name + ", kept as its record (novox/hq ADR 0297)"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return bus.LogIssued{Name: name, Stream: stream, Subject: subject, Writes: true}
|
||||
}
|
||||
|
||||
// Consumers is how many consumers a stream has.
|
||||
func (m *Mesh) Consumers(t *testing.T, stream string) int {
|
||||
t.Helper()
|
||||
info, err := m.js.StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.State.Consumers
|
||||
}
|
||||
|
||||
// ModuleStream makes a module's own stream afresh as the mesh does from the catalogue (novox/hq ADR
|
||||
// 0363 §1): named in capitals, taking `mesh.stream.<module>.event.>`, a subject space EVENTS does not take.
|
||||
func (m *Mesh) ModuleStream(t *testing.T, module, name string) {
|
||||
t.Helper()
|
||||
_ = m.js.DeleteStream(name)
|
||||
if _, err := m.js.AddStream(&nats.StreamConfig{Name: name, Subjects: []string{"mesh.stream." + module + ".event.>"},
|
||||
Retention: nats.LimitsPolicy, Discard: nats.DiscardOld}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ConsumerOn makes a module's durable consumer on a machine on any stream, as Consumer does on EVENTS.
|
||||
func (m *Mesh) ConsumerOn(t *testing.T, stream, node, module string, filters []string, ackWait time.Duration) {
|
||||
t.Helper()
|
||||
cfg := &nats.ConsumerConfig{Durable: node + "_" + module, AckPolicy: nats.AckExplicitPolicy,
|
||||
AckWait: ackWait, MaxDeliver: 10, DeliverPolicy: nats.DeliverNewPolicy}
|
||||
if len(filters) == 1 {
|
||||
cfg.FilterSubject = filters[0]
|
||||
} else {
|
||||
cfg.FilterSubjects = filters
|
||||
}
|
||||
if _, err := m.js.AddConsumer(stream, cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// PendingOn is what a module's consumer on a stream still holds, as Pending is on EVENTS.
|
||||
func (m *Mesh) PendingOn(t *testing.T, stream, node, module string) (ackPending, notDelivered uint64) {
|
||||
t.Helper()
|
||||
info, err := m.js.ConsumerInfo(stream, node+"_"+module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return uint64(info.NumAckPending), info.NumPending
|
||||
}
|
||||
|
||||
// Messages is how many messages a stream holds.
|
||||
func (m *Mesh) Messages(t *testing.T, stream string) uint64 {
|
||||
t.Helper()
|
||||
info, err := m.js.StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.State.Msgs
|
||||
}
|
||||
|
||||
// RemoveConsumer deletes a module's consumer on a machine on a stream, as the mesh does when a grant
|
||||
// it cannot narrow is withdrawn (novox/hq ADR 0363 §6).
|
||||
func (m *Mesh) RemoveConsumer(t *testing.T, stream, node, module string) {
|
||||
t.Helper()
|
||||
if err := m.js.DeleteConsumer(stream, node+"_"+module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,3 +73,56 @@ func TestEachTestHasABusOfItsOwn(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// What merge-check.sh said was "NOT TESTED HERE" is a skip with its reason: no node, or no package, each says so.
|
||||
func TestBundlesCannotBeLaunchedWithoutNodeOrTheSDKAndTheSkipSaysWhy(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if why := bundlesWhy(root, false); !strings.Contains(why, "node is not installed") {
|
||||
t.Errorf("no node said %q", why)
|
||||
}
|
||||
if why := bundlesWhy(root, true); !strings.Contains(why, "@novox/mesh-sdk") {
|
||||
t.Errorf("no package said %q", why)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "node_modules", "@novox", "mesh-sdk"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := bundlesWhy(root, true); why != "" {
|
||||
t.Errorf("node and the package present said %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// The checker runs node-tools' Go level by Go's convention and its Node level by package.json's test script
|
||||
// (novox/hq ADR 0529): no script of the repository's own runs beside them, and no module but node-tools is built from
|
||||
// here (ADR 0524 §5, the toolchain image retired).
|
||||
func TestTheRepositoryHoldsNoCheckScriptAndNodeToolsIsItsOnlyModule(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "..")
|
||||
if _, err := os.Stat(filepath.Join(root, "merge-check.sh")); err == nil {
|
||||
t.Errorf("merge-check.sh is back: the checker's levels run the tests")
|
||||
}
|
||||
var found []string
|
||||
err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && (d.Name() == "node_modules" || d.Name() == ".git") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == "module.json" {
|
||||
found = append(found, filepath.ToSlash(p))
|
||||
}
|
||||
// ADR 0604 section 4: the TypeScript that called the old catalog module's tools is gone, and nothing
|
||||
// under node-tools names one (the needle is built so this file does not name it either).
|
||||
if !d.IsDir() && strings.Contains(filepath.ToSlash(p), "/node-tools/") {
|
||||
if body, rerr := os.ReadFile(p); rerr == nil && strings.Contains(string(body), "catalog"+"_modules") {
|
||||
t.Errorf("%s names a catalog tool of the retired module", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(found) != 1 || !strings.HasSuffix(found[0], "/node-tools/module.json") {
|
||||
t.Errorf("the module manifests are %v, want node-tools' alone", found)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// Package npmlock says whether a package's lock file still describes its package.json (novox/hq issue 624).
|
||||
//
|
||||
// `npm ci` installs only from a lock file in step with package.json, and refuses the whole install when it
|
||||
// is not. node-tools' lock fell behind twice — the SDK's range moved and the toolchain's development
|
||||
// dependencies were added to package.json alone — and nothing here noticed, because the image's build runs
|
||||
// `npm install` and never reads the lock; the lab's runs, which run `npm ci`, stopped at setup instead.
|
||||
// The check is offline and needs no node: it reads the two files the way `npm ci` compares them first.
|
||||
package npmlock
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
DevDependencies map[string]string `json:"devDependencies"`
|
||||
OptionalDependencies map[string]string `json:"optionalDependencies"`
|
||||
PeerDependencies map[string]string `json:"peerDependencies"`
|
||||
}
|
||||
|
||||
type lock struct {
|
||||
LockfileVersion int `json:"lockfileVersion"`
|
||||
Packages map[string]manifest `json:"packages"`
|
||||
}
|
||||
|
||||
// Check reads dir's package.json and package-lock.json and names every way the lock is out of step: a
|
||||
// declared range the lock's root entry records differently or not at all, a range the lock records that
|
||||
// package.json no longer declares, and a declared dependency the lock resolved no package for. None means
|
||||
// in step.
|
||||
func Check(dir string) ([]string, error) {
|
||||
var pkg manifest
|
||||
if err := readJSON(filepath.Join(dir, "package.json"), &pkg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lk lock
|
||||
if err := readJSON(filepath.Join(dir, "package-lock.json"), &lk); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lk.LockfileVersion < 2 {
|
||||
return []string{fmt.Sprintf("package-lock.json is lockfileVersion %d; npm ci needs 2 or later here", lk.LockfileVersion)}, nil
|
||||
}
|
||||
root, ok := lk.Packages[""]
|
||||
if !ok {
|
||||
return []string{`package-lock.json has no root entry (packages[""])`}, nil
|
||||
}
|
||||
var out []string
|
||||
for _, kind := range []struct {
|
||||
name string
|
||||
want, have map[string]string
|
||||
}{
|
||||
{"dependencies", pkg.Dependencies, root.Dependencies},
|
||||
{"devDependencies", pkg.DevDependencies, root.DevDependencies},
|
||||
{"optionalDependencies", pkg.OptionalDependencies, root.OptionalDependencies},
|
||||
{"peerDependencies", pkg.PeerDependencies, root.PeerDependencies},
|
||||
} {
|
||||
for _, name := range sortedKeys(kind.want) {
|
||||
have, ok := kind.have[name]
|
||||
switch {
|
||||
case !ok:
|
||||
out = append(out, fmt.Sprintf("%s %s %q is in package.json and not in the lock", kind.name, name, kind.want[name]))
|
||||
case have != kind.want[name]:
|
||||
out = append(out, fmt.Sprintf("%s %s is %q in package.json and %q in the lock", kind.name, name, kind.want[name], have))
|
||||
}
|
||||
if kind.name == "peerDependencies" {
|
||||
continue
|
||||
}
|
||||
if _, ok := lk.Packages["node_modules/"+name]; !ok {
|
||||
out = append(out, fmt.Sprintf("%s %s has no resolved package in the lock", kind.name, name))
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(kind.have) {
|
||||
if _, ok := kind.want[name]; !ok {
|
||||
out = append(out, fmt.Sprintf("%s %s %q is in the lock and no longer in package.json", kind.name, name, kind.have[name]))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func readJSON(path string, v any) error {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(b, v); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]string) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package npmlock
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestNodeToolsLockIsInStepWithItsPackage: the lab installs node-tools with `npm ci`, which refuses a lock
|
||||
// out of step with package.json, so a change to package.json without its lock fails here and not at a
|
||||
// lab run's setup (novox/hq issue 624).
|
||||
func TestNodeToolsLockIsInStepWithItsPackage(t *testing.T) {
|
||||
out, err := Check(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(out) > 0 {
|
||||
t.Fatalf("node-tools/package-lock.json is out of step with package.json; run `npm install --package-lock-only` in node-tools:\n%s", strings.Join(out, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestALockBehindItsPackageIsNamed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, dir, "package.json", `{"dependencies":{"@novox/mesh-sdk":"^0.1.6","nats":"^2.29.0"},"devDependencies":{"pg":"^8"}}`)
|
||||
write(t, dir, "package-lock.json", `{"lockfileVersion":3,"packages":{
|
||||
"":{"dependencies":{"@novox/mesh-sdk":"^0.1.0","nats":"^2.29.0","left":"^1"}},
|
||||
"node_modules/@novox/mesh-sdk":{"version":"0.1.1"},
|
||||
"node_modules/nats":{"version":"2.29.3"}}}`)
|
||||
out, err := Check(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(out, "\n")
|
||||
for _, want := range []string{
|
||||
`dependencies @novox/mesh-sdk is "^0.1.6" in package.json and "^0.1.0" in the lock`,
|
||||
`dependencies left "^1" is in the lock and no longer in package.json`,
|
||||
`devDependencies pg "^8" is in package.json and not in the lock`,
|
||||
`devDependencies pg has no resolved package in the lock`,
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if len(out) != 4 {
|
||||
t.Errorf("want 4 findings, got %d:\n%s", len(out), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALockInStepIsQuiet(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, dir, "package.json", `{"dependencies":{"nats":"^2.29.0"}}`)
|
||||
write(t, dir, "package-lock.json", `{"lockfileVersion":3,"packages":{"":{"dependencies":{"nats":"^2.29.0"}},"node_modules/nats":{"version":"2.29.3"}}}`)
|
||||
out, err := Check(dir)
|
||||
if err != nil || len(out) != 0 {
|
||||
t.Fatalf("want quiet, got %v %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOldLockfileVersionIsNamed(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write(t, dir, "package.json", `{}`)
|
||||
write(t, dir, "package-lock.json", `{"lockfileVersion":1,"dependencies":{}}`)
|
||||
out, err := Check(dir)
|
||||
if err != nil || len(out) != 1 {
|
||||
t.Fatalf("want one finding, got %v %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func write(t *testing.T, dir, name, content string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// Package provide is the runtime's half of node-tools being a provider of `mcp-endpoint`: it runs the mesh SDK's
|
||||
// provider helper, so a consumer handed to the provisioner (claude-code's `${bound:mcp-endpoint:port}`) is answered
|
||||
// from an entry node-tools writes in its own bucket, not from the old binding file (novox/hq ADR 0349
|
||||
// clause 12, issue 791).
|
||||
//
|
||||
// The offer carries no credential: the console's authority is the machine's login (ADR 0152, 0175), so the helper
|
||||
// reads nothing from the vault, needs no consumers' file and the entry names none. node-tools is not marked moved:
|
||||
// its manifest says `"answered-by": "helper"` for the provision, and the helper runs whenever the runtime does
|
||||
// (issue 791), as route-proxy's does (issue 792).
|
||||
package provide
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient/provider"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
const (
|
||||
// Module is the provider: the runtime's own module.
|
||||
Module = "node-tools"
|
||||
// Provision is what node-tools provides, as its manifest says.
|
||||
Provision = "mcp-endpoint"
|
||||
// ServedPort is the port its manifest serves (`serves.mcp-endpoint.port`): where the console listens.
|
||||
ServedPort = 4270
|
||||
)
|
||||
|
||||
// Offers is what node-tools serves, per provision.
|
||||
func Offers() map[string]provider.Offer {
|
||||
return map[string]provider.Offer{Provision: {
|
||||
Credential: provider.NoCredential,
|
||||
Serves: map[string]any{"port": ServedPort},
|
||||
}}
|
||||
}
|
||||
|
||||
// Provisioner is the module holding the seat mesh-provisioner, whose provisions and plans the helper reads: named from this
|
||||
// module's manifest (`reads`; TestTheOfferServesThePortTheManifestServes), since no client names a holder (novox/hq ADR 0625 §4).
|
||||
const Provisioner = "mesh-provisioner"
|
||||
|
||||
// busOf is the helper's bus over the runtime's own state: the provider's bucket is its state `bound`, and the
|
||||
// provisions and plans of the seat mesh-provisioner are the states it reads, by mesh-sdk's names (novox/hq ADR 0570,
|
||||
// issue 1954 L6-T1). The old assignments are read no more (mesh-sdk 692dbf9, L6-P3).
|
||||
type busOf struct{ conn *bus.Conn }
|
||||
|
||||
func (b busOf) state(bucket string) (string, error) {
|
||||
switch bucket {
|
||||
case provider.BucketOf(Module):
|
||||
return "bound", nil
|
||||
case provisions.ProvisionsBucketOf(Provisioner):
|
||||
return provisions.ProvisionsReadOf(Provisioner), nil
|
||||
case provisions.PlansBucketOf(Provisioner):
|
||||
return provisions.PlansReadOf(Provisioner), nil
|
||||
}
|
||||
return "", fmt.Errorf("%s is not a bucket this provider keeps or reads", bucket)
|
||||
}
|
||||
|
||||
func (b busOf) Watch(ctx context.Context, bucket string, handle func(key string, value []byte)) error {
|
||||
name, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stop, err := b.conn.StateWatch(Module, name, "", func(ch bus.StateChange) error {
|
||||
if ch.Op != "put" {
|
||||
// A delete, or an expiry (issue 1952): the key is gone.
|
||||
handle(ch.Key, nil)
|
||||
} else {
|
||||
handle(ch.Key, ch.Value)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { <-ctx.Done(); stop() }()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b busOf) Put(_ context.Context, bucket, key string, value []byte) error {
|
||||
name, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = b.conn.StatePut(Module, name, key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
|
||||
func (b busOf) Delete(_ context.Context, bucket, key string) error {
|
||||
name, err := b.state(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.conn.StateDelete(Module, name, key)
|
||||
}
|
||||
|
||||
// New is the helper answering node-tools's consumers on node.
|
||||
func New(conn *bus.Conn, node string, every time.Duration, logf func(string, ...any)) *provider.Provider {
|
||||
return provider.New(provider.Options{
|
||||
Node: node,
|
||||
Module: Module, Provisioner: Provisioner,
|
||||
Offers: Offers(),
|
||||
Bus: busOf{conn: conn},
|
||||
Every: every,
|
||||
Logf: logf,
|
||||
})
|
||||
}
|
||||
|
||||
// Run answers the consumers until ctx ends. The state is not reachable until the mesh has issued the runtime its
|
||||
// membership and made the buckets, so a start that cannot watch is tried again, not given up.
|
||||
func Run(ctx context.Context, conn *bus.Conn, node string, every time.Duration, logf func(string, ...any)) {
|
||||
for delay := 2 * time.Second; ctx.Err() == nil; delay = min(delay*2, time.Minute) {
|
||||
attempt, cancel := context.WithCancel(ctx)
|
||||
err := New(conn, node, every, logf).Run(attempt)
|
||||
cancel()
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
logf("[mesh-tools] the provider helper stopped: %v; trying again in %s", err, delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start runs the helper when this process is the machine's runtime (the node-tools credential, on a machine's
|
||||
// name), with no condition on the environment, and returns what ends it. A runtime on another credential, a module's
|
||||
// own in a test, is not the provider and answers nobody.
|
||||
func Start(conn *bus.Conn, module, node string, every time.Duration, logf func(string, ...any)) (stop func()) {
|
||||
if module != Module || node == "" {
|
||||
return func() {}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
logf("[mesh-tools] answering %s's consumers of %s through the provider helper", Module, Provision)
|
||||
go Run(ctx, conn, node, every, logf)
|
||||
return cancel
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package provide
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
provisions "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient/provider"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// claude-code's facts.json template, as the catalogue keeps it (modules/claude-code/files/facts.json).
|
||||
const factsTemplate = `{
|
||||
"node": "${machine:name}",
|
||||
"console": "http://127.0.0.1:${bound:mcp-endpoint:port}/mcp"
|
||||
}
|
||||
`
|
||||
|
||||
// factsToday is that file as the controller composes it today, from the port node-tools serves.
|
||||
const factsToday = `{
|
||||
"node": "anchor",
|
||||
"console": "http://127.0.0.1:4270/mcp"
|
||||
}
|
||||
`
|
||||
|
||||
// fill is what the consumer's client does with the template: the machine's name, and each ${bound:<provision>:<key>}
|
||||
// from the provider's entry.
|
||||
func fill(t *testing.T, template string, entry map[string]any) string {
|
||||
t.Helper()
|
||||
values, _ := entry["values"].(map[string]any)
|
||||
out := strings.ReplaceAll(template, "${machine:name}", "anchor")
|
||||
for key, v := range values {
|
||||
raw, _ := json.Marshal(v)
|
||||
out = strings.ReplaceAll(out, "${bound:mcp-endpoint:"+key+"}", string(raw))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTheRuntimeAnswersAHandedConsumersMcpEndpointThroughTheHelper(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
bound := mesh.Bucket(t, "node-tools_bound")
|
||||
plans := mesh.Bucket(t, provisions.PlansBucketOf(Provisioner))
|
||||
provs := mesh.Bucket(t, provisions.ProvisionsBucketOf(Provisioner))
|
||||
|
||||
mem := mt.MembershipOf("node-tools", "anchor", false, nil)
|
||||
mem.State = []bus.StateIssued{
|
||||
{Name: "bound", Bucket: "node-tools_bound", Writes: true},
|
||||
{Name: provisions.PlansReadOf(Provisioner), Bucket: provisions.PlansBucketOf(Provisioner)},
|
||||
{Name: provisions.ProvisionsReadOf(Provisioner), Bucket: provisions.ProvisionsBucketOf(Provisioner)},
|
||||
}
|
||||
mesh.Issue(t, mem)
|
||||
|
||||
// claude-code, handed on this machine, is bound to node-tools here for mcp-endpoint: its plan and its provision
|
||||
// key, written as mesh-provisioner writes them, with mesh-sdk's encoders (ADR 0570 clauses 1-3, issue 1954).
|
||||
plan, err := provisions.EncodePlan(provisions.Plan{Node: "anchor", Module: "claude-code", Plan: catalogue.Plan{
|
||||
Provisions: []catalogue.PlanProvision{{Provision: "mcp-endpoint", Provider: "anchor", ProviderModule: "node-tools",
|
||||
At: "127.0.0.1", From: "anchor", As: "mesh_anchor_claude_code"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := plans.Put("anchor.claude-code", plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := provisions.Key("anchor", "claude-code", "mcp-endpoint", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, err := provisions.Encode(provisions.Provision{Provision: "mcp-endpoint", State: provisions.Asked,
|
||||
Consumer: provisions.End{Node: "anchor", Module: "claude-code"}, Provider: provisions.End{Node: "anchor", Module: "node-tools"},
|
||||
At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := provs.Put(key, asked); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("node-tools")
|
||||
|
||||
said := &mt.Logs{}
|
||||
t.Cleanup(Start(conn, "node-tools", "anchor", 20*time.Millisecond, said.Logf))
|
||||
|
||||
var entry map[string]any
|
||||
mt.Until(t, func() error {
|
||||
kept, err := bound.Get("anchor.claude-code.mcp-endpoint")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(kept.Value(), &entry)
|
||||
})
|
||||
values, _ := entry["values"].(map[string]any)
|
||||
for key, want := range map[string]any{"port": float64(4270), "at": "127.0.0.1", "from": "anchor"} {
|
||||
if values[key] != want {
|
||||
t.Errorf("the entry's %s is %v, want %v (entry %v)", key, values[key], want, entry)
|
||||
}
|
||||
}
|
||||
if entry["as"] != "mesh_anchor_claude_code" {
|
||||
t.Errorf("the entry's as is %v", entry["as"])
|
||||
}
|
||||
if c, has := entry["credential"]; has {
|
||||
t.Errorf("an offer with no credential names one: %v", c)
|
||||
}
|
||||
if got := fill(t, factsTemplate, entry); got != factsToday {
|
||||
t.Errorf("claude-code's facts.json fills to\n%s\nwant today's\n%s", got, factsToday)
|
||||
}
|
||||
}
|
||||
|
||||
// The port the offer serves is the port the manifest serves and the console listens on.
|
||||
func TestTheOfferServesThePortTheManifestServes(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Serves map[string]map[string]any `json:"serves"`
|
||||
Moved bool `json:"moved"`
|
||||
State []string `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Provides []struct {
|
||||
Name string `json:"name"`
|
||||
AnsweredBy string `json:"answered-by"`
|
||||
} `json:"provides"`
|
||||
Resources []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Env map[string]string `json:"env"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := m.Serves[Provision]["port"]; got != float64(ServedPort) {
|
||||
t.Errorf("the manifest serves %v for %s, the offer %d", got, Provision, ServedPort)
|
||||
}
|
||||
if m.Moved {
|
||||
t.Error("node-tools is marked moved: that also hands the runtime's bus account to the vault (issue 791 round 1)")
|
||||
}
|
||||
if len(m.Provides) != 1 || m.Provides[0].Name != Provision || m.Provides[0].AnsweredBy != "helper" {
|
||||
t.Errorf("node-tools does not say its %s is answered by the helper: %+v", Provision, m.Provides)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r.ID == "consumers" {
|
||||
t.Error("the manifest still carries a consumers file the helper does not need")
|
||||
}
|
||||
}
|
||||
for _, r := range m.Reads {
|
||||
if r == "mesh-module-assigner.assignments" {
|
||||
t.Error("the manifest still reads mesh-module-assigner.assignments, which the sdk's helper no longer reads (issue 1954)")
|
||||
}
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Name == "files" || len(a.Env) > 0 {
|
||||
t.Errorf("the build still carries %+v, which only a moved module needs", a)
|
||||
}
|
||||
}
|
||||
// Every bucket the sdk's helper reaches is a state the manifest keeps or reads, by the sdk's names: the provider's
|
||||
// own and the seat mesh-provisioner's provisions and plans (issue 1954 L6-T1); the controller's assignments no more.
|
||||
declared := map[string]bool{}
|
||||
for _, n := range append(m.State, m.Reads...) {
|
||||
declared[n] = true
|
||||
}
|
||||
for bucket, want := range map[string]string{
|
||||
provider.BucketOf(Module): "bound",
|
||||
provisions.ProvisionsBucketOf(Provisioner): "mesh-provisioner.provisions",
|
||||
provisions.PlansBucketOf(Provisioner): "mesh-provisioner.plans",
|
||||
} {
|
||||
got, err := (busOf{}).state(bucket)
|
||||
if err != nil || got != want || !declared[got] {
|
||||
t.Errorf("the helper's bucket %s reaches state %q (%v), want %q declared in the manifest", bucket, got, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only the machine's runtime is the provider: a process on another credential, or with no machine, answers nobody.
|
||||
func TestOnlyTheMachinesRuntimeStartsTheHelper(t *testing.T) {
|
||||
said := &mt.Logs{}
|
||||
for _, c := range []struct{ module, node string }{{"claude-code", "anchor"}, {"node-tools", ""}} {
|
||||
Start(nil, c.module, c.node, time.Second, said.Logf)()
|
||||
}
|
||||
}
|
||||
|
||||
// 1126/B19 (ADR 0590): this manifest is not refused by the check that refuses any manifest naming the controller:
|
||||
// it reads, invokes, claims and requires nothing of it.
|
||||
func TestTheManifestNamesNothingOfTheController(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Read as text, as mesh-catalog's controllerdeps check reads its own: mesh-sdk names no module (issue 774), and the
|
||||
// manifest says it is the tool runner rather than being known by its name.
|
||||
if strings.Contains(string(raw), "mesh-controller") {
|
||||
t.Error("node-tools' manifest names the ended controller")
|
||||
}
|
||||
if !catalogue.IsToolRunner(m) {
|
||||
t.Error("node-tools' manifest does not say it is the tool runner (\"tool-runner\": true)")
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/announce"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
// read here with NATS's own types, each endpoint a subject actually served — and the announcement
|
||||
// follows a re-issued membership.
|
||||
func TestTheRuntimeAnnouncesWhatItServesInTheServicesProtocol(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", false, nil))
|
||||
mesh.Issue(t, mt.MembershipOf("beta", "anchor", false, map[string][]string{"node-shelf": {"list", "clear"}}))
|
||||
@@ -99,10 +101,6 @@ func TestTheRuntimeAnnouncesWhatItServesInTheServicesProtocol(t *testing.T) {
|
||||
if msg, err := nc.Request("$SRV.PING", nil, 2*time.Second); err != nil || !strings.Contains(string(msg.Data), micro.PingResponseType) {
|
||||
t.Errorf("ping: %v %v", msg, err)
|
||||
}
|
||||
if _, err := nc.Request("$SRV.INFO.somebody-else", nil, 300*time.Millisecond); err == nil {
|
||||
t.Error("answered a request for another service")
|
||||
}
|
||||
|
||||
// alpha is re-issued a plain subject: the announcement says so, and says it is interchangeable.
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", true, nil))
|
||||
var after string
|
||||
@@ -119,3 +117,29 @@ func TestTheRuntimeAnnouncesWhatItServesInTheServicesProtocol(t *testing.T) {
|
||||
}
|
||||
_ = bus.Served{}
|
||||
}
|
||||
|
||||
// issue 1966 TLS-3 (ENG-2): the runner states the capabilities of the calls it answers in its PING, on its own instance, which
|
||||
// is what its machine's node-engine asks and takes into the node's record; no bundle is needed for it.
|
||||
func TestTheRunnerStatesTheCapabilitiesOfItsCallsInItsPing(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("alpha", "anchor", false, nil))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
stop, err := Run(nodeTools, nil, nil, (&mt.Logs{}).Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
msg, err := nc.Request("$SRV.PING.node-tools.anchor", nil, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("ping of the instance: %v", err)
|
||||
}
|
||||
var ping micro.Ping
|
||||
if err := json.Unmarshal(msg.Data, &ping); err != nil || ping.Metadata[announce.Capabilities] != "bus-state-calls,ledger-calls" {
|
||||
t.Errorf("the runner states %q (%v), want its bus-state and ledger calls", ping.Metadata[announce.Capabilities], err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// The capabilities the tool runner states of the calls it answers (novox/hq ADR 0595 §9, ADR 0597; design 25 § 12.4): each is
|
||||
// a family of the methods a bundle asks it, named so a manifest's `capabilities` can ask for it, and stated in its announcement
|
||||
// (announce.Capabilities) for its machine's node-engine to take into the node's record. A module whose manifest asks for one is
|
||||
// held off a machine whose runner does not state it.
|
||||
const (
|
||||
CapBusStateCalls = "bus-state-calls"
|
||||
CapLedgerCalls = "ledger-calls"
|
||||
)
|
||||
|
||||
// callFamilies are the methods each capability stands for, the ones BusState and BusStateWatch answer.
|
||||
var callFamilies = map[string][]string{
|
||||
CapBusStateCalls: {buscalls.ObjectsPut, buscalls.ObjectsGet, buscalls.ObjectsList, buscalls.ObjectsDelete, buscalls.ObjectsInfo,
|
||||
buscalls.ObjectsWatch, buscalls.StreamRead, buscalls.StreamFill, buscalls.WorkQueueSize, buscalls.WorkQueueDead},
|
||||
CapLedgerCalls: {buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerHistory, buscalls.LedgerWatch, buscalls.LedgerQuery,
|
||||
buscalls.LedgerRebuild, buscalls.LedgerReindex, buscalls.LedgerSnapshot, buscalls.LedgerLag},
|
||||
}
|
||||
|
||||
// CallCapabilities is every capability of the calls this runner answers, sorted.
|
||||
func CallCapabilities() []string {
|
||||
out := make([]string, 0, len(callFamilies))
|
||||
for c := range callFamilies {
|
||||
out = append(out, c)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// BusState answers a bundle's calls on its module's bus state in the new form (mesh-sdk buscalls; novox/hq ADR 0595
|
||||
// §9, design 25 § 12.5; issue 1966): each as the module whose bundle asks, never a module the request names, so a
|
||||
// bundle reaches its own module's bindings alone.
|
||||
func (b *moduleBus) BusState(method string, params json.RawMessage) (json.RawMessage, error) {
|
||||
conn := b.all.conn
|
||||
var answer any
|
||||
var err error
|
||||
switch method {
|
||||
case buscalls.ObjectsPut:
|
||||
var r buscalls.ObjectsPutRequest
|
||||
if err = decode(method, params, &r); err == nil {
|
||||
answer, err = conn.ObjectsPut(b.module, r)
|
||||
}
|
||||
case buscalls.ObjectsGet:
|
||||
var r buscalls.ObjectsNameRequest
|
||||
if err = decode(method, params, &r); err == nil {
|
||||
var got *buscalls.ObjectData
|
||||
if got, err = conn.ObjectsGet(b.module, r); err == nil && got == nil {
|
||||
return json.RawMessage("null"), nil
|
||||
}
|
||||
answer = got
|
||||
}
|
||||
case buscalls.ObjectsDelete:
|
||||
var r buscalls.ObjectsNameRequest
|
||||
if err = decode(method, params, &r); err == nil {
|
||||
err = conn.ObjectsDelete(b.module, r)
|
||||
answer = buscalls.Done{}
|
||||
}
|
||||
case buscalls.ObjectsList:
|
||||
var r buscalls.ObjectsRequest
|
||||
if err = decode(method, params, &r); err == nil {
|
||||
answer, err = conn.ObjectsList(b.module, r)
|
||||
}
|
||||
case buscalls.ObjectsInfo:
|
||||
var r buscalls.ObjectsRequest
|
||||
if err = decode(method, params, &r); err == nil {
|
||||
answer, err = conn.ObjectsInfo(b.module, r)
|
||||
}
|
||||
case buscalls.StreamRead:
|
||||
var r buscalls.StreamReadRequest
|
||||
if err = decode(method, params, &r); err == nil {
|
||||
answer, err = conn.StreamRead(b.module, r)
|
||||
}
|
||||
case buscalls.StreamFill:
|
||||
var r buscalls.StreamRequest
|
||||
if err = decode(method, params, &r); err == nil {
|
||||
answer, err = conn.StreamFill(b.module, r)
|
||||
}
|
||||
case buscalls.WorkQueueSize:
|
||||
var r buscalls.WorkQueueRequest
|
||||
if err = decode(method, params, &r); err == nil {
|
||||
answer, err = conn.WorkQueueSize(b.module, r)
|
||||
}
|
||||
case buscalls.WorkQueueDead:
|
||||
var r buscalls.WorkQueueDeadRequest
|
||||
if err = decode(method, params, &r); err == nil {
|
||||
answer, err = conn.WorkQueueDead(b.module, r)
|
||||
}
|
||||
case buscalls.LedgerApply, buscalls.LedgerGet, buscalls.LedgerHistory, buscalls.LedgerQuery, buscalls.LedgerRebuild,
|
||||
buscalls.LedgerReindex, buscalls.LedgerSnapshot, buscalls.LedgerLag:
|
||||
answer, err = b.ledgerCall(method, params)
|
||||
default:
|
||||
return nil, fmt.Errorf("the runtime answers no %s", method)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(answer)
|
||||
}
|
||||
|
||||
// BusStateWatch hands a bundle each change of its objects or its ledger, the current ones first.
|
||||
func (b *moduleBus) BusStateWatch(method string, params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
switch method {
|
||||
case buscalls.ObjectsWatch:
|
||||
var r buscalls.ObjectsWatchRequest
|
||||
if err := decode(method, params, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.all.conn.ObjectsWatch(b.module, r, func(change buscalls.ObjectsChange) error {
|
||||
raw, err := json.Marshal(change)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return deliver(raw)
|
||||
})
|
||||
case buscalls.LedgerWatch:
|
||||
return b.ledgerWatch(params, deliver)
|
||||
}
|
||||
return nil, fmt.Errorf("the runtime answers no %s", method)
|
||||
}
|
||||
|
||||
// decode reads a bus-state call's request.
|
||||
func decode(method string, params json.RawMessage, into any) error {
|
||||
if err := json.Unmarshal(params, into); err != nil {
|
||||
return fmt.Errorf("%s's request is not of its shape (mesh-sdk buscalls): %w", method, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
type saidCaller struct {
|
||||
Caller string `json:"caller"`
|
||||
Call string `json:"call"`
|
||||
Arguments map[string]any `json:"arguments"`
|
||||
}
|
||||
|
||||
// **A module is told who called it, from the subject the call arrived on** (novox/hq issue 365): a call on the
|
||||
// caller-named subject reaches the bundle with that caller in the call's `_meta`, for a module's tool, on its
|
||||
// machine's address, and for a seat's verb; a caller named in the arguments is only an argument.
|
||||
func TestACallOnTheCallerNamedSubjectReachesTheModuleWithItsCaller(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", true, map[string][]string{"node-desk": {"who"}}))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
stop, err := Run(nodeTools, []Served{{"ledger", []string{mt.Fixture("caller-echo.py")}}}, nil, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop() // returns once the bundle's exit is handled (novox/hq issue 476)
|
||||
asker, err := bus.Connect(bus.Credential{URL: mt.URL(t), Module: "node-tools", Node: "shanks",
|
||||
User: "shanks.node-tools", Password: "unused"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asker.Logf = func(string, ...any) {}
|
||||
defer asker.Close()
|
||||
|
||||
calls := map[string]bool{}
|
||||
for _, key := range []string{"ledger.who", "ledger.who@anchor", "seat:node-desk.who@anchor"} {
|
||||
var got saidCaller
|
||||
mt.Until(t, func() error {
|
||||
answered, err := asker.Ask(key, map[string]any{"caller": "novox.node-tools",
|
||||
"_meta": map[string]any{"mesh/caller": "novox.controller"}}, "")
|
||||
if err == nil {
|
||||
err = json.Unmarshal(answered.Result, &got)
|
||||
}
|
||||
return err
|
||||
})
|
||||
if got.Caller != "shanks.node-tools" {
|
||||
t.Errorf("%s reached the module with caller %q, want the asker's bus user", key, got.Caller)
|
||||
}
|
||||
// And the id of the call in flight, one per call, which a ledger's apply names (novox/hq ADR 0597 §7).
|
||||
if got.Call == "" || calls[got.Call] {
|
||||
t.Errorf("%s reached the module with call id %q, want one of its own", key, got.Call)
|
||||
}
|
||||
calls[got.Call] = true
|
||||
if got.Arguments["caller"] != "novox.node-tools" {
|
||||
t.Errorf("%s: the arguments did not reach the module as sent: %v", key, got.Arguments)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The subject that names no caller is still served, for one release (retired by hq issue 464), and says so:
|
||||
// the module is named no caller, whatever the body claims.
|
||||
func TestTheSubjectThatNamesNoCallerStillWorksAndNamesNone(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", true, map[string][]string{"node-desk": {"who"}}))
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
stop, err := Run(nodeTools, []Served{{"ledger", []string{mt.Fixture("caller-echo.py")}}}, nil, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop() // returns once the bundle's exit is handled (novox/hq issue 476)
|
||||
old, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer old.Close()
|
||||
body, _ := json.Marshal(map[string]any{"_meta": map[string]any{"mesh/caller": "novox.controller"}})
|
||||
for _, subject := range []string{"mesh.mod.ledger.tool.who", "mesh.mod.ledger.tool.who.anchor", "mesh.seat.node-desk.tool.who.anchor"} {
|
||||
var reply struct {
|
||||
Result saidCaller `json:"result"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
msg, err := old.Request(subject, body, 5*time.Second)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(msg.Data, &reply)
|
||||
}
|
||||
return err
|
||||
})
|
||||
if reply.Error != "" || reply.Result.Caller != "" {
|
||||
t.Errorf("%s: the module was named caller %q (error %q), want none", subject, reply.Result.Caller, reply.Error)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -50,6 +52,7 @@ func countLines(t *testing.T, path, want string) int {
|
||||
// is its bus — it binds the module's own consumer, hands each event to the bundle, and acknowledges it
|
||||
// only when the bundle has taken it; one the bundle failed or died on is offered again.
|
||||
func TestTheRuntimeHandsAModulesEventsToItsBundleAndAcknowledgesThemOnlyWhenTaken(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
restore := bus.NakDelay
|
||||
bus.NakDelay = 300 * time.Millisecond
|
||||
@@ -128,6 +131,7 @@ func TestTheRuntimeHandsAModulesEventsToItsBundleAndAcknowledgesThemOnlyWhenTake
|
||||
|
||||
// A long-running bundle that exits is started again, without waiting for a call (ADR 0198).
|
||||
func TestALongRunningBundleThatExitsIsStartedAgain(t *testing.T) {
|
||||
mt.NeedBundles(t)
|
||||
mesh := mt.New(t)
|
||||
firstPause := launch.RestartFirst
|
||||
launch.RestartFirst = 100 * time.Millisecond
|
||||
@@ -155,3 +159,37 @@ func TestALongRunningBundleThatExitsIsStartedAgain(t *testing.T) {
|
||||
|
||||
func fmtSprintf(format string, args ...any) string { return fmt.Sprintf(format, args...) }
|
||||
func errorf(format string, args ...any) error { return fmt.Errorf(format, args...) }
|
||||
|
||||
// A bundle loaded again sets its event and heartbeat handlers while the consumer's loop hands a word or an event on: no data race (-race),
|
||||
// and the heartbeat is asked of the bus only while some bundle holds a handler.
|
||||
func TestAHeartbeatHandlerSetAgainWhileWordsAreHandedOnIsNotARace(t *testing.T) {
|
||||
m := &moduleEvents{}
|
||||
all := &consumers{logf: t.Logf, of: map[string]*moduleEvents{"mod": m}}
|
||||
slot := &beatSlot{}
|
||||
m.beats = []*beatSlot{slot}
|
||||
if m.beatAsked() {
|
||||
t.Fatal("a heartbeat is asked while no bundle gave a handler")
|
||||
}
|
||||
took := &deliverSlot{}
|
||||
took.set(func(json.RawMessage) error { return nil })
|
||||
m.delivers = []*deliverSlot{took}
|
||||
var got atomic.Int32
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for i := 0; i < 200; i++ {
|
||||
slot.set(func(json.RawMessage) { got.Add(1) })
|
||||
took.set(func(json.RawMessage) error { got.Add(1); return nil })
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 200; i++ {
|
||||
all.handBeat(m, bus.Beat{At: time.Now()})
|
||||
if err := all.handOut("mod", m, bus.Envelope{Key: "a.b", Body: json.RawMessage(`{}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
<-done
|
||||
if !m.beatAsked() {
|
||||
t.Error("a bundle gave a handler and no heartbeat is asked")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// The ledger credentials (novox/hq ADR 0597 §9, design 25 § 12.4; issue 1966 T6): one per module assignment, held by
|
||||
// the machine's tool runner in its own secret `ledgers`, a map of module to credential (`{}` when none), which the vault
|
||||
// rewrites on each mint and retirement. **A module's ledger history is appended, and its snapshots written, only on
|
||||
// that module's own ledger connection**; everything else stays on the runner's own.
|
||||
|
||||
// LedgersFile is where the runner reads its own secret `ledgers`: MESH_LEDGERS_FILE when set, else beside its broker
|
||||
// credential, both in its directory mesh-state (`${dir:mesh-state}/ledgers`). Its manifest declares the secret only once
|
||||
// the vault holds it on every machine (B3 of issue 1966): until then a file not there means no ledger credentials, and
|
||||
// every ledger write is refused naming the missing one.
|
||||
const LedgersFile = "MESH_LEDGERS_FILE"
|
||||
|
||||
// LedgersFileOf is the path of the secret `ledgers` for a runtime whose broker credential is at broker.
|
||||
func LedgersFileOf(broker string) string {
|
||||
if file := os.Getenv(LedgersFile); file != "" {
|
||||
return file
|
||||
}
|
||||
if broker == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(filepath.Dir(broker), "ledgers")
|
||||
}
|
||||
|
||||
// LedgerCredentials are the ledger connections of the modules on this machine, read from the secret `ledgers` and
|
||||
// followed as it changes, with no restart.
|
||||
type LedgerCredentials struct {
|
||||
file string
|
||||
dial func(bus.Credential) (*bus.Conn, error)
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
creds map[string]bus.Credential
|
||||
conns map[string]*bus.Conn
|
||||
why string // why the file could not be read, when it could not
|
||||
stop func()
|
||||
}
|
||||
|
||||
// FollowLedgerCredentials reads the secret `ledgers` at file and follows it (WatchFile), dialling a module's connection
|
||||
// with dial the first time it is asked for.
|
||||
func FollowLedgerCredentials(file string, dial func(bus.Credential) (*bus.Conn, error), logf func(string, ...any)) *LedgerCredentials {
|
||||
l := &LedgerCredentials{file: file, dial: dial, logf: logf, creds: map[string]bus.Credential{}, conns: map[string]*bus.Conn{}}
|
||||
l.read()
|
||||
stop, err := WatchFile(file, l.read, logf, "a change of the ledger credentials is taken only after a restart")
|
||||
if err != nil {
|
||||
logf("[mesh-tools] cannot follow the ledger credentials at %s: %v; a change is taken only after a restart", file, err)
|
||||
} else {
|
||||
l.stop = stop
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// read takes the file as it is now: a module whose credential changed or went has its connection closed, and is
|
||||
// dialled again from the new one when next asked for.
|
||||
func (l *LedgerCredentials) read() {
|
||||
creds, why := map[string]bus.Credential{}, ""
|
||||
raw, err := os.ReadFile(l.file)
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
why = "the file " + l.file + " is not there yet"
|
||||
case err != nil:
|
||||
why = err.Error()
|
||||
case len(bytes.TrimSpace(raw)) == 0:
|
||||
why = "the file " + l.file + " is empty"
|
||||
default:
|
||||
if err := json.Unmarshal(raw, &creds); err != nil {
|
||||
why = "the file " + l.file + " is not a map of module to credential: " + err.Error()
|
||||
creds = map[string]bus.Credential{}
|
||||
}
|
||||
}
|
||||
l.mu.Lock()
|
||||
var closing []*bus.Conn
|
||||
for module, conn := range l.conns {
|
||||
if now, has := creds[module]; !has || !reflect.DeepEqual(now, l.creds[module]) {
|
||||
closing = append(closing, conn)
|
||||
delete(l.conns, module)
|
||||
}
|
||||
}
|
||||
saidBefore := l.why
|
||||
l.creds, l.why = creds, why
|
||||
l.mu.Unlock()
|
||||
for _, c := range closing {
|
||||
c.Close()
|
||||
}
|
||||
if why != "" && why != saidBefore {
|
||||
l.logf("[mesh-tools] the ledger credentials cannot be read (%s); no ledger is written until they can", why)
|
||||
}
|
||||
}
|
||||
|
||||
// Writer is a module's ledger connection, dialled from its credential the first time it is asked for; refused, naming
|
||||
// what is missing, when the secret holds none for it.
|
||||
func (l *LedgerCredentials) Writer(module string) (*bus.Conn, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if c := l.conns[module]; c != nil {
|
||||
return c, nil
|
||||
}
|
||||
cred, has := l.creds[module]
|
||||
if !has {
|
||||
why := ""
|
||||
if l.why != "" {
|
||||
why = " (" + l.why + ")"
|
||||
}
|
||||
return nil, fmt.Errorf("%s's ledger credential is missing from the tool runner's own secret ledgers%s: its history "+
|
||||
"is published only on its own ledger connection, which the grants part issues with its membership and the "+
|
||||
"vault writes there (novox/hq ADR 0597 §9)", module, why)
|
||||
}
|
||||
if cred.Module == "" {
|
||||
cred.Module = module
|
||||
}
|
||||
c, err := l.dial(cred)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's ledger connection could not be opened: %w", module, err)
|
||||
}
|
||||
c.Logf = l.logf
|
||||
l.conns[module] = c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Stop ends the following and closes every ledger connection.
|
||||
func (l *LedgerCredentials) Stop() {
|
||||
if l.stop != nil {
|
||||
l.stop()
|
||||
}
|
||||
l.mu.Lock()
|
||||
conns := l.conns
|
||||
l.conns = map[string]*bus.Conn{}
|
||||
l.mu.Unlock()
|
||||
for _, c := range conns {
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// 1966/AC22: given the secret ledgers holding m's credential, when m applies, then the append goes on m's connection
|
||||
// (the runner's own account may not publish on m's ledger, as the grants part grants it); given it missing, apply is
|
||||
// refused naming it; a change of the file is taken without a restart.
|
||||
func TestALedgerIsWrittenOnlyOnItsModulesLedgerConnection(t *testing.T) {
|
||||
ledgerSubjects := []string{buscompose.LedgerHistorySubject("m", "records") + ".>", buscompose.LedgerSnapshotSubject("m", "records") + ".>"}
|
||||
users := []*server.User{
|
||||
{Username: "node-tools", Password: "pw", Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: []string{">"}, Deny: ledgerSubjects}}},
|
||||
// As the grants part composes a ledger account (mesh-grants buscompose ledgerPermissions): publish on its
|
||||
// module's ledger histories and snapshots, subscribe its own inbox, nothing else.
|
||||
{Username: "ledger.anchor.m", Password: "pw", Permissions: &server.Permissions{
|
||||
Publish: &server.SubjectPermission{Allow: ledgerSubjects},
|
||||
Subscribe: &server.SubjectPermission{Allow: []string{"_INBOX.ledger.anchor.m.>"}}}},
|
||||
{Username: "admin", Password: "pw"},
|
||||
}
|
||||
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(),
|
||||
NoLog: true, NoSigs: true, Users: users})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(30 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
url := s.ClientURL()
|
||||
|
||||
admin, err := nats.Connect(url, nats.UserInfo("admin", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(admin.Close)
|
||||
js, _ := jetstream.New(admin)
|
||||
ctx := context.Background()
|
||||
for _, cfg := range []jetstream.StreamConfig{
|
||||
{Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.history.>"}},
|
||||
{Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger.m.records.snapshot.>"}, MaxMsgsPerSubject: 1},
|
||||
} {
|
||||
if _, err := js.CreateStream(ctx, cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, b := range []string{"m_records", membership.GrantsBucket} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
mem := mt.MembershipOf("m", "anchor", false, nil)
|
||||
mem.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger,
|
||||
History: "LEDGER_m_records", Subject: "mesh.ledger.m.records.history", State: "m_records", Snapshot: "mesh.ledger.m.records.snapshot",
|
||||
Types: []catalogue.LedgerType{{Name: "decision", Operations: []catalogue.LedgerOperation{
|
||||
{Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}}}}}}}
|
||||
raw, _ := json.Marshal(mem)
|
||||
var sdk membership.Membership
|
||||
_ = json.Unmarshal(raw, &sdk)
|
||||
grants, _ := js.KeyValue(ctx, membership.GrantsBucket)
|
||||
if _, err := grants.Put(ctx, membership.GrantsKey("anchor", "m"), membership.EncodeGrantsMembership(sdk)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn, err := bus.Connect(bus.Credential{URL: url, Module: "node-tools", Node: "anchor", User: "node-tools", Password: "pw"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn.Logf = func(string, ...any) {}
|
||||
t.Cleanup(conn.Close)
|
||||
conn.Follow("m")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("m") == nil {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Not there yet, as on a machine whose vault has not made it: no ledger credentials, and the runner serves on.
|
||||
file := filepath.Join(t.TempDir(), "ledgers")
|
||||
creds := FollowLedgerCredentials(file, bus.Dial, t.Logf)
|
||||
t.Cleanup(creds.Stop)
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
WithLedgerWriters(creds.Writer)(all)
|
||||
m := all.forModule("m").(*moduleBus)
|
||||
apply := func(instance string) error {
|
||||
_, err := m.BusState(buscalls.LedgerApply, json.RawMessage(`{"ledger":"records","instance":"`+instance+`","operation":"title","body":{"title":"T"}}`))
|
||||
return err
|
||||
}
|
||||
|
||||
// Missing: refused, naming it and why.
|
||||
if err := apply("decision/1"); err == nil || !strings.Contains(err.Error(), "m's ledger credential is missing from the tool runner's own secret ledgers (the file "+file+" is not there yet)") {
|
||||
t.Fatalf("an apply with no credential: %v", err)
|
||||
}
|
||||
// Written: taken without a restart, on m's connection (the runner's own account is denied m's ledger subjects, so an
|
||||
// append that lands went on m's), whose account stamps the operation.
|
||||
cred, _ := json.Marshal(map[string]bus.Credential{"m": {URL: url, Node: "anchor", Module: "m", User: "ledger.anchor.m", Password: "pw"}})
|
||||
if err := os.WriteFile(file, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mt.Until(t, func() error { return apply("decision/1") })
|
||||
page, err := m.BusState(buscalls.LedgerHistory, json.RawMessage(`{"ledger":"records","instance":"decision/1"}`))
|
||||
if err != nil || !strings.Contains(string(page), `"author":{"account":"ledger.anchor.m"}`) {
|
||||
t.Fatalf("the history holds %s, %v", page, err)
|
||||
}
|
||||
// Gone again: refused again, the change taken without a restart.
|
||||
if err := os.WriteFile(file, []byte(`{}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mt.Until(t, func() error {
|
||||
if err := apply("decision/2"); err == nil || !strings.Contains(err.Error(), "credential is missing") {
|
||||
return os.ErrExist
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
//go:build integration
|
||||
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/ledger"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// racing is a history whose append lets another writer apply first, once: before its own append, or after it.
|
||||
type racing struct {
|
||||
ledger.History
|
||||
before, after func()
|
||||
}
|
||||
|
||||
func (r *racing) Append(key string, op []byte, expect uint64) (uint64, error) {
|
||||
if f := r.before; f != nil {
|
||||
r.before = nil
|
||||
f()
|
||||
}
|
||||
seq, err := r.History.Append(key, op, expect)
|
||||
if f := r.after; f != nil && err == nil {
|
||||
r.after = nil
|
||||
f()
|
||||
}
|
||||
return seq, err
|
||||
}
|
||||
|
||||
// cut is a state whose next write is lost, as a writer stopped between its history and its state would lose it.
|
||||
type cut struct {
|
||||
ledger.State
|
||||
once bool
|
||||
}
|
||||
|
||||
func (c *cut) Write(key string, value []byte, revision uint64) (uint64, error) {
|
||||
if c.once {
|
||||
c.once = false
|
||||
return 0, errors.New("the writer stopped")
|
||||
}
|
||||
return c.State.Write(key, value, revision)
|
||||
}
|
||||
|
||||
// 1966/AC19: the ledger keeps its promises against the real bus (ADR 0597's How it is checked; design 25 § 12.7), one
|
||||
// case each, and every apply leaves the snapshot equal to the state.
|
||||
func TestTheLedgerKeepsItsPromisesOnTheBus(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx := context.Background()
|
||||
for _, cfg := range []jetstream.StreamConfig{
|
||||
{Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.history.>"}},
|
||||
{Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger.m.records.snapshot.>"}, MaxMsgsPerSubject: 1},
|
||||
} {
|
||||
if _, err := js.CreateStream(ctx, cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, b := range []string{"m_records", "m_records_by_title"} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
types := []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}},
|
||||
Operations: []catalogue.LedgerOperation{
|
||||
{Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true},
|
||||
{Name: "note", Effect: catalogue.EffectAppend, Into: "notes", Fields: []string{"says"}},
|
||||
},
|
||||
Indexes: []catalogue.LedgerIndex{{Name: "by-title", On: []string{"title"}}}}}
|
||||
binding := membership.BusBinding{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger,
|
||||
History: "LEDGER_m_records", Subject: "mesh.ledger.m.records.history", State: "m_records",
|
||||
Indexes: map[string]string{"by-title": "m_records_by_title"}, Snapshot: "mesh.ledger.m.records.snapshot"}
|
||||
mem := mt.MembershipOf("m", "anchor", false, nil)
|
||||
binding.Types = types
|
||||
mem.Bindings = []membership.BusBinding{binding}
|
||||
mesh.Issue(t, mem)
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("m")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("m") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
WithLedgerWriters(func(string) (*bus.Conn, error) { return conn, nil })(all)
|
||||
m := all.forModule("m").(*moduleBus)
|
||||
|
||||
open := func(t *testing.T) *ledger.Ledger {
|
||||
t.Helper()
|
||||
l, err := all.ledger("m", "records")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return l
|
||||
}
|
||||
apply := func(t *testing.T, l *ledger.Ledger, instance, operation, body string) (buscalls.Applied, error) {
|
||||
t.Helper()
|
||||
return l.Apply(buscalls.ApplyRequest{Ledger: "records", Instance: instance, Operation: operation, Body: json.RawMessage(body)},
|
||||
buscalls.Author{Account: "m"})
|
||||
}
|
||||
must := func(t *testing.T, l *ledger.Ledger, instance, operation, body string) buscalls.Applied {
|
||||
t.Helper()
|
||||
got, err := apply(t, l, instance, operation, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
held := func(t *testing.T, key string) ledger.Record {
|
||||
t.Helper()
|
||||
raw, _, err := open(t).State.Get(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var r ledger.Record
|
||||
if raw != nil {
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
snapshotIsState := func(t *testing.T, key string) {
|
||||
t.Helper()
|
||||
l := open(t)
|
||||
state, _, _ := l.State.Get(key)
|
||||
snap, _, _ := l.Snapshots.Get(key)
|
||||
if string(snap) != string(state) {
|
||||
t.Fatalf("the snapshot of %s is %s, its state %s", key, snap, state)
|
||||
}
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name, key string
|
||||
run func(t *testing.T)
|
||||
}{
|
||||
{"two writers race and one is refused", "decision.race", func(t *testing.T) {
|
||||
l := open(t)
|
||||
l.History = &racing{History: l.History, before: func() { must(t, open(t), "decision/race", "note", `{"says":"first"}`) }}
|
||||
_, err := apply(t, l, "decision/race", "note", `{"says":"second"}`)
|
||||
if !errors.Is(err, ledger.ErrRaced) || !strings.Contains(err.Error(), "not taken") {
|
||||
t.Fatalf("the racing writer was answered %v", err)
|
||||
}
|
||||
if got := held(t, "decision.race"); string(got.State) != `{"notes":[{"says":"first"}]}` {
|
||||
t.Fatalf("the state is %s", got.State)
|
||||
}
|
||||
}},
|
||||
{"two interleaved applies leave the state at the higher sequence", "decision.interleave", func(t *testing.T) {
|
||||
var inner buscalls.Applied
|
||||
l := open(t)
|
||||
l.History = &racing{History: l.History, after: func() { inner = must(t, open(t), "decision/interleave", "note", `{"says":"later"}`) }}
|
||||
outer := must(t, l, "decision/interleave", "note", `{"says":"earlier"}`)
|
||||
got := held(t, "decision.interleave")
|
||||
if inner.Sequence <= outer.Sequence || got.Sequence != inner.Sequence ||
|
||||
string(got.State) != `{"notes":[{"says":"earlier"},{"says":"later"}]}` {
|
||||
t.Fatalf("applied %d then %d, the state at %d: %s", outer.Sequence, inner.Sequence, got.Sequence, got.State)
|
||||
}
|
||||
}},
|
||||
{"a write cut between history and state is repaired by the next apply", "decision.cut", func(t *testing.T) {
|
||||
must(t, open(t), "decision/cut", "note", `{"says":"one"}`)
|
||||
l := open(t)
|
||||
l.State = &cut{State: l.State, once: true}
|
||||
if got := must(t, l, "decision/cut", "note", `{"says":"cut"}`); !got.Taken || !got.StateBehind {
|
||||
t.Fatalf("the cut apply was answered %+v", got)
|
||||
}
|
||||
must(t, open(t), "decision/cut", "note", `{"says":"three"}`)
|
||||
if got := held(t, "decision.cut"); string(got.State) != `{"notes":[{"says":"one"},{"says":"cut"},{"says":"three"}]}` {
|
||||
t.Fatalf("the state is %s", got.State)
|
||||
}
|
||||
}},
|
||||
{"a full state bucket answers taken with its sequence and catches up", "decision.full", func(t *testing.T) {
|
||||
must(t, open(t), "decision/full", "note", `{"says":"small"}`)
|
||||
kv, _ := js.KeyValue(ctx, "m_records")
|
||||
status, _ := kv.Status(ctx)
|
||||
if _, err := js.UpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "m_records", MaxBytes: int64(status.Bytes()) + 64}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := must(t, open(t), "decision/full", "note", fmt.Sprintf(`{"says":%q}`, strings.Repeat("x", 512)))
|
||||
if !got.Taken || got.Sequence == 0 || !got.StateBehind {
|
||||
t.Fatalf("the full bucket's apply was answered %+v", got)
|
||||
}
|
||||
if _, err := js.UpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "m_records", MaxBytes: -1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := must(t, open(t), "decision/full", "note", `{"says":"room"}`)
|
||||
if st := held(t, "decision.full"); after.StateBehind || st.Sequence != after.Sequence || !strings.Contains(string(st.State), strings.Repeat("x", 512)) {
|
||||
t.Fatalf("after room: %+v, the state at %d", after, st.Sequence)
|
||||
}
|
||||
}},
|
||||
{"an instance whose operations expired rebuilds from its snapshot", "decision.expired", func(t *testing.T) {
|
||||
must(t, open(t), "decision/expired", "title", `{"title":"Kept"}`)
|
||||
want := held(t, "decision.expired")
|
||||
s, _ := js.Stream(ctx, "LEDGER_m_records")
|
||||
if err := s.Purge(ctx, jetstream.WithPurgeSubject("mesh.ledger.m.records.history.decision.expired")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, _ := js.KeyValue(ctx, "m_records")
|
||||
_ = kv.Purge(ctx, "decision.expired")
|
||||
got, err := open(t).Rebuild(buscalls.RebuildRequest{Instance: "decision/expired", From: buscalls.RebuildFromSnapshot})
|
||||
if st := held(t, "decision.expired"); err != nil || got.Written != 1 || string(st.State) != string(want.State) {
|
||||
t.Fatalf("rebuilt %+v, %v: the state %s, want %s", got, err, st.State, want.State)
|
||||
}
|
||||
}},
|
||||
{"rebuild, reindex and lag repair by hand", "decision.repair", func(t *testing.T) {
|
||||
must(t, open(t), "decision/repair", "title", `{"title":"Repaired"}`)
|
||||
kv, _ := js.KeyValue(ctx, "m_records")
|
||||
_ = kv.Purge(ctx, "decision.repair")
|
||||
l := open(t)
|
||||
lag, err := l.Lag(buscalls.LedgerRequest{})
|
||||
if err != nil || !strings.Contains(fmt.Sprint(lag), "decision/repair") || len(lag.Indexes) == 0 {
|
||||
t.Fatalf("lag says %+v, %v", lag, err)
|
||||
}
|
||||
if got, err := l.Rebuild(buscalls.RebuildRequest{Instance: "decision/repair"}); err != nil || got.Written != 1 {
|
||||
t.Fatalf("rebuilt %+v, %v", got, err)
|
||||
}
|
||||
if _, err := l.Reindex(buscalls.ReindexRequest{Index: "by-title"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if lag, err := l.Lag(buscalls.LedgerRequest{}); err != nil || len(lag.Instances) != 0 || len(lag.Indexes) != 0 {
|
||||
t.Fatalf("lag after the repairs says %+v, %v", lag, err)
|
||||
}
|
||||
q, err := l.Query(buscalls.QueryRequest{Index: "by-title", Key: "Repaired.>"})
|
||||
if err != nil || len(q.Entries) != 1 {
|
||||
t.Fatalf("queried %+v, %v", q, err)
|
||||
}
|
||||
}},
|
||||
{"a forged author is refused", "decision.forged", func(t *testing.T) {
|
||||
_, err := m.BusState(buscalls.LedgerApply, json.RawMessage(`{"ledger":"records","instance":"decision/forged","operation":"title","body":{"title":"T","author":"op@shanks"},"call":"forged"}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "the call forged, which is not a tool call in flight") {
|
||||
t.Fatalf("a forged call was answered %v", err)
|
||||
}
|
||||
if last, _ := open(t).History.Last("decision.forged"); last != 0 {
|
||||
t.Fatalf("the forged apply was appended at %d", last)
|
||||
}
|
||||
}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
c.run(t)
|
||||
if c.key != "decision.forged" {
|
||||
snapshotIsState(t, c.key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/ledger"
|
||||
)
|
||||
|
||||
// A module's ledgers as the tool runner serves them (novox/hq ADR 0597, design 25 § 12.4, § 12.5; issue 1966): opened
|
||||
// for the module whose bundle or whose repair tool asks, from its own bindings alone, with its types and the connection
|
||||
// its history is published on.
|
||||
|
||||
// ledgerSource is how the runner opens a module's ledger beside its binding: the connection its history and snapshots
|
||||
// are published on (the module's ledger credential, ADR 0597 §9).
|
||||
type ledgerSource struct {
|
||||
writer func(module string) (*bus.Conn, error)
|
||||
}
|
||||
|
||||
// callsInFlight are the tool calls the runtime is serving, each with the module whose bundle serves it and the caller
|
||||
// the runtime proved from the subject the call arrived on (bus.CalledSubject, novox/hq issue 365).
|
||||
type callsInFlight struct {
|
||||
mu sync.Mutex
|
||||
of map[string]callInFlight
|
||||
}
|
||||
|
||||
type callInFlight struct{ module, caller string }
|
||||
|
||||
// begin keeps a call of module in flight for caller, under an id no bundle can guess, until end.
|
||||
func (f *callsInFlight) begin(module, caller string) (string, func()) {
|
||||
var raw [16]byte
|
||||
_, _ = rand.Read(raw[:])
|
||||
id := hex.EncodeToString(raw[:])
|
||||
f.mu.Lock()
|
||||
if f.of == nil {
|
||||
f.of = map[string]callInFlight{}
|
||||
}
|
||||
f.of[id] = callInFlight{module, caller}
|
||||
f.mu.Unlock()
|
||||
return id, func() {
|
||||
f.mu.Lock()
|
||||
delete(f.of, id)
|
||||
f.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// authorOf is the author an apply of module naming call is stamped with (ADR 0597 §7): the module's account, and the
|
||||
// caller the runtime proved for that call; naming no call, the account alone. A call not in flight, or another
|
||||
// module's, is refused, naming it. Nothing the bundle writes is taken as the author.
|
||||
func (f *callsInFlight) authorOf(module, account, call string) (buscalls.Author, error) {
|
||||
author := buscalls.Author{Account: account}
|
||||
if call == "" {
|
||||
return author, nil
|
||||
}
|
||||
f.mu.Lock()
|
||||
inFlight, ok := f.of[call]
|
||||
f.mu.Unlock()
|
||||
switch {
|
||||
case !ok:
|
||||
return author, fmt.Errorf("the apply names the call %s, which is not a tool call in flight here; an apply names the "+
|
||||
"call it serves as its tools/call's _meta %s gave it, or none (novox/hq ADR 0597 §7)", call, buscalls.MetaCall)
|
||||
case inFlight.module != module:
|
||||
return author, fmt.Errorf("the apply names the call %s, which is another module's tool call; %s names only its own "+
|
||||
"(novox/hq ADR 0597 §7)", call, module)
|
||||
}
|
||||
author.Caller = inFlight.caller
|
||||
return author, nil
|
||||
}
|
||||
|
||||
// Begin keeps one of the module's tool calls in flight for the caller the runtime proved (launch.Bus).
|
||||
func (b *moduleBus) Begin(caller string) (string, func()) { return b.all.calls.begin(b.module, caller) }
|
||||
|
||||
// ledger opens one of a module's ledgers by its local name.
|
||||
func (c *consumers) ledger(module, local string) (*ledger.Ledger, error) {
|
||||
b, err := c.conn.Binding(module, local, catalogue.StorageLedger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Its types travel on its binding, copied by the grants part from the entry the manifest-accepted event carries
|
||||
// (mesh-sdk membership.BusBinding.Types): the runner reads no other part's state for them (ADR 0596).
|
||||
ts := b.Types
|
||||
if len(ts) == 0 {
|
||||
return nil, fmt.Errorf("%s's ledger %q is bound with no types, so no operation can be checked against one: its "+
|
||||
"binding carries the types its manifest asks for in `requires` (novox/hq ADR 0597 §1)", module, local)
|
||||
}
|
||||
var writer *bus.Conn
|
||||
if c.ledgers.writer != nil {
|
||||
if writer, err = c.ledgers.writer(module); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
history, state, snapshots, err := c.conn.LedgerStores(module, b, writer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The owning module's authenticated account: the user of the connection its history is published on.
|
||||
account := module
|
||||
if writer != nil && writer.Credential().User != "" {
|
||||
account = writer.Credential().User
|
||||
}
|
||||
return &ledger.Ledger{Module: module, Name: local, Account: account, Types: ts, History: history, State: state,
|
||||
Snapshots: snapshots, Indexes: c.conn.LedgerIndexes(b), Logf: c.logf}, nil
|
||||
}
|
||||
|
||||
// ledgerCall answers a bundle's ledger call, as the module whose bundle asks.
|
||||
func (b *moduleBus) ledgerCall(method string, params json.RawMessage) (any, error) {
|
||||
var named struct {
|
||||
Ledger string `json:"ledger"`
|
||||
}
|
||||
if err := decode(method, params, &named); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if method == buscalls.LedgerApply {
|
||||
var r buscalls.ApplyRequest
|
||||
if err := decode(method, params, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The call is judged first: a forged one is refused before anything of the ledger is touched.
|
||||
if _, err := b.all.calls.authorOf(b.module, "", r.Call); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l, err := b.all.ledger(b.module, r.Ledger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
author, err := b.all.calls.authorOf(b.module, l.Account, r.Call)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return l.Apply(r, author)
|
||||
}
|
||||
l, err := b.all.ledger(b.module, named.Ledger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch method {
|
||||
case buscalls.LedgerGet:
|
||||
var r buscalls.InstanceRequest
|
||||
_ = json.Unmarshal(params, &r)
|
||||
return l.Get(r)
|
||||
case buscalls.LedgerHistory:
|
||||
var r buscalls.HistoryRequest
|
||||
_ = json.Unmarshal(params, &r)
|
||||
return l.ReadHistory(r)
|
||||
case buscalls.LedgerQuery:
|
||||
var r buscalls.QueryRequest
|
||||
_ = json.Unmarshal(params, &r)
|
||||
return l.Query(r)
|
||||
case buscalls.LedgerRebuild:
|
||||
var r buscalls.RebuildRequest
|
||||
_ = json.Unmarshal(params, &r)
|
||||
return l.Rebuild(r)
|
||||
case buscalls.LedgerReindex:
|
||||
var r buscalls.ReindexRequest
|
||||
_ = json.Unmarshal(params, &r)
|
||||
return l.Reindex(r)
|
||||
case buscalls.LedgerSnapshot:
|
||||
var r buscalls.InstanceRequest
|
||||
_ = json.Unmarshal(params, &r)
|
||||
return l.Snapshot(r)
|
||||
case buscalls.LedgerLag:
|
||||
var r buscalls.LedgerRequest
|
||||
_ = json.Unmarshal(params, &r)
|
||||
return l.Lag(r)
|
||||
}
|
||||
return nil, fmt.Errorf("the runtime answers no %s", method)
|
||||
}
|
||||
|
||||
// ledgerWatch hands a bundle each change of its ledger's states.
|
||||
func (b *moduleBus) ledgerWatch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
var r buscalls.LedgerWatchRequest
|
||||
if err := decode(buscalls.LedgerWatch, params, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l, err := b.all.ledger(b.module, r.Ledger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return l.Watch(r, func(change buscalls.InstanceChange) error {
|
||||
raw, err := json.Marshal(change)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return deliver(raw)
|
||||
})
|
||||
}
|
||||
|
||||
// hasLedger is whether a module's membership binds a ledger.
|
||||
func hasLedger(conn *bus.Conn, module string) bool {
|
||||
for _, b := range conn.Bindings(module) {
|
||||
if b.Storage == catalogue.StorageLedger {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The repair tools the runner serves for every module owning a ledger (design 25 § 12.5), named so no bundle's tool is
|
||||
// taken for one.
|
||||
const (
|
||||
RepairRebuild = "ledger_rebuild"
|
||||
RepairReindex = "ledger_reindex"
|
||||
RepairSnapshot = "ledger_snapshot"
|
||||
RepairLag = "ledger_lag"
|
||||
)
|
||||
|
||||
// repairClash is why a bundle's tools of a module cannot be served beside the runner's own ledger repairs: a tool named as
|
||||
// one of them, which the runner serves for every module owning a ledger, would be taken for it or hide it. It names the
|
||||
// tool and the repair it clashes with; empty when none does.
|
||||
func repairClash(module string, tools []launch.Tool) string {
|
||||
for _, t := range tools {
|
||||
for _, repair := range []string{RepairRebuild, RepairReindex, RepairSnapshot, RepairLag} {
|
||||
if t.Name == repair {
|
||||
return fmt.Sprintf("%s's bundle serves a tool named %s, which is also the ledger repair the runner serves for %s "+
|
||||
"(novox/hq ADR 0597 §10): rename the bundle's tool", module, t.Name, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// The effects a tool carries (novox/hq ADR 0387 §1).
|
||||
const (
|
||||
EffectReads = "reads"
|
||||
EffectActs = "acts"
|
||||
)
|
||||
|
||||
// repairTools are a module's ledger repairs as tools, each limited to its own ledgers: the request's ledger is a local
|
||||
// name of the module whose tool is called.
|
||||
func (c *consumers) repairTools(module string) []launch.Tool {
|
||||
schema := func(props string, required ...string) json.RawMessage {
|
||||
req, _ := json.Marshal(append([]string{"ledger"}, required...))
|
||||
return json.RawMessage(`{"type":"object","properties":{"ledger":{"type":"string","description":"the ledger's local name"}` +
|
||||
props + `},"required":` + string(req) + `}`)
|
||||
}
|
||||
instance := `,"instance":{"type":"string","description":"one instance, such as decision/0290; every instance when empty"}`
|
||||
run := func(do func(l *ledger.Ledger, args json.RawMessage) (any, error)) func(string, json.RawMessage) (json.RawMessage, error) {
|
||||
return func(_ string, args json.RawMessage) (json.RawMessage, error) {
|
||||
var named struct {
|
||||
Ledger string `json:"ledger"`
|
||||
}
|
||||
if err := json.Unmarshal(args, &named); err != nil || named.Ledger == "" {
|
||||
return nil, errors.New("a repair names the ledger, by its local name")
|
||||
}
|
||||
l, err := c.ledger(module, named.Ledger)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer, err := do(l, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(answer)
|
||||
}
|
||||
}
|
||||
return []launch.Tool{
|
||||
{Name: RepairRebuild, Effect: EffectActs, Input: schema(instance + `,"from":{"type":"string","enum":["history","snapshot"],"description":"derive from the whole history (the default) or from the snapshot and the operations after it"}`),
|
||||
Description: "Derive a ledger instance's state again from its history or its snapshot, and write it and its snapshot (novox/hq ADR 0597 §10).",
|
||||
Run: run(func(l *ledger.Ledger, args json.RawMessage) (any, error) {
|
||||
var r buscalls.RebuildRequest
|
||||
_ = json.Unmarshal(args, &r)
|
||||
return l.Rebuild(r)
|
||||
})},
|
||||
{Name: RepairReindex, Effect: EffectActs, Input: schema(`,"index":{"type":"string","description":"the declared index"}`, "index"),
|
||||
Description: "Build a ledger's declared index again from its states, derived from its history (novox/hq ADR 0597 §4, §10).",
|
||||
Run: run(func(l *ledger.Ledger, args json.RawMessage) (any, error) {
|
||||
var r buscalls.ReindexRequest
|
||||
_ = json.Unmarshal(args, &r)
|
||||
return l.Reindex(r)
|
||||
})},
|
||||
{Name: RepairSnapshot, Effect: EffectActs, Input: schema(instance),
|
||||
Description: "Rewrite the snapshot of a ledger instance that is behind its state (novox/hq ADR 0597 §8, §10).",
|
||||
Run: run(func(l *ledger.Ledger, args json.RawMessage) (any, error) {
|
||||
var r buscalls.InstanceRequest
|
||||
_ = json.Unmarshal(args, &r)
|
||||
return l.Snapshot(r)
|
||||
})},
|
||||
{Name: RepairLag, Effect: EffectReads, Input: schema(""),
|
||||
Description: "Say every ledger instance whose state or snapshot is behind its history, and each declared index behind it (novox/hq ADR 0597 §10).",
|
||||
Run: run(func(l *ledger.Ledger, args json.RawMessage) (any, error) {
|
||||
var r buscalls.LedgerRequest
|
||||
_ = json.Unmarshal(args, &r)
|
||||
return l.Lag(r)
|
||||
})},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/membership"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// 1966/AC16: given a module with a ledger binding, when the runner lists its tools, then the four repair tools are
|
||||
// among them, marked acts or reads; a module without one has none, and a binding written later brings them.
|
||||
func TestTheLedgerRepairsAreToolsOfEveryLedgerOwner(t *testing.T) {
|
||||
mesh := mt.New(t)
|
||||
owner := mt.MembershipOf("owner", "anchor", true, nil)
|
||||
owner.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger,
|
||||
History: "LEDGER_owner_records", Subject: "mesh.ledger.owner.records.history", State: "owner_records",
|
||||
Snapshot: "mesh.ledger.owner.records.snapshot"}}
|
||||
plain := mt.MembershipOf("plain", "anchor", true, nil)
|
||||
mesh.Issue(t, owner)
|
||||
mesh.Issue(t, plain)
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
stop, err := Run(nodeTools, []Served{{"owner", []string{mt.Fixture("caller-echo.py")}},
|
||||
{"plain", []string{mt.Fixture("caller-echo.py")}}}, nil, func(string, ...any) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
asker := connect(t, "console", "workstation")
|
||||
listed := func(module string) (string, error) {
|
||||
answered, err := asker.Ask(module+"."+ToolsVerb, map[string]any{}, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var got ToolsAnswer
|
||||
if err := json.Unmarshal(answered.Result, &got); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var names []string
|
||||
for _, tool := range got.Tools {
|
||||
names = append(names, tool.Name+":"+tool.Effect)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return strings.Join(names, " "), nil
|
||||
}
|
||||
for _, c := range []struct{ name, module, want string }{
|
||||
{"a ledger owner lists the four repairs", "owner", "ledger_lag:reads ledger_rebuild:acts ledger_reindex:acts ledger_snapshot:acts who:"},
|
||||
{"a module with no ledger has none", "plain", "who:"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var got string
|
||||
mt.Until(t, func() (err error) { got, err = listed(c.module); return })
|
||||
if got != c.want {
|
||||
t.Fatalf("%s lists %q, want %q", c.module, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("a ledger bound later brings them", func(t *testing.T) {
|
||||
plain.Bindings = owner.Bindings
|
||||
mesh.Issue(t, plain)
|
||||
mt.Until(t, func() error {
|
||||
got, err := listed("plain")
|
||||
if err == nil && !strings.Contains(got, "ledger_rebuild:acts") {
|
||||
err = errors.New("listed " + got)
|
||||
}
|
||||
return err
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// aLedger raises one ledger of module m on the bus as the bus module would create it, binds it in m's membership, and
|
||||
// answers the runtime's buses for m and for other; its types travel on its binding, and a second binding carries none.
|
||||
func aLedger(t *testing.T) (m, other *moduleBus, mesh *mt.Mesh) {
|
||||
t.Helper()
|
||||
mesh = mt.New(t)
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, _ := jetstream.New(nc)
|
||||
ctx := context.Background()
|
||||
for _, cfg := range []jetstream.StreamConfig{
|
||||
{Name: "LEDGER_m_records", Subjects: []string{"mesh.ledger.m.records.history.>"}},
|
||||
{Name: "SNAPSHOT_m_records", Subjects: []string{"mesh.ledger.m.records.snapshot.>"}, MaxMsgsPerSubject: 1},
|
||||
} {
|
||||
if _, err := js.CreateStream(ctx, cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "m_records"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mem := mt.MembershipOf("m", "anchor", false, nil)
|
||||
mem.Bindings = []membership.BusBinding{{Local: "records", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger,
|
||||
History: "LEDGER_m_records", Subject: "mesh.ledger.m.records.history", State: "m_records", Snapshot: "mesh.ledger.m.records.snapshot",
|
||||
Types: []catalogue.LedgerType{{Name: "decision", Schema: map[string]catalogue.LedgerField{"title": {Type: "string"}},
|
||||
Operations: []catalogue.LedgerOperation{{Name: "title", Effect: catalogue.EffectSetFields, Fields: []string{"title"}, Author: true}}}}},
|
||||
{Local: "untyped", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, History: "LEDGER_m_records",
|
||||
State: "m_records"},
|
||||
{Local: "elsewhere", Storage: catalogue.StorageLedger, Template: catalogue.TemplateLedger, History: "LEDGER_m_records",
|
||||
Subject: "mesh.ledger.m.records.history", State: "m_records", Types: []catalogue.LedgerType{{Name: "decision"}}}}
|
||||
mesh.Issue(t, mem)
|
||||
mesh.Issue(t, mt.MembershipOf("other", "anchor", false, nil))
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("m")
|
||||
conn.Follow("other")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("m") == nil || conn.Membership("other") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}, ledgers: ledgerSource{
|
||||
writer: func(string) (*bus.Conn, error) { return conn, nil },
|
||||
}}
|
||||
return all.forModule("m").(*moduleBus), all.forModule("other").(*moduleBus), mesh
|
||||
}
|
||||
|
||||
// 1966/AC17, 1966/AC18: the author an apply is stamped with is the caller the runtime proved for the call in flight it
|
||||
// names, whatever author the bundle wrote; another module's call, or one not in flight, is refused naming it (ADR 0597 §7).
|
||||
func TestAnApplyIsStampedWithTheCallerTheRuntimeProved(t *testing.T) {
|
||||
m, other, _ := aLedger(t)
|
||||
call, end := m.Begin("op@shanks")
|
||||
defer end()
|
||||
theirs, endTheirs := other.Begin("op@shanks")
|
||||
defer endTheirs()
|
||||
forged := `"body":{"title":"T","author":{"account":"x","caller":"forged"}}`
|
||||
for _, c := range []struct{ name, params, caller, refusal string }{
|
||||
{"1966/AC17 a call in flight stamps its proven caller", `{"ledger":"records","instance":"decision/1","operation":"title",` + forged + `,"call":"` + call + `","author":"forged"}`, "op@shanks", ""},
|
||||
{"naming no call stamps the account alone", `{"ledger":"records","instance":"decision/2","operation":"title",` + forged + `}`, "", ""},
|
||||
{"1966/AC18 another module's call is refused, naming it", `{"ledger":"records","instance":"decision/3","operation":"title",` + forged + `,"call":"` + theirs + `"}`, "", "the call " + theirs + ", which is another module's"},
|
||||
{"a ledger bound with no types is refused, naming it", `{"ledger":"untyped","instance":"decision/3","operation":"title",` + forged + `}`, "", `m's ledger "untyped" is bound with no types`},
|
||||
{"a binding naming subjects its account is not granted is refused, naming both", `{"ledger":"elsewhere","instance":"decision/3","operation":"title",` + forged + `}`, "", "bound with the history subject mesh.ledger.m.records.history, and its ledger account is granted mesh.ledger.m.elsewhere.history alone"},
|
||||
{"a call not in flight is refused, naming it", `{"ledger":"records","instance":"decision/3","operation":"title",` + forged + `,"call":"c0ffee"}`, "", "the call c0ffee, which is not a tool call in flight"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var req struct {
|
||||
Instance string `json:"instance"`
|
||||
}
|
||||
_ = json.Unmarshal([]byte(c.params), &req)
|
||||
_, err := m.BusState(buscalls.LedgerApply, json.RawMessage(c.params))
|
||||
if c.refusal != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.refusal) {
|
||||
t.Fatalf("applied with %v, want refused with %q", err, c.refusal)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := m.BusState(buscalls.LedgerHistory, json.RawMessage(`{"ledger":"records","instance":"`+req.Instance+`"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var page buscalls.HistoryPage
|
||||
_ = json.Unmarshal(raw, &page)
|
||||
if len(page.Operations) != 1 || page.Operations[0].Author != (buscalls.Author{Account: "m", Caller: c.caller}) {
|
||||
t.Fatalf("the history holds %+v, want the author {m %s}", page.Operations, c.caller)
|
||||
}
|
||||
state, _ := m.BusState(buscalls.LedgerGet, json.RawMessage(`{"ledger":"records","instance":"`+req.Instance+`"}`))
|
||||
if !strings.Contains(string(state), `"author":{"account":"m"`) || strings.Contains(string(state), "forged") {
|
||||
t.Fatalf("the state took the bundle's author: %s", state)
|
||||
}
|
||||
})
|
||||
}
|
||||
end()
|
||||
if _, err := m.BusState(buscalls.LedgerApply, json.RawMessage(`{"ledger":"records","instance":"decision/4","operation":"title","call":"`+call+`"}`)); err == nil {
|
||||
t.Fatal("a call that ended was still taken as in flight")
|
||||
}
|
||||
}
|
||||
|
||||
// issue 1966 TLS-2: a bundle's tool named as one of the runner's ledger repairs is refused at the load, naming the tool and the
|
||||
// repair; any other name is served.
|
||||
func TestABundleToolNamedAsALedgerRepairIsRefused(t *testing.T) {
|
||||
for _, c := range []struct{ name, tool, want string }{
|
||||
{"rebuild", RepairRebuild, "m's bundle serves a tool named ledger_rebuild, which is also the ledger repair the runner serves for m"},
|
||||
{"reindex", RepairReindex, "ledger_reindex"},
|
||||
{"snapshot", RepairSnapshot, "ledger_snapshot"},
|
||||
{"lag", RepairLag, "ledger_lag"},
|
||||
{"another name", "ledger_status", ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := repairClash("m", []launch.Tool{{Name: "first"}, {Name: c.tool}})
|
||||
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
|
||||
t.Errorf("clash %q, want one containing %q", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// issue 1966 TLS-3: every method of a capability the runner states is a call it answers, and the two states its calls under
|
||||
// no other name; none answers "the runtime answers no ...".
|
||||
func TestEveryCallOfAStatedCapabilityIsAnswered(t *testing.T) {
|
||||
m, _, _ := aLedger(t)
|
||||
if got := strings.Join(CallCapabilities(), ","); got != "bus-state-calls,ledger-calls" {
|
||||
t.Fatalf("the runner states %s", got)
|
||||
}
|
||||
for capability, methods := range callFamilies {
|
||||
for _, method := range methods {
|
||||
t.Run(capability+" "+method, func(t *testing.T) {
|
||||
_, err := m.BusState(method, json.RawMessage(`"not a request"`))
|
||||
if err != nil && strings.Contains(err.Error(), "the runtime answers no") {
|
||||
if stop, werr := m.BusStateWatch(method, json.RawMessage(`"not a request"`), func(json.RawMessage) error { return nil }); werr != nil && strings.Contains(werr.Error(), "the runtime answers no") {
|
||||
t.Errorf("%s is stated under %s and answered by neither BusState nor BusStateWatch", method, capability)
|
||||
} else if stop != nil {
|
||||
stop()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
"github.com/novox/mesh-tools/node-tools/internal/launch"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// logBuses is a module, issues, that keeps a log `changes` on a real bus, and a module, other, on the
|
||||
// same runtime that was issued no log — each reached as a bundle reaches it: mesh/log.<verb> with its
|
||||
// JSON, through its module's bus (novox/hq ADR 0297).
|
||||
func logBuses(t *testing.T, maxBytes int64) (issues, other launch.Bus, mesh *mt.Mesh) {
|
||||
t.Helper()
|
||||
mesh = mt.New(t)
|
||||
m := mt.MembershipOf("issues", "anchor", false, nil)
|
||||
changes := mesh.Log(t, "issues", "changes", maxBytes)
|
||||
m.Logs = []bus.LogIssued{changes}
|
||||
mesh.Issue(t, m)
|
||||
// other reads issues' log, as its manifest's reads declare it (ADR 0571): `<module>.<log>`, without writing it.
|
||||
o := mt.MembershipOf("other", "anchor", false, nil)
|
||||
o.Logs = []bus.LogIssued{{Name: "issues.changes", Stream: changes.Stream, Subject: changes.Subject}}
|
||||
mesh.Issue(t, o)
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
conn.Follow("issues")
|
||||
conn.Follow("other")
|
||||
mt.Until(t, func() error {
|
||||
if conn.Membership("issues") == nil || conn.Membership("other") == nil {
|
||||
return errors.New("not issued yet")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
return all.forModule("issues"), all.forModule("other"), mesh
|
||||
}
|
||||
|
||||
// logAsk is a bundle's mesh/log.<verb>, with its params exactly as they go over the wire.
|
||||
func logAsk(b launch.Bus, verb, params string) (json.RawMessage, error) {
|
||||
return b.Log(verb, json.RawMessage(params))
|
||||
}
|
||||
|
||||
func appendOf(t *testing.T, b launch.Bus, params string) uint64 {
|
||||
t.Helper()
|
||||
raw, err := logAsk(b, "append", params)
|
||||
if err != nil {
|
||||
t.Fatalf("append %s: %v", params, err)
|
||||
}
|
||||
var r struct {
|
||||
Sequence *uint64 `json:"sequence"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &r); err != nil || r.Sequence == nil {
|
||||
t.Fatalf("append answered %s, not {sequence}", raw)
|
||||
}
|
||||
return *r.Sequence
|
||||
}
|
||||
|
||||
// wirePage is a read's answer with exactly the contract's field names.
|
||||
type wirePage struct {
|
||||
Entries []struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
At string `json:"at"`
|
||||
} `json:"entries"`
|
||||
Next uint64 `json:"next"`
|
||||
Last uint64 `json:"last"`
|
||||
}
|
||||
|
||||
func readOf(t *testing.T, b launch.Bus, params string) wirePage {
|
||||
t.Helper()
|
||||
raw, err := logAsk(b, "read", params)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", params, err)
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
_ = json.Unmarshal(raw, &fields)
|
||||
for _, f := range []string{"entries", "next", "last"} {
|
||||
if _, ok := fields[f]; !ok {
|
||||
t.Fatalf("read answered %s, without %q", raw, f)
|
||||
}
|
||||
}
|
||||
var p wirePage
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func infoOf(t *testing.T, b launch.Bus) map[string]uint64 {
|
||||
t.Helper()
|
||||
raw, err := logAsk(b, "info", `{"log":"changes"}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var info map[string]uint64
|
||||
if err := json.Unmarshal(raw, &info); err != nil {
|
||||
t.Fatalf("info answered %s: %v", raw, err)
|
||||
}
|
||||
for _, f := range []string{"entries", "bytes", "max_bytes", "first", "last"} {
|
||||
if _, ok := info[f]; !ok {
|
||||
t.Fatalf("info answered %s, without %q", raw, f)
|
||||
}
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// Each verb against a real bus: append answers the sequence, read pages from a sequence on, of a key
|
||||
// pattern, with the sequence to continue from (0 at the end) and the log's last, each entry's time in
|
||||
// RFC 3339; last is a key's last entry or null; info says entries, bytes, cap, first and last — and no
|
||||
// consumer is made for any of it.
|
||||
func TestALogAnswersEachVerbOnTheBus(t *testing.T) {
|
||||
b, other, mesh := logBuses(t, 1<<20)
|
||||
before := time.Now().Add(-time.Second)
|
||||
|
||||
var seqs []uint64
|
||||
for i, key := range []string{"12", "13", "12", "14.notes", "12"} {
|
||||
seqs = append(seqs, appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":%q,"value":{"op":"move","n":%d}}`, key, i)))
|
||||
}
|
||||
for i := 1; i < len(seqs); i++ {
|
||||
if seqs[i] != seqs[i-1]+1 {
|
||||
t.Fatalf("sequences %v are not the log's order", seqs)
|
||||
}
|
||||
}
|
||||
|
||||
// Everything, no keys and no limit: the default page holds all five.
|
||||
all := readOf(t, b, `{"log":"changes","from":0}`)
|
||||
if len(all.Entries) != 5 || all.Next != 0 || all.Last != seqs[4] {
|
||||
t.Fatalf("a read of everything: %d entries, next %d, last %d", len(all.Entries), all.Next, all.Last)
|
||||
}
|
||||
for i, e := range all.Entries {
|
||||
at, err := time.Parse(time.RFC3339, e.At)
|
||||
if err != nil || at.Before(before) || at.After(time.Now().Add(time.Second)) {
|
||||
t.Errorf("entry %d's time %q is not RFC 3339 and now: %v", e.Sequence, e.At, err)
|
||||
}
|
||||
if e.Sequence != seqs[i] || string(e.Value) != fmt.Sprintf(`{"op":"move","n":%d}`, i) {
|
||||
t.Errorf("entry %d: %d %s %s", i, e.Sequence, e.Key, e.Value)
|
||||
}
|
||||
}
|
||||
if all.Entries[3].Key != "14.notes" {
|
||||
t.Errorf("a key of two names read back as %q", all.Entries[3].Key)
|
||||
}
|
||||
|
||||
// One key's entries, a page of two at a time.
|
||||
first := readOf(t, b, `{"log":"changes","from":1,"keys":"12","limit":2}`)
|
||||
if len(first.Entries) != 2 || first.Entries[0].Sequence != seqs[0] || first.Entries[1].Sequence != seqs[2] ||
|
||||
first.Next != seqs[2]+1 {
|
||||
t.Fatalf("the first page of key 12: %+v", first)
|
||||
}
|
||||
second := readOf(t, b, fmt.Sprintf(`{"log":"changes","from":%d,"keys":"12","limit":2}`, first.Next))
|
||||
if len(second.Entries) != 1 || second.Entries[0].Sequence != seqs[4] || second.Next != 0 {
|
||||
t.Fatalf("the second page of key 12: %+v", second)
|
||||
}
|
||||
// A pattern: `*` one name, `>` the rest.
|
||||
if p := readOf(t, b, `{"log":"changes","from":1,"keys":"14.*"}`); len(p.Entries) != 1 || p.Entries[0].Key != "14.notes" {
|
||||
t.Fatalf("14.*: %+v", p)
|
||||
}
|
||||
if p := readOf(t, b, `{"log":"changes","from":1,"keys":"14.>"}`); len(p.Entries) != 1 {
|
||||
t.Fatalf("14.>: %+v", p)
|
||||
}
|
||||
// Past the end: nothing, nothing follows.
|
||||
if p := readOf(t, b, fmt.Sprintf(`{"log":"changes","from":%d}`, seqs[4]+1)); len(p.Entries) != 0 || p.Next != 0 || p.Last != seqs[4] {
|
||||
t.Fatalf("a read past the end: %+v", p)
|
||||
}
|
||||
if p := readOf(t, b, `{"log":"changes","from":1,"keys":"99"}`); len(p.Entries) != 0 || p.Next != 0 {
|
||||
t.Fatalf("a key with no entries: %+v", p)
|
||||
}
|
||||
|
||||
raw, err := logAsk(b, "last", `{"log":"changes","key":"12"}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var last map[string]json.RawMessage
|
||||
_ = json.Unmarshal(raw, &last)
|
||||
if string(last["sequence"]) != fmt.Sprint(seqs[4]) || string(last["key"]) != `"12"` ||
|
||||
string(last["value"]) != `{"op":"move","n":4}` || len(last["at"]) == 0 {
|
||||
t.Fatalf("the last entry of 12: %s", raw)
|
||||
}
|
||||
if raw, err := logAsk(b, "last", `{"log":"changes","key":"99"}`); err != nil || string(raw) != "null" {
|
||||
t.Fatalf("the last entry of a key with none: %s %v", raw, err)
|
||||
}
|
||||
|
||||
info := infoOf(t, b)
|
||||
if info["entries"] != 5 || info["first"] != seqs[0] || info["last"] != seqs[4] || info["max_bytes"] != 1<<20 ||
|
||||
info["bytes"] == 0 {
|
||||
t.Fatalf("info: %v", info)
|
||||
}
|
||||
if n := mesh.Consumers(t, "LOG_issues_changes"); n != 0 {
|
||||
t.Fatalf("the runtime made %d consumers of the log; it uses none", n)
|
||||
}
|
||||
|
||||
// From a time (ADR 0571): `since` in place of `from`, RFC 3339; both at once are refused.
|
||||
cut := all.Entries[2].At
|
||||
if p := readOf(t, b, fmt.Sprintf(`{"log":"changes","since":%q}`, cut)); len(p.Entries) == 0 || p.Entries[0].Sequence > seqs[2] ||
|
||||
p.Entries[len(p.Entries)-1].Sequence != seqs[4] {
|
||||
t.Fatalf("a read since %s: %+v", cut, p)
|
||||
}
|
||||
if p := readOf(t, b, fmt.Sprintf(`{"log":"changes","since":%q}`, time.Now().Add(time.Hour).Format(time.RFC3339))); len(p.Entries) != 0 {
|
||||
t.Fatalf("a read since an hour ahead: %+v", p)
|
||||
}
|
||||
if _, err := logAsk(b, "read", `{"log":"changes","from":2,"since":"2026-10-29T00:00:00Z"}`); err == nil {
|
||||
t.Fatal("a read from a sequence and a time was not refused")
|
||||
}
|
||||
// The newest of each key, in the log's order; `**` for the rest, as mesh-sdk says it.
|
||||
raw, err = logAsk(b, "newest", `{"log":"changes"}`)
|
||||
var newest []struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
if err != nil || json.Unmarshal(raw, &newest) != nil || len(newest) != 3 || newest[0].Key != "13" || newest[1].Key != "14.notes" ||
|
||||
newest[2].Sequence != seqs[4] {
|
||||
t.Fatalf("newest: %s %v", raw, err)
|
||||
}
|
||||
if raw, err := logAsk(b, "newest", `{"log":"changes","keys":"14.**"}`); err != nil || !strings.Contains(string(raw), `"14.notes"`) ||
|
||||
strings.Contains(string(raw), `"12"`) {
|
||||
t.Fatalf("newest of 14.**: %s %v", raw, err)
|
||||
}
|
||||
// Another module's log, read by the name its reads declare; never written.
|
||||
if p := readOf(t, other, `{"log":"issues.changes","from":1}`); len(p.Entries) != 5 {
|
||||
t.Fatalf("other's read of issues.changes: %+v", p)
|
||||
}
|
||||
if _, err := logAsk(other, "append", `{"log":"issues.changes","key":"1","value":{}}`); err == nil ||
|
||||
!strings.Contains(err.Error(), "without writing it") {
|
||||
t.Fatalf("other's append to the log it reads: %v", err)
|
||||
}
|
||||
// A watch hands each entry from where it is asked, then each new one, one at a time.
|
||||
heard := make(chan bus.LogEntry, 16)
|
||||
stop, err := b.LogWatch(json.RawMessage(fmt.Sprintf(`{"log":"changes","keys":"12","from":%d}`, seqs[2])), func(raw json.RawMessage) error {
|
||||
var e bus.LogEntry
|
||||
_ = json.Unmarshal(raw, &e)
|
||||
heard <- e
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
next := appendOf(t, b, `{"log":"changes","key":"12","value":{"op":"late"}}`)
|
||||
for _, want := range []uint64{seqs[2], seqs[4], next} {
|
||||
select {
|
||||
case e := <-heard:
|
||||
if e.Sequence != want || e.Key != "12" {
|
||||
t.Fatalf("the watch handed %d %s, want %d", e.Sequence, e.Key, want)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("the watch did not hand entry %d", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A page holds about 4 MiB at most, whatever its limit, and says where to continue.
|
||||
func TestALogPageIsBoundedByItsBytes(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 64<<20)
|
||||
big := `"` + strings.Repeat("x", 250*1024) + `"`
|
||||
for i := 0; i < 20; i++ {
|
||||
appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":"k%d","value":%s}`, i, big))
|
||||
}
|
||||
p := readOf(t, b, `{"log":"changes","from":1,"limit":1000}`)
|
||||
if len(p.Entries) == 0 || len(p.Entries) >= 20 || p.Next != p.Entries[len(p.Entries)-1].Sequence+1 {
|
||||
t.Fatalf("a page of 20 entries of 250 KiB: %d entries, next %d", len(p.Entries), p.Next)
|
||||
}
|
||||
got := len(p.Entries)
|
||||
for p.Next != 0 {
|
||||
p = readOf(t, b, fmt.Sprintf(`{"log":"changes","from":%d,"limit":1000}`, p.Next))
|
||||
got += len(p.Entries)
|
||||
}
|
||||
if got != 20 {
|
||||
t.Fatalf("paging read %d of 20 entries", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An append with `expect` is written only when the key's last entry is the one expected — 0 when it
|
||||
// has none — and is otherwise refused with the error a lost mesh/state.update answers: the same code,
|
||||
// the same error, and nothing written. An append without `expect` is not checked.
|
||||
func TestAnAppendAtAStaleExpectIsRefusedAsALostUpdate(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 1<<20)
|
||||
isLost := func(err error) bool {
|
||||
var coded interface{ ErrorCode() int }
|
||||
return errors.As(err, &coded) && coded.ErrorCode() == launch.CodeChanged && errors.Is(err, bus.ErrStateChanged)
|
||||
}
|
||||
|
||||
first := appendOf(t, b, `{"log":"changes","key":"7","value":{"op":"open"},"expect":0}`)
|
||||
if _, err := logAsk(b, "append", `{"log":"changes","key":"7","value":{"op":"open"},"expect":0}`); !isLost(err) {
|
||||
t.Fatalf("a second open of one key at expect 0: %v", err)
|
||||
}
|
||||
second := appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"move"},"expect":%d}`, first))
|
||||
if _, err := logAsk(b, "append", fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"note"},"expect":%d}`, first)); !isLost(err) {
|
||||
t.Fatalf("an append at a stale expect: %v", err)
|
||||
}
|
||||
if _, err := logAsk(b, "append", fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"note"},"expect":%d}`, second+100)); !isLost(err) {
|
||||
t.Fatalf("an append at an expect not yet written: %v", err)
|
||||
}
|
||||
// Another key's entries do not make this key's expect stale.
|
||||
appendOf(t, b, `{"log":"changes","key":"8","value":{"op":"open"}}`)
|
||||
third := appendOf(t, b, fmt.Sprintf(`{"log":"changes","key":"7","value":{"op":"note"},"expect":%d}`, second))
|
||||
// No expect: no check.
|
||||
appendOf(t, b, `{"log":"changes","key":"7","value":{"op":"set"}}`)
|
||||
|
||||
if info := infoOf(t, b); info["entries"] != 5 {
|
||||
t.Fatalf("refused appends wrote: %d entries, not 5", info["entries"])
|
||||
}
|
||||
p := readOf(t, b, `{"log":"changes","from":1,"keys":"7"}`)
|
||||
if len(p.Entries) != 4 || p.Entries[2].Sequence != third {
|
||||
t.Fatalf("key 7 holds %+v", p.Entries)
|
||||
}
|
||||
|
||||
// The refusal is the very error a lost state update is: its code reaches a bundle as CodeChanged.
|
||||
_, err := logAsk(b, "append", `{"log":"changes","key":"7","value":{},"expect":0}`)
|
||||
var coded interface{ ErrorCode() int }
|
||||
if !errors.As(err, &coded) || coded.ErrorCode() != -32010 {
|
||||
t.Fatalf("the refusal carries no lost-update code: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A log is its owner's alone: another module on the same runtime is refused it with the reason, and so
|
||||
// is a log of a name the owner was not issued — before anything is sent.
|
||||
func TestALogNotIssuedIsRefusedWithTheReason(t *testing.T) {
|
||||
issues, other, _ := logBuses(t, 1<<20)
|
||||
for verb, params := range map[string]string{
|
||||
"append": `{"log":"changes","key":"1","value":{}}`,
|
||||
"read": `{"log":"changes","from":1}`,
|
||||
"last": `{"log":"changes","key":"1"}`,
|
||||
"info": `{"log":"changes"}`,
|
||||
} {
|
||||
_, err := logAsk(other, verb, params)
|
||||
if err == nil || !strings.Contains(err.Error(), `other keeps no log called "changes"`) ||
|
||||
!strings.Contains(err.Error(), "issued: issues.changes") {
|
||||
t.Errorf("%s of a log not issued: %v", verb, err)
|
||||
}
|
||||
}
|
||||
_, err := logAsk(issues, "append", `{"log":"moves","key":"1","value":{}}`)
|
||||
if err == nil || !strings.Contains(err.Error(), `issues keeps no log called "moves"`) ||
|
||||
!strings.Contains(err.Error(), "issued: changes") {
|
||||
t.Errorf("an append to a log of a name not issued: %v", err)
|
||||
}
|
||||
if info := infoOf(t, issues); info["entries"] != 0 {
|
||||
t.Fatalf("a refused append wrote: %v", info)
|
||||
}
|
||||
}
|
||||
|
||||
// What the bus cannot hold, or the contract does not allow, is refused with the reason and not sent:
|
||||
// a value that is not JSON or over 256 KiB, a key that is not names joined by dots or over 255 bytes, a
|
||||
// key pattern with `>` not last, a limit over 1000.
|
||||
func TestWhatALogCannotHoldIsRefusedWithTheReason(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 1<<20)
|
||||
over := `"` + strings.Repeat("x", 256*1024) + `"`
|
||||
for params, reason := range map[string]string{
|
||||
`{"log":"changes","key":"1","value":nope}`: "names no log",
|
||||
`{"log":"changes","key":"1"}`: "is JSON",
|
||||
`{"log":"changes","key":"1","value":` + over + `}`: "at most 262144 bytes",
|
||||
`{"log":"changes","key":"a b","value":1}`: "is not a log key",
|
||||
`{"log":"changes","key":"a..b","value":1}`: "is not a log key",
|
||||
`{"log":"changes","key":"a.*","value":1}`: "is not a log key",
|
||||
`{"log":"changes","key":"` + strings.Repeat("k", 256) + `","value":1}`: "is not a log key",
|
||||
`{"key":"1","value":1}`: "names no log",
|
||||
} {
|
||||
if _, err := logAsk(b, "append", params); err == nil || !strings.Contains(err.Error(), reason) {
|
||||
t.Errorf("append %.80s: %v, want %q", params, err, reason)
|
||||
}
|
||||
}
|
||||
for params, reason := range map[string]string{
|
||||
`{"log":"changes","from":1,"keys":"a.>.b"}`: "is not a key pattern",
|
||||
`{"log":"changes","from":1,"limit":1001}`: "1 to 1000",
|
||||
`{"log":"changes","from":1,"limit":-1}`: "1 to 1000",
|
||||
} {
|
||||
if _, err := logAsk(b, "read", params); err == nil || !strings.Contains(err.Error(), reason) {
|
||||
t.Errorf("read %s: %v, want %q", params, err, reason)
|
||||
}
|
||||
}
|
||||
if info := infoOf(t, b); info["entries"] != 0 {
|
||||
t.Fatalf("a refused append wrote: %v", info)
|
||||
}
|
||||
// The largest value allowed is taken, also with the header an expect carries.
|
||||
largest := `"` + strings.Repeat("x", 256*1024-2) + `"`
|
||||
appendOf(t, b, `{"log":"changes","key":"1","value":`+largest+`,"expect":0}`)
|
||||
appendOf(t, b, `{"log":"changes","key":"2","value":`+largest+`}`)
|
||||
}
|
||||
|
||||
// A value naming a credential anywhere in it is refused, as mesh/state.put refuses one (novox/hq ADR
|
||||
// 0201 §13): a log keeps what it is given for good. A value that only counts tokens is taken.
|
||||
func TestALogRefusesAValueNamingACredential(t *testing.T) {
|
||||
b, _, _ := logBuses(t, 1<<20)
|
||||
for _, value := range []string{`{"op":"set","api_key":"s"}`, `{"op":"note","with":{"headers":{"Authorization":"Bearer s"}}}`} {
|
||||
_, err := logAsk(b, "append", `{"log":"changes","key":"1","value":`+value+`}`)
|
||||
if err == nil || !strings.Contains(err.Error(), "names a credential") {
|
||||
t.Errorf("append of %s: %v", value, err)
|
||||
}
|
||||
}
|
||||
if info := infoOf(t, b); info["entries"] != 0 {
|
||||
t.Fatalf("a refused append wrote: %v", info)
|
||||
}
|
||||
appendOf(t, b, `{"log":"changes","key":"1","value":{"op":"note","maxTokens":4096}}`)
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
)
|
||||
|
||||
// A module's code reads its values at runtime (novox/hq ADR 0349 clause 14 as ADR 0370 changes it, issue 685).
|
||||
//
|
||||
// A bundle asks `mesh/provision.get` and `mesh/provision.watch` with `{kind, key}`; the runtime asks its machine's
|
||||
// provisioner client **as the module whose bundle asks** — the module this bus was made for, from the bundle it
|
||||
// launched, never a name the bundle sent (ADR 0370 §4) — and hands the answer back. A watch hands the current value
|
||||
// first and then each change: the client says which value moved, and the runtime asks again. A secret is refused
|
||||
// here, with the reason, and never asked: the client serves no secret on the tool runner's word (§2), and a module
|
||||
// under an account of its own asks the vault itself, which waits on issue 569.
|
||||
|
||||
// The kinds a bundle may ask.
|
||||
const (
|
||||
provisionConfig = "config"
|
||||
provisionBound = "bound"
|
||||
provisionSecret = "secret"
|
||||
)
|
||||
|
||||
// provisionAsked is what a bundle names. Anything else it sends, a module's name among it, is not read.
|
||||
type provisionAsked struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// ProvisionChange is what a watch hands a bundle: the value, or, when it is no longer answered, why.
|
||||
type ProvisionChange struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
// Refused is why the value is no longer answered: the module taken off the machine, the key no longer declared.
|
||||
Refused string `json:"refused,omitempty"`
|
||||
}
|
||||
|
||||
func readProvisionAsked(verb string, params json.RawMessage) (provisionAsked, error) {
|
||||
var asked provisionAsked
|
||||
if err := json.Unmarshal(params, &asked); err != nil || asked.Kind == "" || asked.Key == "" {
|
||||
return asked, fmt.Errorf("mesh/provision.%s names no value: {kind, key}", verb)
|
||||
}
|
||||
switch asked.Kind {
|
||||
case provisionConfig, provisionBound:
|
||||
return asked, nil
|
||||
case provisionSecret:
|
||||
return asked, fmt.Errorf("the secret %s is not asked for at runtime while the module's code runs in the "+
|
||||
"tool runner: the vault hands a secret only to a caller the bus proves, and the bus proves only the tool "+
|
||||
"runner here (novox/hq ADR 0370 §2, ADR 0349 §11); the module's secrets reach it through its files, "+
|
||||
"until it runs under a bus account of its own (issue 569)", asked.Key)
|
||||
}
|
||||
return asked, fmt.Errorf("mesh/provision.%s: %q is not a kind of value a module reads at runtime: config or bound",
|
||||
verb, asked.Kind)
|
||||
}
|
||||
|
||||
// Provision answers a bundle's `get` of one of its module's values.
|
||||
func (b *moduleBus) Provision(verb string, params json.RawMessage) (json.RawMessage, error) {
|
||||
if verb != "get" {
|
||||
return nil, fmt.Errorf("the runtime answers no mesh/provision.%s", verb)
|
||||
}
|
||||
asked, err := readProvisionAsked(verb, params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.all.conn.AskProvisioner(b.module, asked.Kind, asked.Key)
|
||||
}
|
||||
|
||||
// ProvisionWatch hands deliver the value now and each change after, until stopped. A value not answered now is
|
||||
// refused now, with the client's reason.
|
||||
func (b *moduleBus) ProvisionWatch(params json.RawMessage, deliver func(json.RawMessage) error) (func(), error) {
|
||||
asked, err := readProvisionAsked("watch", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := b.all.conn.AskProvisioner(b.module, asked.Kind, asked.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w := &provisionWatch{bus: b, asked: asked, deliver: deliver, last: value}
|
||||
if err := w.hand(ProvisionChange{Kind: asked.Kind, Key: asked.Key, Value: value}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stop, err := b.all.provisioned.add(b.all, b.module, w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stop, nil
|
||||
}
|
||||
|
||||
// provisionWatch is one watch a bundle holds: what it asked, and what it was handed last.
|
||||
type provisionWatch struct {
|
||||
bus *moduleBus
|
||||
asked provisionAsked
|
||||
deliver func(json.RawMessage) error
|
||||
mu sync.Mutex // one change at a time, in order
|
||||
last json.RawMessage
|
||||
refused string
|
||||
stopped bool
|
||||
}
|
||||
|
||||
func (w *provisionWatch) hand(change ProvisionChange) error {
|
||||
raw, err := json.Marshal(change)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return w.deliver(raw)
|
||||
}
|
||||
|
||||
// again asks once more after the client said something moved, and hands the bundle what changed.
|
||||
func (w *provisionWatch) again() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped {
|
||||
return
|
||||
}
|
||||
value, err := w.bus.all.conn.AskProvisioner(w.bus.module, w.asked.Kind, w.asked.Key)
|
||||
change := ProvisionChange{Kind: w.asked.Kind, Key: w.asked.Key}
|
||||
var refused *bus.ProvisionRefused
|
||||
switch {
|
||||
case errors.As(err, &refused):
|
||||
if w.refused == refused.Reason {
|
||||
return
|
||||
}
|
||||
change.Refused = refused.Reason
|
||||
case err != nil:
|
||||
// The client not reached: asked again on its next word, which it says whenever it comes up.
|
||||
w.bus.all.logf("[mesh-tools] %s's watch of its %s %s could not ask again: %v", w.bus.module, w.asked.Kind,
|
||||
w.asked.Key, err)
|
||||
return
|
||||
default:
|
||||
if w.refused == "" && bytes.Equal(value, w.last) {
|
||||
return
|
||||
}
|
||||
change.Value = value
|
||||
}
|
||||
if err := w.hand(change); err != nil {
|
||||
w.bus.all.logf("[mesh-tools] %s did not take a change of its %s %s: %v", w.bus.module, w.asked.Kind,
|
||||
w.asked.Key, err)
|
||||
return
|
||||
}
|
||||
w.last, w.refused = value, change.Refused
|
||||
}
|
||||
|
||||
// provisionWatches is every watch on this runtime, by module, and the one subscription that hears the client.
|
||||
type provisionWatches struct {
|
||||
mu sync.Mutex
|
||||
of map[string]map[*provisionWatch]bool
|
||||
stop func()
|
||||
}
|
||||
|
||||
func (p *provisionWatches) add(all *consumers, module string, w *provisionWatch) (func(), error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.stop == nil {
|
||||
stop, err := all.conn.OnProvisionChanged(func(module string, changed bus.ProvisionChanged) {
|
||||
for _, w := range p.told(module, changed) {
|
||||
go w.again()
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the runtime cannot hear its machine's provisioner client: %w", err)
|
||||
}
|
||||
p.stop = stop
|
||||
}
|
||||
if p.of == nil {
|
||||
p.of = map[string]map[*provisionWatch]bool{}
|
||||
}
|
||||
if p.of[module] == nil {
|
||||
p.of[module] = map[*provisionWatch]bool{}
|
||||
}
|
||||
p.of[module][w] = true
|
||||
return func() {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
delete(p.of[module], w)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// told is the watches of module that what moved concerns.
|
||||
func (p *provisionWatches) told(module string, changed bus.ProvisionChanged) []*provisionWatch {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var out []*provisionWatch
|
||||
for w := range p.of[module] {
|
||||
if w.asked.Kind == changed.Kind && (changed.Key == "" || changed.Key == w.asked.Key) {
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stopModule stops every watch of one module, whose bundle is loaded again.
|
||||
func (p *provisionWatches) stopModule(module string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for w := range p.of[module] {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
}
|
||||
delete(p.of, module)
|
||||
}
|
||||
|
||||
func (p *provisionWatches) stopAll() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for _, ws := range p.of {
|
||||
for w := range ws {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
}
|
||||
}
|
||||
p.of = nil
|
||||
if p.stop != nil {
|
||||
p.stop()
|
||||
p.stop = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A module's code reads its values at runtime through moduleBus.Provision and ProvisionWatch (novox/hq ADR 0349
|
||||
// §14, ADR 0370, issue 685), against a real bus with a stand-in for the machine's provisioner client: the runtime
|
||||
// asks as the module whose bundle asks, whatever the bundle names; a watch hands the current value, then each change
|
||||
// the client says moved, and a refusal once; a secret is refused without asking.
|
||||
|
||||
// standInClient answers prov.ask.<node> from its values, keyed `<module>.<kind>.<key>`, and keeps every ask.
|
||||
type standInClient struct {
|
||||
mu sync.Mutex
|
||||
values map[string]string
|
||||
asks []string
|
||||
nc *nats.Conn
|
||||
}
|
||||
|
||||
func aStandInClient(t *testing.T, node string, values map[string]string) *standInClient {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
c := &standInClient{values: values, nc: nc}
|
||||
if _, err := nc.Subscribe("prov.ask."+node, func(m *nats.Msg) {
|
||||
var a struct{ Module, Kind, Key string }
|
||||
_ = json.Unmarshal(m.Data, &a)
|
||||
c.mu.Lock()
|
||||
c.asks = append(c.asks, string(m.Data))
|
||||
v, ok := c.values[a.Module+"."+a.Kind+"."+a.Key]
|
||||
c.mu.Unlock()
|
||||
if !ok {
|
||||
_ = m.Respond([]byte(`{"error":"` + a.Module + ` declares no config value ` + a.Key + `"}`))
|
||||
return
|
||||
}
|
||||
_ = m.Respond([]byte(`{"value":` + v + `}`))
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = nc.Flush()
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *standInClient) set(key, value string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if value == "" {
|
||||
delete(c.values, key)
|
||||
return
|
||||
}
|
||||
c.values[key] = value
|
||||
}
|
||||
|
||||
func (c *standInClient) asked() []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return append([]string(nil), c.asks...)
|
||||
}
|
||||
|
||||
func (c *standInClient) say(t *testing.T, node, module, body string) {
|
||||
t.Helper()
|
||||
if err := c.nc.Publish("prov.changed."+node+"."+module, []byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundleAsksForItsValuesAsItsOwnModule(t *testing.T) {
|
||||
client := aStandInClient(t, "anchor", map[string]string{
|
||||
"claude-code.config.managed_settings": `{"spinnerTipsEnabled":false}`,
|
||||
"dunst.config.managed_settings": `"dunst's"`,
|
||||
})
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
b := all.forModule("claude-code").(*moduleBus)
|
||||
|
||||
// Its own value, though the bundle names another module: the runtime names the module from the bundle.
|
||||
got, err := b.Provision("get", json.RawMessage(`{"kind":"config","key":"managed_settings","module":"dunst"}`))
|
||||
if err != nil || string(got) != `{"spinnerTipsEnabled":false}` {
|
||||
t.Fatalf("answered %s, %v", got, err)
|
||||
}
|
||||
if asks := client.asked(); len(asks) != 1 || asks[0] != `{"key":"managed_settings","kind":"config","module":"claude-code"}` {
|
||||
t.Fatalf("the client was asked %v", asks)
|
||||
}
|
||||
// The client's refusal, in its words.
|
||||
if _, err := b.Provision("get", json.RawMessage(`{"kind":"config","key":"nothing"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "declares no config value nothing") {
|
||||
t.Errorf("an undeclared key: %v", err)
|
||||
}
|
||||
// A secret, refused here and never asked; and what is not an ask.
|
||||
before := len(client.asked())
|
||||
for params, want := range map[string]string{
|
||||
`{"kind":"secret","key":"gitea-read"}`: "the vault hands a secret only to a caller the bus proves",
|
||||
`{"kind":"setting","key":"role"}`: "not a kind of value",
|
||||
`{"key":"role"}`: "names no value",
|
||||
} {
|
||||
if _, err := b.Provision("get", json.RawMessage(params)); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%s: %v, want %q", params, err, want)
|
||||
}
|
||||
}
|
||||
if _, err := b.ProvisionWatch(json.RawMessage(`{"kind":"secret","key":"gitea-read"}`), func(json.RawMessage) error {
|
||||
return nil
|
||||
}); err == nil {
|
||||
t.Error("a secret was watched")
|
||||
}
|
||||
if n := len(client.asked()); n != before {
|
||||
t.Errorf("the client was asked for what the runtime refuses: %v", client.asked()[before:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundlesWatchIsHandedTheValueAndEachChange(t *testing.T) {
|
||||
client := aStandInClient(t, "anchor", map[string]string{"claude-code.config.managed_settings": `{"a":1}`})
|
||||
conn := connect(t, "node-tools", "anchor")
|
||||
all := &consumers{conn: conn, logf: t.Logf, of: map[string]*moduleEvents{}}
|
||||
t.Cleanup(all.stopAll)
|
||||
b := all.forModule("claude-code").(*moduleBus)
|
||||
|
||||
handed := make(chan ProvisionChange, 16)
|
||||
stop, err := b.ProvisionWatch(json.RawMessage(`{"kind":"config","key":"managed_settings","module":"dunst"}`),
|
||||
func(raw json.RawMessage) error {
|
||||
var c ProvisionChange
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
t.Errorf("handed %s", raw)
|
||||
}
|
||||
handed <- c
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := func(what string) ProvisionChange {
|
||||
t.Helper()
|
||||
select {
|
||||
case c := <-handed:
|
||||
return c
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("nothing handed: %s", what)
|
||||
}
|
||||
return ProvisionChange{}
|
||||
}
|
||||
nothing := func(what string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case c := <-handed:
|
||||
t.Errorf("%s: handed %+v", what, c)
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
if c := next("the current value"); string(c.Value) != `{"a":1}` || c.Kind != "config" || c.Key != "managed_settings" {
|
||||
t.Fatalf("first handed %+v", c)
|
||||
}
|
||||
|
||||
// A change, said by the client: asked again, handed.
|
||||
client.set("claude-code.config.managed_settings", `{"a":2}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config","key":"managed_settings"}`)
|
||||
if c := next("the change"); string(c.Value) != `{"a":2}` {
|
||||
t.Fatalf("after the change, handed %+v", c)
|
||||
}
|
||||
// Every key of the kind said to have moved, the value the same: nothing handed. Another module's or another
|
||||
// key's word: not even asked.
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
nothing("the same value again")
|
||||
before := len(client.asked())
|
||||
client.say(t, "anchor", "dunst", `{"kind":"config","key":"managed_settings"}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config","key":"role"}`)
|
||||
nothing("another's word")
|
||||
if n := len(client.asked()); n != before {
|
||||
t.Errorf("asked again for another's word: %v", client.asked()[before:])
|
||||
}
|
||||
// No longer answered: handed the refusal, once.
|
||||
client.set("claude-code.config.managed_settings", "")
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
if c := next("the refusal"); c.Refused == "" || len(c.Value) != 0 {
|
||||
t.Fatalf("handed %+v, want the refusal", c)
|
||||
}
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
nothing("the same refusal again")
|
||||
// Answered again, and then stopped: nothing after.
|
||||
client.set("claude-code.config.managed_settings", `{"a":3}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
if c := next("answered again"); string(c.Value) != `{"a":3}` {
|
||||
t.Fatalf("handed %+v", c)
|
||||
}
|
||||
stop()
|
||||
client.set("claude-code.config.managed_settings", `{"a":4}`)
|
||||
client.say(t, "anchor", "claude-code", `{"kind":"config"}`)
|
||||
nothing("a stopped watch")
|
||||
for _, a := range client.asked() {
|
||||
if !strings.Contains(a, `"module":"claude-code"`) {
|
||||
t.Errorf("asked as another module: %s", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-tools/node-tools/internal/bus"
|
||||
mt "github.com/novox/mesh-tools/node-tools/internal/meshtest"
|
||||
)
|
||||
|
||||
// A runtime serves a module's bundle as soon as it is in place, again when it is replaced, and no
|
||||
// more once it is gone, with no restart (issue 1945, note 23: mesh-files' bundle arrived on a machine
|
||||
// a quarter of an hour after its runtime started, and its tools were missing until a hand restart).
|
||||
|
||||
// syncLogs is a logger that keeps the lines, safe to read while the runtime's watcher writes.
|
||||
type syncLogs struct {
|
||||
mu sync.Mutex
|
||||
lines []string
|
||||
}
|
||||
|
||||
func (l *syncLogs) Logf(format string, args ...any) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.lines = append(l.lines, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *syncLogs) Has(fragments ...string) bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
for _, line := range l.lines {
|
||||
all := true
|
||||
for _, f := range fragments {
|
||||
all = all && strings.Contains(line, f)
|
||||
}
|
||||
if all {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (l *syncLogs) All() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return strings.Join(l.lines, "\n")
|
||||
}
|
||||
|
||||
// bundleServing is a tools bundle, MCP over stdio with no SDK, whose one tool is named tool.
|
||||
func bundleServing(tool string) string { return bundleScript(tool, false, false) }
|
||||
|
||||
// bundleScript is a tools bundle whose one tool is named tool. With subscribe it asks the runtime for its
|
||||
// module's events and writes "<tool> subscribed" and "<tool> handled <n>" to $EVENT_LOG; with writeBeside it
|
||||
// writes a file beside itself as it starts, as a bundle keeping a cache or a socket there does.
|
||||
func bundleScript(tool string, subscribe, writeBeside bool) string {
|
||||
py := func(b bool) string {
|
||||
if b {
|
||||
return "True"
|
||||
}
|
||||
return "False"
|
||||
}
|
||||
return `#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
TOOL = "` + tool + `"
|
||||
SUBSCRIBE = ` + py(subscribe) + `
|
||||
if ` + py(writeBeside) + `:
|
||||
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "cache"), "w") as f:
|
||||
f.write(TOOL + " " + str(os.getpid()))
|
||||
def say(m):
|
||||
m["jsonrpc"] = "2.0"; sys.stdout.write(json.dumps(m) + "\n"); sys.stdout.flush()
|
||||
def log(line):
|
||||
with open(os.environ["EVENT_LOG"], "a") as f:
|
||||
f.write(line + "\n")
|
||||
TOOLS = [{"name": TOOL, "description": "answers its name", "inputSchema": {"type": "object", "properties": {}}}]
|
||||
for line in sys.stdin:
|
||||
req = json.loads(line); rid = req.get("id"); m = req.get("method"); p = req.get("params") or {}
|
||||
if m is None:
|
||||
if rid == "subscribe":
|
||||
log(TOOL + " subscribed")
|
||||
continue
|
||||
if m == "initialize":
|
||||
say({"id": rid, "result": {"protocolVersion": "2025-03-26", "capabilities": {"tools": {}}, "serverInfo": {"name": "ledger", "version": "1"}}})
|
||||
elif m == "tools/list":
|
||||
say({"id": rid, "result": {"tools": TOOLS}})
|
||||
if SUBSCRIBE:
|
||||
say({"id": "subscribe", "method": "mesh/subscribe", "params": {}})
|
||||
elif m == "tools/call":
|
||||
say({"id": rid, "result": {"content": [{"type": "text", "text": json.dumps({"tool": TOOL})}]}})
|
||||
elif m == "mesh/event":
|
||||
body = (p.get("envelope") or {}).get("body") or {}
|
||||
log(TOOL + " handled " + str(body.get("n")))
|
||||
say({"id": rid, "result": {}})
|
||||
elif rid is not None:
|
||||
say({"id": rid, "result": {}})
|
||||
`
|
||||
}
|
||||
|
||||
// place puts a bundle serving tool at path the way a delivery does: written beside it, made
|
||||
// executable, and renamed into place, its directories made as needed.
|
||||
func place(t *testing.T, path, tool string) {
|
||||
t.Helper()
|
||||
placeScript(t, path, bundleServing(tool))
|
||||
}
|
||||
|
||||
func placeScript(t *testing.T, path, script string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmp := path + ".new"
|
||||
if err := os.WriteFile(tmp, []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// fastSettle makes the watcher act on a burst of changes quickly in a test.
|
||||
func fastSettle(t *testing.T) {
|
||||
was := Settle
|
||||
Settle = 50 * time.Millisecond
|
||||
t.Cleanup(func() { Settle = was })
|
||||
}
|
||||
|
||||
type reloadBed struct {
|
||||
mesh *mt.Mesh
|
||||
nodeTools *bus.Conn
|
||||
asker *bus.Conn
|
||||
logs *syncLogs
|
||||
entry string
|
||||
}
|
||||
|
||||
func newReloadBed(t *testing.T) *reloadBed {
|
||||
t.Helper()
|
||||
fastSettle(t)
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
return &reloadBed{
|
||||
mesh: mesh,
|
||||
nodeTools: connect(t, "node-tools", "anchor"),
|
||||
asker: connect(t, "console", "workstation"),
|
||||
logs: &syncLogs{},
|
||||
// Below directories that do not exist yet, as on a machine before the module's first delivery.
|
||||
entry: filepath.Join(t.TempDir(), "bundles", "ledger", "tools", "serve.py"),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *reloadBed) run(t *testing.T) {
|
||||
t.Helper()
|
||||
stop, err := Run(b.nodeTools, []Served{{"ledger", []string{b.entry}}}, nil, b.logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
}
|
||||
|
||||
func (b *reloadBed) answers(t *testing.T, tool string) {
|
||||
t.Helper()
|
||||
mt.Until(t, func() error {
|
||||
got, err := call(t, b.asker, "ledger."+tool+"@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ledger.%s is not served: %v\n%s", tool, err, b.logs.All())
|
||||
}
|
||||
if !strings.Contains(got, `"tool":"`+tool+`"`) {
|
||||
return fmt.Errorf("ledger.%s answered %s", tool, got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (b *reloadBed) silent(t *testing.T, tool string) {
|
||||
t.Helper()
|
||||
mt.Until(t, func() error {
|
||||
if _, err := call(t, b.asker, "ledger."+tool+"@anchor", map[string]any{}); err == nil {
|
||||
return fmt.Errorf("ledger.%s is still served\n%s", tool, b.logs.All())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func announced(t *testing.T) string {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(mt.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
msg, err := nc.Request("$SRV.INFO.node-tools.anchor", nil, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var i micro.Info
|
||||
if err := json.Unmarshal(msg.Data, &i); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, e := range i.Endpoints {
|
||||
out = append(out, e.Subject)
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
// 1945/AC19: a bundle placed after the runtime started is served, and announced, with no restart.
|
||||
func TestABundlePlacedAfterStartIsServed(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
b.run(t)
|
||||
if !b.logs.Has("ledger's bundle", "does not exist") {
|
||||
t.Errorf("the missing bundle was not said as missing:\n%s", b.logs.All())
|
||||
}
|
||||
place(t, b.entry, "first")
|
||||
b.answers(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
if a := announced(t); !strings.Contains(a, "mesh.mod.ledger.tool.first.anchor") {
|
||||
return fmt.Errorf("discovery does not announce the placed bundle's tool:\n%s", a)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// 1945/AC20: a bundle replaced while served is loaded again: its new tool is served, its old one no more.
|
||||
func TestAReplacedBundleIsReloaded(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
place(t, b.entry, "second")
|
||||
b.answers(t, "second")
|
||||
b.silent(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
a := announced(t)
|
||||
if !strings.Contains(a, "mesh.mod.ledger.tool.second.anchor") || strings.Contains(a, "mesh.mod.ledger.tool.first.anchor") {
|
||||
return fmt.Errorf("discovery does not follow the replaced bundle:\n%s", a)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// The bundle's whole directory swapped for a new one, as an archive unpacked beside it and renamed in.
|
||||
dir := filepath.Dir(b.entry)
|
||||
fresh := dir + ".unpacked"
|
||||
place(t, filepath.Join(fresh, filepath.Base(b.entry)), "third")
|
||||
if err := os.Rename(dir, dir+".old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(fresh, dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.answers(t, "third")
|
||||
b.silent(t, "second")
|
||||
|
||||
// And changed again inside the directory swapped in: the watch follows the new directory.
|
||||
place(t, b.entry, "fourth")
|
||||
b.answers(t, "fourth")
|
||||
}
|
||||
|
||||
// A replaced bundle's old tool stays gone when the module's membership is issued again: a stopped tool
|
||||
// leaves no membership handler behind to bind it once more, and a call on it starts no unmanaged second
|
||||
// copy of the bundle (issue 1945, review round 2).
|
||||
func TestAReplacedBundlesOldToolStaysGoneOnTheNextMembership(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
place(t, b.entry, "second")
|
||||
b.answers(t, "second")
|
||||
b.silent(t, "first")
|
||||
|
||||
b.mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
b.answers(t, "second")
|
||||
// Long enough for the reissued membership to have been heard, before the old tool is asked.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if got, err := call(t, b.asker, "ledger.first@anchor", map[string]any{}); err == nil {
|
||||
t.Fatalf("ledger.first is served again after the membership was reissued, and answered %s\n%s", got, b.logs.All())
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: a bundle removed while served is no longer served, and its module says why.
|
||||
func TestARemovedBundleIsNoLongerServed(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
place(t, b.entry, "first")
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
if err := os.RemoveAll(filepath.Dir(b.entry)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.silent(t, "first")
|
||||
mt.Until(t, func() error {
|
||||
got, err := call(t, b.asker, "ledger.tools@anchor", map[string]any{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var answer ToolsAnswer
|
||||
if err := json.Unmarshal([]byte(got), &answer); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(answer.Tools) != 0 || !strings.Contains(answer.Failed, "does not exist") {
|
||||
return fmt.Errorf("ledger's tools after its bundle was removed: %s", got)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if a := announced(t); strings.Contains(a, "mesh.mod.ledger.tool.first") {
|
||||
t.Errorf("discovery still announces the removed bundle's tool:\n%s", a)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC22: a bundle that cannot be started is said with its real reason: a missing file does not
|
||||
// exist, and a file without an execute bit is not executable with its mode.
|
||||
func TestABundleThatCannotStartSaysTheRealReason(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
missing := filepath.Join(dir, "nowhere", "serve.py")
|
||||
if why := notStartable(missing); !strings.Contains(why, missing+" does not exist") || strings.Contains(why, "not executable") {
|
||||
t.Errorf("a missing bundle said: %q", why)
|
||||
}
|
||||
plain := filepath.Join(dir, "serve.py")
|
||||
if err := os.WriteFile(plain, []byte(bundleServing("first")), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := notStartable(plain); !strings.Contains(why, plain+" is not executable (mode -rw-r--r--)") {
|
||||
t.Errorf("a bundle without an execute bit said: %q", why)
|
||||
}
|
||||
if why := notStartable(dir); !strings.Contains(why, "is a directory") {
|
||||
t.Errorf("a directory said: %q", why)
|
||||
}
|
||||
if err := os.Chmod(plain, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := notStartable(plain); why != "" {
|
||||
t.Errorf("a startable bundle said: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC23: an event a module's consumer hears while the module is loaded again is offered again, never
|
||||
// a panic of the runtime, and the bundle loaded in its place is handed the next offer.
|
||||
func TestAnEventHeardDuringAReloadIsOfferedAgainNotAPanic(t *testing.T) {
|
||||
c := &consumers{logf: func(string, ...any) {}, of: map[string]*moduleEvents{}}
|
||||
var got []string
|
||||
before := func(json.RawMessage) error { got = append(got, "before"); return nil }
|
||||
old := &moduleEvents{delivers: []*deliverSlot{{f: before}}}
|
||||
c.of["ledger"] = old
|
||||
c.dropModule("ledger")
|
||||
env := bus.Envelope{Key: "alpha.happened", Body: json.RawMessage(`{"n":1}`)}
|
||||
hand := func(m *moduleEvents) (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("an event heard during a reload panicked the runtime: %v", r)
|
||||
}
|
||||
}()
|
||||
return c.handOut("ledger", m, env)
|
||||
}
|
||||
if err := hand(old); err == nil {
|
||||
t.Error("an event heard while the module is loaded again was taken, not offered again")
|
||||
}
|
||||
after := func(json.RawMessage) error { got = append(got, "after"); return nil }
|
||||
fresh := &moduleEvents{delivers: []*deliverSlot{{f: after}}}
|
||||
c.of["ledger"] = fresh
|
||||
if err := hand(fresh); err != nil {
|
||||
t.Errorf("the bundle loaded again was not handed the event: %v", err)
|
||||
}
|
||||
if err := hand(old); err == nil {
|
||||
t.Error("the stopped consumer's handler handed an event on after the reload")
|
||||
}
|
||||
if strings.Join(got, ",") != "after" {
|
||||
t.Errorf("handed to %v, want the bundle loaded again alone", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC23, whole: a module's events reach its bundle before a reload, around it, and after it.
|
||||
func TestAModulesEventsReachItsBundleLoadedAgain(t *testing.T) {
|
||||
fastSettle(t)
|
||||
restore := bus.NakDelay
|
||||
bus.NakDelay = 200 * time.Millisecond
|
||||
t.Cleanup(func() { bus.NakDelay = restore })
|
||||
mesh := mt.New(t)
|
||||
mesh.Issue(t, mt.MembershipOf("ledger", "anchor", false, nil))
|
||||
mesh.Consumer(t, "anchor", "ledger", []string{"mesh.mod.alpha.event.>"}, 2*time.Second)
|
||||
nodeTools := connect(t, "node-tools", "anchor")
|
||||
logs := &syncLogs{}
|
||||
dir := t.TempDir()
|
||||
log := filepath.Join(dir, "events.log")
|
||||
entry := filepath.Join(dir, "bundles", "ledger", "tools", "serve.py")
|
||||
has := func(line string) error {
|
||||
b, _ := os.ReadFile(log)
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if l == line {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("no %q yet in:\n%s\nruntime:\n%s", line, b, logs.All())
|
||||
}
|
||||
placeScript(t, entry, bundleScript("first", true, false))
|
||||
stop, err := Run(nodeTools, []Served{{"ledger", []string{entry}}}, map[string]map[string]string{"ledger": {"EVENT_LOG": log}}, logs.Logf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
mt.Until(t, func() error { return has("first subscribed") })
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 1})
|
||||
mt.Until(t, func() error { return has("first handled 1") })
|
||||
|
||||
placeScript(t, entry, bundleScript("second", true, false))
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 2})
|
||||
mt.Until(t, func() error {
|
||||
if has("first handled 2") != nil && has("second handled 2") != nil {
|
||||
return has("second handled 2")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
mt.Until(t, func() error { return has("second subscribed") })
|
||||
mesh.Emit(t, "mesh.mod.alpha.event.happened", map[string]any{"n": 3})
|
||||
mt.Until(t, func() error { return has("second handled 3") })
|
||||
}
|
||||
|
||||
// 1945/AC24: a bundle that writes a file beside itself as it starts is loaded once at start and once
|
||||
// when it is replaced, never again by its own writing.
|
||||
func TestABundleWritingBesideItselfIsNotLoadedAgainByIt(t *testing.T) {
|
||||
b := newReloadBed(t)
|
||||
t.Setenv("EVENT_LOG", filepath.Join(t.TempDir(), "unused.log"))
|
||||
placeScript(t, b.entry, bundleScript("first", false, true))
|
||||
b.run(t)
|
||||
b.answers(t, "first")
|
||||
reloads := func() int {
|
||||
return strings.Count(b.logs.All(), "loading it again")
|
||||
}
|
||||
time.Sleep(20 * Settle)
|
||||
if n := reloads(); n != 0 {
|
||||
t.Fatalf("the bundle was loaded again %d time(s) by its own writing:\n%s", n, b.logs.All())
|
||||
}
|
||||
placeScript(t, b.entry, bundleScript("second", false, true))
|
||||
b.answers(t, "second")
|
||||
time.Sleep(20 * Settle)
|
||||
if n := reloads(); n != 1 {
|
||||
t.Errorf("replaced once, loaded again %d time(s):\n%s", n, b.logs.All())
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user