From 0a414d576a293c437689f5dabf74ea57d20f625a Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 4 Sep 2026 23:08:54 +0200 Subject: [PATCH] e2e: assigned sonarr + grafana prove the two runtime config paths (ADR 0051/0052) assigned-sonarr proves the Servarr detection path: the runtime discovers its API key from the app's config.xml and serves its tools. assigned-grafana proves the settings path: the operator states URL and token as settings, the mesh merges them into the module's config file, and the runtime serves from that with nothing in the manifest. Both green. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF --- scenarios/grafana-node.yml | 28 +++ scenarios/sonarr-node.yml | 28 +++ test/integration/assigned-grafana.test.ts | 216 +++++++++++++++++++++ test/integration/assigned-sonarr.test.ts | 220 ++++++++++++++++++++++ 4 files changed, 492 insertions(+) create mode 100644 scenarios/grafana-node.yml create mode 100644 scenarios/sonarr-node.yml create mode 100644 test/integration/assigned-grafana.test.ts create mode 100644 test/integration/assigned-sonarr.test.ts diff --git a/scenarios/grafana-node.yml b/scenarios/grafana-node.yml new file mode 100644 index 0000000..dac3864 --- /dev/null +++ b/scenarios/grafana-node.yml @@ -0,0 +1,28 @@ +# One machine that becomes a mesh and assigns itself grafana's tool runtime, configured by settings. +# +# plex/sonarr prove a runtime that self-detects its credential; this proves the other half of the +# config story (novox/hq ADR 0051 + 0052): the operator states grafana's URL and token as the +# assignment's settings, the mesh merges them into the config file the runtime reads, and the +# runtime registers and serves grafana's tools from that — no credential baked into the manifest. +scenario: grafana-node + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + memory: 3GiB + cpus: 2 + +images: + - postgres:17-alpine + - cloudamqp/lavinmq:latest + - mesh-control:development + - mesh-runtime-grafana:development + +place: + all: [host, runtime] diff --git a/scenarios/sonarr-node.yml b/scenarios/sonarr-node.yml new file mode 100644 index 0000000..85dcaf9 --- /dev/null +++ b/scenarios/sonarr-node.yml @@ -0,0 +1,28 @@ +# One machine that becomes a mesh and assigns itself sonarr's tool runtime. +# +# plex-node proved a tools+events module that self-detects its token from a mounted config dir; this +# proves the same self-configuring pattern generalises to the Servarr family (novox/hq ADR 0052): +# sonarr's runtime detects its API key from the server's config.xml and serves sonarr's tools over a +# mesh-issued scoped account, with no live Sonarr to reach. +scenario: sonarr-node + +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + inbound: allow + memory: 3GiB + cpus: 2 + +images: + - postgres:17-alpine + - cloudamqp/lavinmq:latest + - mesh-control:development + - mesh-runtime-sonarr:development + +place: + all: [host, runtime] diff --git a/test/integration/assigned-grafana.test.ts b/test/integration/assigned-grafana.test.ts new file mode 100644 index 0000000..0b0b37f --- /dev/null +++ b/test/integration/assigned-grafana.test.ts @@ -0,0 +1,216 @@ +/** + * The mesh assigns grafana's tool runtime, configured entirely by the assignment's settings — the + * ADR 0051 + 0052 case: config is the assignment's, delivered as a settings-merged file the runtime + * reads, not a credential baked into the manifest. + * + * plex/sonarr prove a runtime that self-detects its key from the app's own config. This proves the + * other half: the operator states grafana's URL and an API token as settings for this node, the + * control plane merges them into the module's mergeable config file, and the runtime reads that file + * at start, registers grafana's tools, and serves them under its scoped account. There is no live + * Grafana — that the serve queue is bound is the proof the settings reached the runtime and its + * tools loaded from them. + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock + * scripts/build-module-runtime.sh grafana builds mesh-runtime-grafana:development into the local + * daemon, which scenarios/grafana-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it. + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll } from "./harness.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" + : false; + +const SCENARIO = "grafana-node"; +const MACHINE = "anchor"; + +let instanceId = ""; +let stocked: string[] = []; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, MACHINE, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(command, timeoutMs); + if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`); + return out; +} + +async function mesh(command: string, timeoutMs?: number): Promise { + return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); +} + +function pinned(repository: string): string { + const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); + assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); + return found; +} + +function bundleFor(images: string[]): string { + let text = readFileSync(bundle, "utf8"); + for (const ref of images) { + const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); + const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); + text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); + } + return text; +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +async function settled(withinMs = 480_000): Promise { + const until = Date.now() + withinMs; + let last = ""; + while (Date.now() < until) { + const asked = await on(`docker exec mesh-control /mesh-control status --json`); + if (asked.ok) { + try { + const state = JSON.parse(asked.out) as { + wrong: { node: string; outcome: string }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + const bad = state.wrong.find((w) => w.node === MACHINE); + if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`); + const word = state.reported.find((r) => r.node === MACHINE); + if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return; + last = asked.out; + } catch (err) { + if (err instanceof Error && err.message.includes("did not apply")) throw err; + last = asked.out; + } + } + await new Promise((r) => setTimeout(r, 5000)); + } + throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`); +} + +before(async () => { + if (skip) return; + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + stocked = raised.images; + + await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); + await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); + const up = await must(`docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { + assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); + } + + await mesh(`node add ${MACHINE}`); + const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`)); + await must(`${HOST_PATH} enrol --token ${quote(token)}`); + await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); +}, { timeout: 1_800_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 600_000 }); + +test("the mesh assigns grafana's runtime, configured by settings, and it serves its tools", { + skip, timeout: 900_000, +}, async () => { + // A grafana manifest with no credential in it: its runtime, and a mergeable config file the + // settings will fill. This is the whole point of ADR 0051 — the manifest carries defaults and + // structure, the assignment carries the URL and token. + const manifest = JSON.stringify({ + module: "grafana", + version: "1", + emits: ["module.grafana.alert.firing"], + "own-secrets": { broker: "/var/lib/mesh/grafana/broker" }, + resources: [ + { id: "mesh-state", type: "directory", path: "/var/lib/mesh/grafana", mode: "0700" }, + { id: "config", type: "file", path: "/var/lib/mesh/grafana/config.json", mode: "0600", content: "{}\n", merge: "json" }, + { + id: "runtime", type: "container", name: "mesh-grafana", image: pinned("mesh-runtime-grafana"), + network: "host", + volumes: [ + "/var/lib/mesh/grafana/broker:/run/secrets/broker:ro", + "/var/lib/mesh/grafana/config.json:/run/config/config.json:ro", + ], + env: { + MESH_BROKER_FILE: "/run/secrets/broker", + MESH_GRAFANA_CONFIG_FILE: "/run/config/config.json", + }, + }, + ], + }); + await must(`printf %s ${quote(manifest)} > /tmp/grafana.json && docker cp /tmp/grafana.json mesh-control:/grafana.json`); + await mesh("module add /grafana.json"); + + // The operator states grafana's URL and API token as settings for this node — the config the + // runtime will read. Nothing about them is in the manifest. + const settings = JSON.stringify({ url: "http://127.0.0.1:3000", token: "lab-grafana-token" }); + await must(`printf %s ${quote(settings)} > /tmp/grafana-settings.json && docker cp /tmp/grafana-settings.json mesh-control:/grafana-settings.json`); + await mesh(`settings set grafana /grafana-settings.json --node ${MACHINE}`); + + const issued = await mesh(`module issue grafana --node ${MACHINE}`); + assert.match(issued, /scoped to what it emits and consumes/, issued); + await mesh(`assign ${MACHINE} grafana`); + await mesh(`push ${MACHINE}`); + await settled(); + + const running = await must(`docker ps --format '{{.Names}}'`); + assert.match(running, /mesh-grafana/, + `grafana's runtime was assigned and is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`); + + // The settings reached the node: the rendered config file carries what was set, not the manifest's + // empty default. + const config = await must(`cat /var/lib/mesh/grafana/config.json`); + assert.match(config, /lab-grafana-token/, `the settings did not merge into the config file:\n${config}`); + + const credential = await must(`cat /var/lib/mesh/grafana/broker`); + assert.match(credential, /"url":"amqps:\/\/anchor-grafana:/, `not the scoped account:\n${credential}`); + assert.doesNotMatch(credential, /guest:guest/, "grafana's runtime holds the broker's own account"); + + // The runtime read that config, built its client from the settings-provided token, registered its + // tools, and bound their serve queues — the queue on the broker is the proof the settings-config + // path reached serving, with no credential in the manifest and no live Grafana. + let served = ""; + const untilServing = Date.now() + 60_000; + while (Date.now() < untilServing) { + served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`); + if (/serve\.grafana\.grafana_status/.test(served)) break; + await new Promise((r) => setTimeout(r, 3000)); + } + assert.match(served, /serve\.grafana\.grafana_status/, + `grafana's runtime never bound its serve queue (settings not read?):\n` + + `${(await on(`docker logs mesh-grafana 2>&1 | tail -20`)).out}\n---\n${served}`); + + const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`); + assert.match(users, /anchor-grafana/, `the scoped account is not on the broker:\n${users}`); +}); diff --git a/test/integration/assigned-sonarr.test.ts b/test/integration/assigned-sonarr.test.ts new file mode 100644 index 0000000..1232d37 --- /dev/null +++ b/test/integration/assigned-sonarr.test.ts @@ -0,0 +1,220 @@ +/** + * The mesh assigns sonarr's tool runtime, and it serves sonarr's tools over an account the mesh + * delivered — the Servarr case of novox/hq ADR 0052. + * + * assigned-plex proved a tools+events module that self-detects its token from a mounted config dir. + * This proves that self-configuring pattern generalises to the Servarr family: sonarr's runtime + * detects its API key from the server's own config.xml (a file resource stands in for the running + * Sonarr here), registers its tools, and serves them under a scoped account. There is no live Sonarr + * to reach — that the serve queue is bound is the proof the key was detected and the tools loaded. + * + * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock + * scripts/build-module-runtime.sh sonarr builds mesh-runtime-sonarr:development into the local + * daemon, which scenarios/sonarr-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it. + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { raise } from "../../src/lifecycle/raise.ts"; +import { destroy, exec } from "../../src/lifecycle/operate.ts"; +import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll } from "./harness.ts"; + +const capability = await labIsUsable(); +const binary = hostBinaryPath(); +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !binary || !existsSync(binary) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" + : false; + +const SCENARIO = "sonarr-node"; +const MACHINE = "anchor"; + +let instanceId = ""; +let stocked: string[] = []; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, MACHINE, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + if (marker < 0) return { out: stdout, ok: false }; + return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; +} + +async function must(command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(command, timeoutMs); + if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`); + return out; +} + +async function mesh(command: string, timeoutMs?: number): Promise { + return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); +} + +function pinned(repository: string): string { + const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); + assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); + return found; +} + +function bundleFor(images: string[]): string { + let text = readFileSync(bundle, "utf8"); + for (const ref of images) { + const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); + const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); + text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); + } + return text; +} + +function tokenFrom(said: string): string { + const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(found, `no token in:\n${said}`); + return found; +} + +async function settled(withinMs = 480_000): Promise { + const until = Date.now() + withinMs; + let last = ""; + while (Date.now() < until) { + const asked = await on(`docker exec mesh-control /mesh-control status --json`); + if (asked.ok) { + try { + const state = JSON.parse(asked.out) as { + wrong: { node: string; outcome: string }[]; + waiting: { node: string }[]; + reported: { node: string; outcome: string; current: boolean }[]; + }; + const bad = state.wrong.find((w) => w.node === MACHINE); + if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`); + const word = state.reported.find((r) => r.node === MACHINE); + if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return; + last = asked.out; + } catch (err) { + if (err instanceof Error && err.message.includes("did not apply")) throw err; + last = asked.out; + } + } + await new Promise((r) => setTimeout(r, 5000)); + } + throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`); +} + +before(async () => { + if (skip) return; + + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + stocked = raised.images; + + await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); + await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); + const up = await must(`docker ps --format '{{.Names}}'`); + for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { + assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); + } + + await mesh(`node add ${MACHINE}`); + const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`)); + await must(`${HOST_PATH} enrol --token ${quote(token)}`); + await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); +}, { timeout: 1_800_000 }); + +after(async () => { + if (instanceId) await destroy(instanceId); + await destroyAll(`${SCENARIO}-`); +}, { timeout: 600_000 }); + +test("the mesh assigns sonarr's runtime, and it detects its key and serves its tools", { + skip, timeout: 900_000, +}, async () => { + // A minimal sonarr manifest: its tool runtime, and a config.xml the runtime detects its API key + // from — the file resource stands in for the running Sonarr that would write it. No Sonarr server + // or media mounts; the tools simply have nothing live to reach. + const manifest = JSON.stringify({ + module: "sonarr", + version: "1", + emits: ["module.sonarr.episode.grabbed", "module.sonarr.download.completed"], + consumes: [], + "own-secrets": { broker: "/var/lib/mesh/sonarr/broker" }, + resources: [ + { id: "mesh-state", type: "directory", path: "/var/lib/mesh/sonarr", mode: "0700" }, + { id: "config", type: "directory", path: "/services/sonarr/config", mode: "0700" }, + { + id: "config-xml", type: "file", path: "/services/sonarr/config/config.xml", mode: "0644", + content: "\n 8989\n labdetectedapikey0000000000000000\n\n", + }, + { + id: "runtime", type: "container", name: "mesh-sonarr", image: pinned("mesh-runtime-sonarr"), + network: "host", + volumes: [ + "/var/lib/mesh/sonarr/broker:/run/secrets/broker:ro", + "/services/sonarr/config:/var/lib/sonarr/config:ro", + ], + env: { + MESH_BROKER_FILE: "/run/secrets/broker", + MESH_SONARR_URL: "http://127.0.0.1:8989", + MESH_SONARR_CONFIG_DIR: "/var/lib/sonarr/config", + }, + }, + ], + }); + await must(`printf %s ${quote(manifest)} > /tmp/sonarr.json && docker cp /tmp/sonarr.json mesh-control:/sonarr.json`); + await mesh("module add /sonarr.json"); + + const issued = await mesh(`module issue sonarr --node ${MACHINE}`); + assert.match(issued, /scoped to what it emits and consumes/, issued); + await mesh(`assign ${MACHINE} sonarr`); + await mesh(`push ${MACHINE}`); + await settled(); + + const running = await must(`docker ps --format '{{.Names}}'`); + assert.match(running, /mesh-sonarr/, + `sonarr's runtime was assigned and is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`); + + const credential = await must(`cat /var/lib/mesh/sonarr/broker`); + assert.match(credential, /"url":"amqps:\/\/anchor-sonarr:/, `not the scoped account:\n${credential}`); + assert.doesNotMatch(credential, /guest:guest/, "sonarr's runtime holds the broker's own account"); + assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/, "no fingerprint to pin the broker"); + + // The runtime detected its API key from config.xml, registered its tools, and bound their serve + // queues — the queue on the broker is the proof the whole chain worked with no live Sonarr. + let served = ""; + const untilServing = Date.now() + 60_000; + while (Date.now() < untilServing) { + served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`); + if (/serve\.sonarr\.sonarr_status/.test(served)) break; + await new Promise((r) => setTimeout(r, 3000)); + } + assert.match(served, /serve\.sonarr\.sonarr_status/, + `sonarr's runtime never bound its serve queue (key not detected?):\n` + + `${(await on(`docker logs mesh-sonarr 2>&1 | tail -20`)).out}\n---\n${served}`); + + // A caller invokes sonarr_status over the mesh: it routes to the assigned runtime, which runs + // sonarr's real code and reports Sonarr unreachable (there is none). A reply — not a timeout — is + // the proof the invocation reached the runtime under its scoped account. + const invoked = await on( + `docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` + + `${pinned("mesh-runtime-sonarr")} invoke sonarr sonarr_status`, + 120_000, + ); + assert.doesNotMatch(invoked.out, /timed out/, + `sonarr_status timed out — nothing served the invocation:\n${invoked.out}`); + + const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`); + assert.match(users, /anchor-sonarr/, `the scoped account is not on the broker:\n${users}`); +});