import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdtemp, writeFile, readFile, readdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { compareVersions } from "../dist/primitives/index.js"; import { registerModuleTools, collectTools, collectToolsEach, resetTools, serveTools, listTools, invokeTool } from "../dist/tools/index.js"; import { runProvisioner } from "../dist/provisioner/index.js"; import { emit, on, type Event } from "../dist/events/index.js"; import { useBroker } from "../dist/messaging/index.js"; test("semver orders releases", () => { assert.equal(compareVersions("1.2.3", "1.2.10"), -1); assert.equal(compareVersions("2.0.0", "1.9.9"), 1); assert.equal(compareVersions("v1.0.0", "1.0.0"), 0); }); test("module tools register and collect, a thrower is skipped not fatal", () => { resetTools(); registerModuleTools("umami", () => [ { name: "umami_stats", description: "d", input: {}, run: async () => 1 }, ]); registerModuleTools("broken", () => { throw new Error("no token"); }); const collected = collectTools({}); const umami = collected.find((c) => c.module === "umami"); const broken = collected.find((c) => c.module === "broken"); assert.equal(umami?.tools.length, 1); assert.equal(broken?.tools.length, 0); }); test("provisioner creates each consumer with the mesh's login and password; remove waits for a person's delete", async () => { const dir = await mkdtemp(join(tmpdir(), "prov-")); const created: { as: string; password: string; values: unknown }[] = []; const removed: string[] = []; // What the mesh delivers: the password it minted (as the host leaves it after unsealing) and a // contributions file naming the consumer's login and where that password is. await writeFile(join(dir, "webapp.secret"), "minted-pw\n"); const receives = join(dir, "analytics.json"); const doc = (given: unknown[]): string => JSON.stringify({ contributions: 1, requirement: "analytics", given }); await writeFile( receives, doc([{ from: "webapp", node: "anchor", as: "webapp-anchor", secret: join(dir, "webapp.secret"), values: { name: "webapp" } }]), ); const stop = runProvisioner( "analytics", { async create(p) { created.push({ as: p.as, password: p.password, values: p.values }); }, async remove(p) { removed.push(p.as); }, }, { receives, everyMs: 20, stableForMs: 0 }, ); await waitFor(() => created.length === 1, 2000); // The adapter is handed the mesh's login and the mesh's password — not one it generated, and the // trailing newline of the unsealed file is stripped. assert.equal(created[0].as, "webapp-anchor"); assert.equal(created[0].password, "minted-pw"); assert.deepEqual(created[0].values, { name: "webapp" }); // The mesh drops the consumer: it is RETIRED, mark only — this adapter cannot disable, so remove is // NOT called (novox/hq ADR 0230); it keeps its access until a person deletes it. await writeFile(receives, doc([])); for (let i = 0; i < 200 && (await stop.retirement()).retired.length === 0; i++) await new Promise((r) => setTimeout(r, 10)); const r = await stop.retirement(); assert.equal(r.retires_by, "mark-only"); assert.deepEqual(r.retired.map((x) => [x.consumer, x.access]), [["webapp-anchor", "kept"]]); await new Promise((r) => setTimeout(r, 100)); assert.deepEqual(removed, [], "remove was called to retire"); // Only a person's delete reaches remove. await stop.deleteRetired("webapp-anchor", { why: "gone for good", confirm: "webapp-anchor" }); assert.deepEqual(removed, ["webapp-anchor"]); stop(); }); test("provisioner hands the adapter what the mesh derived, and still has it at withdrawal", async () => { // novox/hq ADR 0201: the provider declares what it derives for each consumer and the mesh fills // it. The adapter must be able to read it on create, must see a changed derivation as a change // worth re-applying, and must still be told it when the consumer is withdrawn — by which time // the mesh has already dropped it from the file. const dir = await mkdtemp(join(tmpdir(), "prov-derived-")); const created: unknown[] = []; const removed: unknown[] = []; await writeFile(join(dir, "ncloud.secret"), "minted-pw\n"); const receives = join(dir, "store.json"); const doc = (given: unknown[]): string => JSON.stringify({ contributions: 1, requirement: "s3-bucket", given }); const entry = (bucket: string) => ({ from: "nextcloud", node: "anchor", as: "mesh_anchor_ncloud", secret: join(dir, "ncloud.secret"), values: {}, derived: { bucket }, }); await writeFile(receives, doc([entry("mesh-anchor-ncloud")])); const stop = runProvisioner( "s3-bucket", { async create(p) { created.push(p.derived); }, async remove(p) { removed.push(p.derived); }, }, { receives, everyMs: 20, stableForMs: 0 }, ); await waitFor(() => created.length === 1, 2000); assert.deepEqual(created[0], { bucket: "mesh-anchor-ncloud" }); // A changed derivation is a changed consumer: the login and the password are the same, and the // resource it is being given is not. Trusted from memory, this would never be applied. await writeFile(receives, doc([entry("mesh-anchor-files")])); await waitFor(() => created.length === 2, 2000); assert.deepEqual(created[1], { bucket: "mesh-anchor-files" }); // Retired, then deleted by a person: the file no longer says anything about this consumer, and the // adapter is still told which bucket to take away. await writeFile(receives, doc([])); for (let i = 0; i < 200 && (await stop.retirement()).retired.length === 0; i++) await new Promise((r) => setTimeout(r, 10)); assert.deepEqual(removed, []); await stop.deleteRetired("mesh_anchor_ncloud", { why: "unassigned", confirm: "mesh_anchor_ncloud" }); assert.deepEqual(removed[0], { bucket: "mesh-anchor-files" }); stop(); }); test("provisioner stops calling an unreadable secret a race, and says what to look at", async () => { // novox/hq issue 225. A permission the host will never grant reads exactly like a file that has // not arrived yet, and three hours of "not readable yet" is how a provider that served nobody // looked healthy. After a while the wording has to change, because the word "yet" is the fault. const dir = await mkdtemp(join(tmpdir(), "prov-unreadable-")); const receives = join(dir, "db.json"); await writeFile( receives, JSON.stringify({ contributions: 1, requirement: "db", given: [{ from: "photos", node: "anchor", as: "mesh_anchor_photos", secret: join(dir, "never-written.secret"), values: {} }], }), ); const said: string[] = []; const wasError = console.error; console.error = (line: unknown) => { said.push(String(line)); }; let stop: (() => void) | undefined; try { stop = runProvisioner("db", { async create() {}, async remove() {} }, { receives, everyMs: 5 }); await waitFor(() => said.some((l) => l.includes("CANNOT READ")), 3000); } finally { stop?.(); console.error = wasError; } const patient = said.filter((l) => l.includes("not readable yet")); const loud = said.filter((l) => l.includes("CANNOT READ")); assert.ok(patient.length > 0, "it never gave the host a chance to write the file"); assert.ok(patient.length <= 12, `it called a permanent refusal a race ${patient.length} times`); assert.ok(loud.length > 0, "it never stopped calling it a race"); // And the loud one says where to look, because the answer is ownership and not the mesh. assert.ok(loud[0].includes("who owns the file"), `the loud line does not say what to check: ${loud[0]}`); assert.ok(loud[0].includes("issue 225"), "the loud line does not name the record"); }); test("provisioner removes nothing while its contributions file cannot be used (novox/hq issue 241)", async () => { const dir = await mkdtemp(join(tmpdir(), "prov-")); const created: string[] = []; const removed: string[] = []; await writeFile(join(dir, "a.secret"), "pw-a\n"); await writeFile(join(dir, "b.secret"), "pw-b\n"); const receives = join(dir, "postgres-database.json"); const doc = (given: unknown[]): string => JSON.stringify({ contributions: 1, requirement: "postgres-database", given }); const both = doc([ { from: "forge", node: "anchor", as: "mesh_anchor_forge", secret: join(dir, "a.secret") }, { from: "mail", node: "anchor", as: "mesh_anchor_mail", secret: join(dir, "b.secret") }, ]); await writeFile(receives, both); const stop = runProvisioner( "postgres-database", { async create(p) { created.push(p.as); }, async remove(p) { removed.push(p.as); }, }, { receives, everyMs: 20, stableForMs: 0 }, ); await waitFor(() => created.length === 2, 2000); // Each way the file can say nothing usable, held for many passes: nothing is withdrawn. for (const broken of [ async () => rm(receives), async () => writeFile(receives, "{ half a fi"), async () => writeFile(receives, JSON.stringify({ contributions: 1, requirement: "s3-bucket", given: [] })), async () => writeFile(receives, JSON.stringify({ contributions: 1, requirement: "postgres-database" })), ]) { await broken(); await new Promise((r) => setTimeout(r, 200)); assert.deepEqual(removed, [], "a consumer was removed while the file could not be used"); await writeFile(receives, both); await new Promise((r) => setTimeout(r, 100)); } // The file read and saying so is a result — but two of two held is over the bound (ADR 0230): // nothing is retired without a person, and approving that exact set retires both. await writeFile(receives, doc([])); for (let i = 0; i < 200 && (await stop.retirement()).waiting === null; i++) await new Promise((r) => setTimeout(r, 10)); assert.notEqual((await stop.retirement()).waiting, null, "the retirement never waited for a person"); assert.deepEqual(removed, [], "two of two consumers were retired without a person"); await stop.approve(["mesh_anchor_mail", "mesh_anchor_forge"], { why: "both unassigned on purpose", by: "test" }); // Retired mark-only: still nothing removed; that is a deletion, a person's further act. assert.deepEqual(removed, []); assert.deepEqual((await stop.retirement()).retired.map((x) => x.consumer).sort(), ["mesh_anchor_forge", "mesh_anchor_mail"]); stop(); }); test("provisioner applies again what the backend no longer holds, and trusts memory without holds", async () => { const dir = await mkdtemp(join(tmpdir(), "prov-holds-")); await writeFile(join(dir, "webapp.secret"), "minted-pw\n"); const receives = join(dir, "cache.json"); await writeFile( receives, JSON.stringify({ contributions: 1, requirement: "cache", given: [{ from: "webapp", node: "anchor", as: "webapp-anchor", secret: join(dir, "webapp.secret") }] }), ); // A backend that forgets: what was created is held until it "restarts". const backend = new Set(); let creates = 0; let asked = 0; const stop = runProvisioner( "cache", { async create(p) { creates++; backend.add(`${p.as}:${p.password}`); }, async remove(p) { for (const k of backend) if (k.startsWith(`${p.as}:`)) backend.delete(k); }, async holds(p) { asked++; return backend.has(`${p.as}:${p.password}`); }, }, { receives, everyMs: 10, verifyEveryMs: 30 }, ); await waitFor(() => creates === 1, 2000); // While the backend holds it, asking changes nothing: no second create. await waitFor(() => asked >= 2, 2000); assert.equal(creates, 1); // The backend restarts and forgets. The contributions did not change; only asking can notice. backend.clear(); await waitFor(() => creates === 2, 2000); assert.ok(backend.has("webapp-anchor:minted-pw")); stop(); // An adapter without holds is trusted from memory, as before: a forgotten backend stays forgotten. const forgetful = new Set(); let plainCreates = 0; const stopPlain = runProvisioner( "cache", { async create(p) { plainCreates++; forgetful.add(p.as); }, async remove() {}, }, { receives, everyMs: 10, verifyEveryMs: 10 }, ); await waitFor(() => plainCreates === 1, 2000); forgetful.clear(); await new Promise((r) => setTimeout(r, 100)); assert.equal(plainCreates, 1); stopPlain(); }); test("provisioner keeps a consumer applied when the backend cannot be asked", async () => { const dir = await mkdtemp(join(tmpdir(), "prov-unreachable-")); await writeFile(join(dir, "webapp.secret"), "minted-pw"); const receives = join(dir, "cache.json"); await writeFile( receives, JSON.stringify({ requirement: "cache", given: [{ as: "webapp-anchor", secret: join(dir, "webapp.secret") }] }), ); let creates = 0; let asked = 0; const stop = runProvisioner( "cache", { async create() { creates++; }, async remove() {}, async holds() { asked++; throw new Error("connection refused"); }, }, { receives, everyMs: 10, verifyEveryMs: 20 }, ); await waitFor(() => asked >= 3, 2000); // Unable to ask is not evidence of loss: nothing is applied again. assert.equal(creates, 1); stop(); }); test("provisioner backs off when create does not satisfy holds, and says so", async () => { const dir = await mkdtemp(join(tmpdir(), "prov-brake-")); await writeFile(join(dir, "webapp.secret"), "minted-pw"); const receives = join(dir, "cache.json"); await writeFile(receives, JSON.stringify({ requirement: "cache", given: [{ as: "webapp-anchor", secret: join(dir, "webapp.secret") }] })); let creates = 0; let asked = 0; const logged: string[] = []; const original = console.error; console.error = (...a: unknown[]) => logged.push(a.join(" ")); try { const stop = runProvisioner( "cache", { async create() { creates++; }, async remove() {}, async holds() { asked++; return false; // an adapter that disagrees with itself: nothing create does is ever held }, }, { receives, everyMs: 5, verifyEveryMs: 20 }, ); await new Promise((r) => setTimeout(r, 400)); stop(); } finally { console.error = original; } // Without the brake this would be ~20 creates (one per 20ms check). With doubling waits // (20, 40, 80, 160ms…) it is a handful. assert.ok(creates >= 3 && creates <= 7, `creates: ${creates}`); assert.equal(asked, creates - 1); assert.ok(logged.some((l) => l.includes("create does not produce what holds checks"))); }); test("provisioner treats a failed or hung holds as could-not-ask, and never logs the password", async () => { const dir = await mkdtemp(join(tmpdir(), "prov-timeout-")); await writeFile(join(dir, "a.secret"), "s3cret-pw"); await writeFile(join(dir, "b.secret"), "other-pw"); const receives = join(dir, "cache.json"); await writeFile( receives, JSON.stringify({ requirement: "cache", given: [{ as: "a", secret: join(dir, "a.secret") }, { as: "b", secret: join(dir, "b.secret") }] }), ); let creates = 0; const askedFor: string[] = []; const logged: string[] = []; const original = console.error; console.error = (...a: unknown[]) => logged.push(a.join(" ")); try { const stop = runProvisioner( "cache", { async create() { creates++; }, async remove() {}, holds(p) { askedFor.push(p.as); // "a" fails the way a shelled-out tool does, with its arguments in the message. if (p.as === "a") return Promise.reject(new Error(`Command failed: tool --password ${p.password}`)); return Promise.resolve(true); }, }, { receives, everyMs: 5, verifyEveryMs: 30, holdsTimeoutMs: 20 }, ); await new Promise((r) => setTimeout(r, 200)); stop(); } finally { console.error = original; } assert.equal(creates, 2); // the first pass only; nothing is applied again on "could not ask" // A failure that is not a timeout may be one consumer's alone: the consumers after it are still // asked on every pass. assert.ok(askedFor.filter((as) => as === "b").length >= 2, askedFor.join(",")); assert.ok(logged.some((l) => l.includes("Command failed: tool --password ***"))); assert.ok(!logged.some((l) => l.includes("s3cret-pw"))); // A check that never answers counts as could-not-ask once its time is up. let hungCreates = 0; const hungLogged: string[] = []; console.error = (...a: unknown[]) => hungLogged.push(a.join(" ")); try { const stopHung = runProvisioner( "cache", { async create() { hungCreates++; }, async remove() {}, holds: () => new Promise(() => {}), }, { receives, everyMs: 5, verifyEveryMs: 30, holdsTimeoutMs: 20 }, ); await new Promise((r) => setTimeout(r, 200)); stopHung(); } finally { console.error = original; } assert.equal(hungCreates, 2); assert.ok(hungLogged.some((l) => l.includes("no answer within 20ms"))); }); test("provisioner stops a verify pass at a timeout, and retries a failed re-apply at the next check", async () => { const dir = await mkdtemp(join(tmpdir(), "prov-retry-")); await writeFile(join(dir, "a.secret"), "pa"); await writeFile(join(dir, "b.secret"), "pb"); const receives = join(dir, "cache.json"); await writeFile( receives, JSON.stringify({ requirement: "cache", given: [{ as: "a", secret: join(dir, "a.secret") }, { as: "b", secret: join(dir, "b.secret") }] }), ); const original = console.error; console.error = () => {}; try { // A timeout on "a" means "b" is not asked in that pass. const askedFor: string[] = []; const stopHung = runProvisioner( "cache", { async create() {}, async remove() {}, holds(p) { askedFor.push(p.as); return p.as === "a" ? new Promise(() => {}) : Promise.resolve(true); }, }, { receives, everyMs: 5, verifyEveryMs: 30, holdsTimeoutMs: 10 }, ); await new Promise((r) => setTimeout(r, 150)); stopHung(); assert.ok(askedFor.length >= 2 && askedFor.every((as) => as === "a"), askedFor.join(",")); // A lost consumer whose re-apply fails is retried at the next check, not held back by the brake, // and the retry that succeeds counts as the first re-apply, not the second. let held = true; let createCalls = 0; const logged: string[] = []; console.error = (...a: unknown[]) => logged.push(a.join(" ")); const stop = runProvisioner( "cache", { async create(p) { if (p.as !== "a") return; createCalls++; if (createCalls === 2) throw new Error("write refused"); // the first re-apply fails if (createCalls === 3) held = true; // the retry works }, async remove() {}, async holds(p) { return p.as === "a" ? held : true; }, }, { receives, everyMs: 5, verifyEveryMs: 20 }, ); await waitFor(() => createCalls === 1, 2000); held = false; // the backend loses "a" await waitFor(() => createCalls === 3, 2000); await new Promise((r) => setTimeout(r, 100)); stop(); assert.equal(createCalls, 3); assert.ok(!logged.some((l) => l.includes("create does not produce what holds checks")), logged.join("\n")); } finally { console.error = original; } }); test("modules can SERVE: a real async tool, loaded and invoked over the broker", async () => { resetTools(); // Stand up a fake upstream the tool actually calls over the network — proving a tool that does // real work (not a pure function) serves end to end. const { createServer } = await import("node:http"); const server = createServer((_req, res) => { res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ id: "site-123" })); }); await new Promise((r) => server.listen(0, r)); const addr = server.address() as { port: number }; const base = `http://127.0.0.1:${addr.port}`; // A module registers its tool exactly as umami does — the tool calls the upstream and returns a // result computed from it. registerModuleTools("demo", () => [ { name: "create_site", description: "register a site and return its snippet", input: { domain: { type: "string" } }, run: async (args) => { const res = await fetch(`${base}/api/websites`, { method: "POST" }); const { id } = (await res.json()) as { id: string }; return { domain: String(args.domain), snippet: `` }; }, }, ]); const broker = memBroker(); const stop = await serveTools(broker, {}); // Invoke it the way a caller (mesh-controller's command API) would — over the broker, by module and // tool, each served on its own key `demo.create_site` (novox/hq ADR 0047). const result = (await invokeTool(broker, "demo", "create_site", { domain: "my-app" })) as { snippet: string }; assert.match(result.snippet, /data-website-id="site-123"/); // Discovery works, and an unknown tool is refused rather than silently dropped. assert.deepEqual(listTools({}).map((t) => t.name), ["create_site"]); await assert.rejects(invokeTool(broker, "demo", "nope", {})); stop(); server.close(); }); test("serving refuses one module exposing two tools of the same name", async () => { // Two *modules* may share a tool name — each is served on its own `module.tool` key (ADR 0047). // What is refused is one module exposing the same name twice, where the key would collide. resetTools(); registerModuleTools("a", () => [ { name: "dup", description: "", input: {}, run: async () => 1 }, { name: "dup", description: "", input: {}, run: async () => 2 }, ]); await assert.rejects(serveTools(memBroker(), {}), /exposes two tools named dup/); }); // A minimal in-memory broker: request routes to a registered handle, and publish routes to every // subscriber whose topic pattern matches. Enough to exercise tools and events; the real binding is // the mesh's AMQP one, provided by the hosting runtime. function memBroker() { const handlers = new Map Promise>(); const subs: { pattern: string; handler: (env: { key: string; node: string; body: unknown }) => Promise }[] = []; return { async request(key: string, body: Req): Promise { const h = handlers.get(key); if (!h) throw new Error(`no handler for ${key}`); return (await h(body)) as Res; }, async handle(key: string, handler: (b: Req) => Promise): Promise<() => void> { handlers.set(key, handler as (b: unknown) => Promise); return () => handlers.delete(key); }, async publish(env: { key: string; node: string; body: T }): Promise { for (const s of subs) if (topicMatch(s.pattern, env.key)) await s.handler(env); }, async subscribe(pattern: string, handler: (env: { key: string; node: string; body: T }) => Promise): Promise<() => void> { const entry = { pattern, handler: handler as (env: { key: string; node: string; body: unknown }) => Promise }; subs.push(entry); return () => { const i = subs.indexOf(entry); if (i >= 0) subs.splice(i, 1); }; }, async close(): Promise {}, }; } // AMQP topic matching: `*` one segment, `#` any run of segments. function topicMatch(pattern: string, key: string): boolean { if (pattern === "#") return true; const p = pattern.split("."); const k = key.split("."); let pi = 0; let ki = 0; while (pi < p.length) { if (p[pi] === "#") return true; // simplification: # only as a trailing wildcard if (ki >= k.length) return false; if (p[pi] !== "*" && p[pi] !== k[ki]) return false; pi++; ki++; } return ki === k.length; } test("events: a module emits, a listener and the audit sink (#) both receive it, with metadata", async () => { const broker = memBroker(); useBroker(() => broker); process.env.MESH_MODULE = "umami"; process.env.MESH_NODE = "anchor"; const heard: Event[] = []; const audited: Event[] = []; await on("module.umami.#", async (e) => void heard.push(e)); await on("#", async (e) => void audited.push(e)); // the audit logger is just this await emit("module.umami.site.created", { domain: "my-app" }); await emit("module.plex.play.started", { title: "x" }); // a different module's event // The umami listener heard only umami's event; the audit sink heard both. assert.deepEqual(heard.map((e) => e.type), ["module.umami.site.created"]); assert.deepEqual(audited.map((e) => e.type), ["module.umami.site.created", "module.plex.play.started"]); // The metadata an audit trail needs is present — read back from the ADR 0042 headers, not the body. const e = heard[0]; assert.equal(e.source, "umami"); assert.equal(e.node, "anchor"); assert.equal((e.body as { domain: string }).domain, "my-app"); assert.match(e.at, /^\d{4}-\d{2}-\d{2}T/); // Every event carries a unique id (x-event-id) — the handle a consumer dedups on. assert.ok(e.id, "event has an x-event-id"); assert.notEqual(audited[0].id, audited[1].id); // The body is exactly the domain payload — provenance never leaks into it. assert.deepEqual(Object.keys(e.body as object), ["domain"]); delete process.env.MESH_MODULE; delete process.env.MESH_NODE; }); async function waitFor(cond: () => boolean, ms: number): Promise { const start = Date.now(); while (!cond()) { if (Date.now() - start > ms) throw new Error("timed out waiting"); await new Promise((r) => setTimeout(r, 10)); } } test("each registration is given the environment asked for it, and sees no other's (ADR 0192)", () => { resetTools(); const seen: Record = {}; registerModuleTools("alpha", (env) => { seen.alpha = { ...env }; return [{ name: "a", description: "", input: {}, run: async () => env.ALPHA_FILE }]; }); registerModuleTools("beta", (env) => { seen.beta = { ...env }; return [{ name: "b", description: "", input: {}, run: async () => env.BETA_URL }]; }); const envs: Record = { alpha: { ALPHA_FILE: "/a/config.json" }, beta: { BETA_URL: "http://127.0.0.1:3000" } }; const asked: string[] = []; const got = collectToolsEach((module, index) => { asked.push(`${module}#${index}`); return envs[module] ?? {}; }); assert.deepEqual(got.map((g) => g.module), ["alpha", "beta"]); assert.deepEqual(asked, ["alpha#0", "beta#1"]); assert.deepEqual(seen.alpha, { ALPHA_FILE: "/a/config.json" }); assert.deepEqual(seen.beta, { BETA_URL: "http://127.0.0.1:3000" }); resetTools(); });