/** * The mesh plays out the model-access refreshable-grant flow for Anthropic end to end, with the * vendor's OAuth endpoint STUBBED (novox/hq ADR 0050, Phase C). It proves the one property the * carve-out rests on, and the reviewer will scrutinise it: the refresh token is opened ONLY on the * manager node, the control plane is handed only the ACCESS token in the clear (plus an opaque * re-sealed refresh envelope), and a consuming node writes an access-token-only credential and is * never delivered a refresh token — nowhere on the machine, nowhere in the control plane's database. * * The flow, driven deterministically (the runtime images are the real ones the host pulled; the * OAuth endpoint is a tiny node stub the test runs from the same image, so no vendor is reached): * 1. adopt: the manager runtime seals a refresh token at rest to a node key pair (the seal runs on * the manager node; the plaintext never leaves it). The sealed envelope is stored via * `licence set-grant` — the control plane stores ciphertext it cannot open. * 2. refresh: the manager runtime OPENS the envelope with the node's own key, calls the stub token * endpoint, and writes out ONLY { access token, re-sealed refresh envelope }. * 3. submit: `licence submit-refresh` hands the control plane those two things — never the refresh * token in the clear — and it seals the access token to the consumer holder. * 4. deliver: a push delivers the sealed access token to the consumer; the consumer runtime writes * ~/.claude/.credentials.json, access-token-only. * * STUBBED, and flagged in the report: (a) the vendor OAuth endpoint (a node stub); (b) the manager * node's private sealing key, mounted as a test key pair — production needs a host capability to * place the node private key where a manager module reads it, which mesh-host does not have today; * (c) the submit transport (the test invokes `mesh-control licence submit-refresh` on the manager's * output, standing in for the authenticated cross-node call a manager node would make). * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock * Build both runtime images into the local daemon first: * scripts/build-module-runtime.sh anthropic-manager /tmp/anthropic-manager.tar * scripts/build-module-runtime.sh anthropic-consumer /tmp/anthropic-consumer.tar */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { existsSync, readFileSync } from "node:fs"; import { generateKeyPairSync } from "node:crypto"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { labIsUsable, destroyAll } from "./harness.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" : false; const SCENARIO = "anthropic-bed"; const MACHINE = "anchor"; // The fake tokens the flow moves. The whole test is: the second reaches the consumer, the first never // does. const REFRESH_TOKEN = "rt-lab-refresh-must-never-be-delivered"; const ACCESS_TOKEN = "at-lab-access-token-minted-by-the-stub"; const ROTATED_REFRESH = "rt-lab-rotated-still-must-never-be-delivered"; let instanceId = ""; let stocked: string[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { const { stdout } = await exec(instanceId, MACHINE, [ "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, ], timeoutMs); const marker = stdout.lastIndexOf("__exit="); if (marker < 0) return { out: stdout, ok: false }; return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; } async function must(command: string, timeoutMs?: number): Promise { const { out, ok } = await on(command, timeoutMs); if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`); return out; } async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } async function meshTry(command: string): Promise<{ out: string; ok: boolean }> { return on(`docker exec mesh-control /mesh-control ${command}`); } function pinned(repository: string): string { const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); return found; } function bundleFor(images: string[]): string { let text = readFileSync(bundle, "utf8"); for (const ref of images) { const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); } return text; } function tokenFrom(said: string): string { const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); assert.ok(found, `no token in:\n${said}`); return found; } /** A node key pair as the mesh records it: raw 32-byte X25519 keys, standard base64. */ function nodeKeyPair(): { pub: string; priv: string } { const kp = generateKeyPairSync("x25519"); const std = (b64url: string) => Buffer.from(b64url, "base64url").toString("base64"); return { pub: std((kp.publicKey.export({ format: "jwk" }) as { x: string }).x), priv: std((kp.privateKey.export({ format: "jwk" }) as { d: string }).d), }; } /** The local image id the host pulled for a runtime, so the test can drive it deterministically. */ async function imageId(substr: string): Promise { const out = (await must(`docker images --no-trunc --format '{{.ID}} {{.Repository}}' | grep ${quote(substr)} | head -1`)).trim(); const id = out.split(/\s+/)[0] ?? ""; assert.ok(id.startsWith("sha256:") || id.length > 0, `no local image matched ${substr}:\n${out}`); return id; } async function settled(withinMs = 600_000): Promise { const until = Date.now() + withinMs; let last = ""; while (Date.now() < until) { const asked = await meshTry(`status --json`); if (asked.ok) { try { const state = JSON.parse(asked.out) as { wrong: { node: string; outcome: string }[]; waiting: { node: string }[]; reported: { node: string; outcome: string; current: boolean }[]; }; const bad = state.wrong.find((w) => w.node === MACHINE); if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`); const word = state.reported.find((r) => r.node === MACHINE); if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return; last = asked.out; } catch (err) { if (err instanceof Error && err.message.includes("did not apply")) throw err; last = asked.out; } } await new Promise((r) => setTimeout(r, 5000)); } throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`); } before(async () => { if (skip) return; const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; stocked = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); const up = await must(`docker ps --format '{{.Names}}'`); for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); } await mesh(`node add ${MACHINE}`); const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`)); await must(`${HOST_PATH} enrol --token ${quote(token)}`); await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); }, { timeout: 1_800_000 }); after(async () => { if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 600_000 }); test("model access refreshes on the manager node and delivers only the access token, never the refresh token", { skip, timeout: 1_500_000, }, async () => { const managerImage = pinned("mesh-runtime-anthropic-manager"); const consumerImage = pinned("mesh-runtime-anthropic-consumer"); // --- the licence, its manager, and the consumer holder ------------------------------------------ await mesh(`licence add anthropic personal --serves '{"model":"a-model"}'`); await mesh(`licence manager personal ${MACHINE}`); await mesh(`licence use personal ${MACHINE} anthropic-consumer`); // --- both runtimes are placed so the host pulls their images (which the test then drives) -------- // Inline manifests, env pointed at the stub, mirroring the committed module.json. The scheduled // containers install as present state (ADR 0053); the test drives the entrypoints directly for a // deterministic flow rather than waiting on cron. const managerManifest = JSON.stringify({ module: "anthropic-manager", version: "1", "own-secrets": { broker: "/var/lib/mesh/anthropic-manager/broker" }, emits: ["module.anthropic-manager.usage.read"], resources: [ { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-manager", mode: "0700" }, { id: "keys", type: "directory", path: "/var/lib/mesh/anthropic-manager/keys", mode: "0700" }, { id: "out", type: "directory", path: "/var/lib/mesh/anthropic-manager/out", mode: "0700" }, { id: "refresh", type: "container", name: "mesh-anthropic-manager-refresh", image: managerImage, network: "host", schedule: "*/9 * * * *", args: ["run", "/app/modules/anthropic-manager/dist/refresh/index.js"], volumes: ["/var/lib/mesh/anthropic-manager:/run/state"], env: { MESH_ANTHROPIC_LICENCE: "personal", MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token", MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage", MESH_ANTHROPIC_GRANT_FILE: "/run/state/grant.json", MESH_NODE_SEALING_PUBLIC_FILE: "/run/state/keys/sealing.pub", MESH_NODE_SEALING_PRIVATE_FILE: "/run/state/keys/sealing.priv", MESH_ANTHROPIC_ACCESS_OUT: "/run/state/out/access-token", MESH_ANTHROPIC_GRANT_OUT: "/run/state/out/grant.json", MESH_ANTHROPIC_USAGE_OUT: "/run/state/out/usage.json", }, }, ], }); const consumerManifest = JSON.stringify({ module: "anthropic-consumer", version: "1", requires: ["model-access"], binds: { "model-access": "/var/lib/anthropic-consumer/model.json" }, secrets: { "model-access": "/var/lib/anthropic-consumer/access-token" }, "own-secrets": { broker: "/var/lib/mesh/anthropic-consumer/broker" }, emits: ["module.anthropic-consumer.usage.session"], resources: [ { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-consumer", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/anthropic-consumer", mode: "0700" }, { id: "claude-home", type: "directory", path: "/var/lib/anthropic-consumer/claude", mode: "0700" }, { id: "apply", type: "container", name: "mesh-anthropic-consumer-apply", image: consumerImage, network: "host", schedule: "*/9 * * * *", args: ["run", "/app/modules/anthropic-consumer/dist/apply/index.js"], volumes: ["/var/lib/anthropic-consumer:/run/state"], env: { MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/access-token", MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json", MESH_CLAUDE_CREDENTIALS_FILE: "/run/state/claude/.credentials.json", MESH_CLAUDE_IDENTITY_FILE: "/run/state/claude/.claude.json", }, }, ], }); for (const [name, body] of [["anthropic-manager", managerManifest], ["anthropic-consumer", consumerManifest]] as const) { await must(`printf %s ${quote(body)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); await mesh(`module add /${name}.json`); await mesh(`module issue ${name} --node ${MACHINE}`); await mesh(`assign ${MACHINE} ${name}`); } await mesh(`push ${MACHINE}`); await settled(); // The images the host pulled to run the scheduled containers are now local; drive them directly. const managerId = await imageId("anthropic-manager"); const consumerId = await imageId("anthropic-consumer"); // --- the stubbed node private key, mounted (FLAGGED) -------------------------------------------- // Production needs a host capability to place the node private key where the manager reads it; // mesh-host has none today. Here the test mounts a generated key pair as that key. const keys = nodeKeyPair(); await must(`printf %s ${quote(keys.pub)} > /var/lib/mesh/anthropic-manager/keys/sealing.pub`); await must(`printf %s ${quote(keys.priv)} > /var/lib/mesh/anthropic-manager/keys/sealing.priv`); // --- the OAuth stub: a tiny node server run from the manager image itself ----------------------- const stub = [ "const http=require('http');", "http.createServer((req,res)=>{let b='';req.on('data',c=>b+=c);req.on('end',()=>{", " if(req.url.startsWith('/token')){res.setHeader('content-type','application/json');", ` res.end(JSON.stringify({access_token:${JSON.stringify(ACCESS_TOKEN)},refresh_token:${JSON.stringify(ROTATED_REFRESH)},expires_in:3600,refresh_token_expires_in:2592000,subscription_type:'pro'}));return;}`, " if(req.url.startsWith('/usage')){res.setHeader('content-type','application/json');", " res.end(JSON.stringify({five_hour:{utilization:12,resets_at:'2026-01-01T00:00:00Z'},seven_day:{utilization:3}}));return;}", " res.statusCode=404;res.end('no');});}).listen(9099,'127.0.0.1',()=>console.log('stub up'));", ].join("\n"); await must(`printf %s ${quote(stub)} > /var/lib/mesh/anthropic-manager/stub.js`); await must(`docker rm -f oauth-stub 2>/dev/null; docker run -d --name oauth-stub --network host --entrypoint node -v /var/lib/mesh/anthropic-manager/stub.js:/run/stub.js:ro ${managerId} /run/stub.js`); // Give it a moment to bind. await must(`for i in $(seq 1 20); do curl -s -X POST http://127.0.0.1:9099/token >/dev/null && break; sleep 1; done`); // --- 1. adopt: seal the refresh token at rest, on the manager node ------------------------------ await must(`printf %s ${quote(REFRESH_TOKEN)} > /var/lib/mesh/anthropic-manager/refresh-token`); await must( `docker run --rm --network host -v /var/lib/mesh/anthropic-manager:/run/state ` + `-e MESH_ANTHROPIC_REFRESH_TOKEN_FILE=/run/state/refresh-token ` + `-e MESH_NODE_SEALING_PUBLIC_FILE=/run/state/keys/sealing.pub ` + `-e MESH_ANTHROPIC_GRANT_OUT=/run/state/grant.json ` + `${managerId} run /app/modules/anthropic-manager/dist/adopt/index.js`, ); const envelope = await must(`cat /var/lib/mesh/anthropic-manager/grant.json`); assert.doesNotMatch(envelope, new RegExp(REFRESH_TOKEN), `the adopted envelope holds the refresh token in the clear:\n${envelope}`); // Store the sealed envelope in the control plane — which never sees the refresh token. await must(`docker cp /var/lib/mesh/anthropic-manager/grant.json mesh-control:/grant.json`); await mesh(`licence set-grant personal --file /grant.json`); // --- 2. refresh: open on the manager node, call the stub, write access token + re-sealed refresh - await must( `docker run --rm --network host -v /var/lib/mesh/anthropic-manager:/run/state ` + `-e MESH_ANTHROPIC_LICENCE=personal ` + `-e MESH_ANTHROPIC_TOKEN_ENDPOINT=http://127.0.0.1:9099/token ` + `-e MESH_ANTHROPIC_USAGE_ENDPOINT=http://127.0.0.1:9099/usage ` + `-e MESH_ANTHROPIC_GRANT_FILE=/run/state/grant.json ` + `-e MESH_NODE_SEALING_PUBLIC_FILE=/run/state/keys/sealing.pub ` + `-e MESH_NODE_SEALING_PRIVATE_FILE=/run/state/keys/sealing.priv ` + `-e MESH_ANTHROPIC_ACCESS_OUT=/run/state/out/access-token ` + `-e MESH_ANTHROPIC_GRANT_OUT=/run/state/out/grant.json ` + `-e MESH_ANTHROPIC_USAGE_OUT=/run/state/out/usage.json ` + `${managerId} run /app/modules/anthropic-manager/dist/refresh/index.js`, ); const producedAccess = (await must(`cat /var/lib/mesh/anthropic-manager/out/access-token`)).trim(); assert.equal(producedAccess, ACCESS_TOKEN, "the manager did not mint the stub's access token"); const newEnvelope = await must(`cat /var/lib/mesh/anthropic-manager/out/grant.json`); assert.doesNotMatch(newEnvelope, new RegExp(ROTATED_REFRESH), `the re-sealed envelope holds the rotated refresh token in the clear:\n${newEnvelope}`); // Licence-grain usage was read and recorded. const usage = await must(`cat /var/lib/mesh/anthropic-manager/out/usage.json`); assert.match(usage, /"sessionPct":12/, `the licence-grain usage reading is wrong:\n${usage}`); // --- 3. submit: the control plane is handed only the access token + opaque envelope ------------- await must(`docker cp /var/lib/mesh/anthropic-manager/out/access-token mesh-control:/access-token`); await must(`docker cp /var/lib/mesh/anthropic-manager/out/grant.json mesh-control:/new-grant.json`); const submitted = await mesh(`licence submit-refresh personal --access-file /access-token --grant-file /new-grant.json`); assert.match(submitted, /sealed to 1 holder/, submitted); // --- 4. deliver: the consumer gets the sealed access token and writes the credential ------------- await mesh(`push ${MACHINE}`); await settled(); // Drive the consumer's apply once the token has been delivered to its secret path. let delivered = false; for (let i = 0; i < 20 && !delivered; i++) { delivered = (await on(`test -s /var/lib/anthropic-consumer/access-token`)).ok; if (!delivered) await new Promise((r) => setTimeout(r, 3000)); } assert.ok(delivered, "the sealed access token was never delivered to the consumer's secret path"); await must( `docker run --rm --network host -v /var/lib/anthropic-consumer:/run/state ` + `-e MESH_MODEL_ACCESS_SECRET_FILE=/run/state/access-token ` + `-e MESH_MODEL_ACCESS_BIND_FILE=/run/state/model.json ` + `-e MESH_CLAUDE_CREDENTIALS_FILE=/run/state/claude/.credentials.json ` + `-e MESH_CLAUDE_IDENTITY_FILE=/run/state/claude/.claude.json ` + `${consumerId} run /app/modules/anthropic-consumer/dist/apply/index.js`, ); // --- the invariant, asserted from every angle --------------------------------------------------- const creds = await must(`cat /var/lib/anthropic-consumer/claude/.credentials.json`); assert.match(creds, new RegExp(ACCESS_TOKEN), `the access token did not reach the credential file:\n${creds}`); const parsed = JSON.parse(creds) as { claudeAiOauth?: { accessToken?: string; refreshToken?: string } }; assert.equal(parsed.claudeAiOauth?.accessToken, ACCESS_TOKEN); assert.ok(!parsed.claudeAiOauth?.refreshToken, "the consumer was given a refresh token"); // The refresh token — original or rotated — is nowhere on the consuming node. for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) { const found = await on(`grep -rq ${quote(secret)} /var/lib/anthropic-consumer`); assert.ok(!found.ok, `a refresh token is on the consuming node under /var/lib/anthropic-consumer`); } // The control plane's own database holds the refresh token only as ciphertext. const grantRow = await must( `docker exec mesh-store psql -U postgres -d licences -qAt -c "select token, wrapped_key from refresh_grant where licence='personal'"`, ); assert.ok(grantRow.trim().length > 0, "no refresh grant was stored"); for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) { assert.doesNotMatch(grantRow, new RegExp(secret), `the refresh token is in the control plane's database in the clear:\n${grantRow}`); } // And the holder's sealed column carries the access token's seal, never a refresh token. const holder = await must( `docker exec mesh-store psql -U postgres -d licences -qAt -c "select coalesce(sealed,'') from licence_holder where licence='personal'"`, ); assert.ok(holder.trim().length > 0, "nothing was sealed to the holder"); for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) { assert.doesNotMatch(holder, new RegExp(secret), "a refresh token is in the holder row"); } await must(`docker rm -f oauth-stub 2>/dev/null || true`); });