/** * The mesh plays out the model-access refreshable-grant flow for Anthropic end to end, with the * vendor's OAuth endpoint STUBBED (novox/hq ADR 0050). It proves the one property the carve-out rests * on, and the reviewer will scrutinise it: the refresh token is delivered ONLY to the manager node — * as an ordinary sealed credential the HOST unseals — the control plane is handed only the ACCESS * token in the clear (plus an opaque re-sealed refresh box), and a consuming node writes an * access-token-only credential and is never delivered a refresh token — nowhere on the machine, * nowhere in the control plane's database. * * **What changed from the earlier cut, and why this is simpler.** The refresh token no longer rides a * bespoke at-rest envelope the module opens with a node private key the mesh must somehow place — a * module is never given a node's private key, so that path could not exist. It rides the ORDINARY * sealed-delivery path instead: mesh-controller (via the manager module at adoption) seals it to the * manager node's PUBLIC key, and the HOST unseals it with that node's real private key and mounts the * cleartext at the manager module's bound secret path — exactly as a consumer's db password arrives. * So there is no fake node key pair mounted here any more; the host's own real sealing key does the * unsealing, and the manager module does no crypto beyond re-sealing a rotated token to the same * public key. * * The flow, driven deterministically (the runtime images are the real ones the host pulled; the OAuth * endpoint is a tiny node stub run from the same image, so no vendor is reached): * 1. deploy the manager and adopt: the manager holder is delivered its bound facts (carrying the * node's PUBLIC sealing key). The manager runtime seals the operator's refresh token to that key * and hands the box to `licence set-grant` — the control plane stores ciphertext it cannot open. * 2. the host unseals: a push delivers the sealed refresh token to the manager, and the host mounts * the cleartext at the manager module's secret path — the one place a refresh token is readable. * 3. refresh: the manager runtime reads that cleartext, calls the stub token endpoint, re-seals a * rotated refresh token to the node's public key, and writes out ONLY { access token, sealed box }. * 4. submit: `licence submit-refresh` hands the control plane those two things — never the refresh * token in the clear — and it seals the access token to the consumer holder. * 5. deliver: a push delivers the sealed access token to the consumer; the consumer runtime writes * ~/.claude/.credentials.json, access-token-only. * * STUBBED, and flagged in the report: (a) the vendor OAuth endpoint (a node stub); (b) the submit * transport (the test invokes `mesh-controller licence submit-refresh` on the manager's output, standing * in for the authenticated cross-node call a manager node would make). The node-private-key stub of * the earlier cut is GONE — the host uses its own real key. * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock * Build both runtime images into the local daemon first: * scripts/build-module-runtime.sh anthropic-manager /tmp/anthropic-manager.tar * scripts/build-module-runtime.sh anthropic-consumer /tmp/anthropic-consumer.tar */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { existsSync, readFileSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" : false; const SCENARIO = "anthropic-bed"; const MACHINE = "anchor"; // The fake tokens the flow moves. The whole test is: the second reaches the consumer, the first never // does — except on the manager node, which is allowed to read it. const REFRESH_TOKEN = "rt-lab-refresh-only-the-manager-may-read"; const ACCESS_TOKEN = "at-lab-access-token-minted-by-the-stub"; const ROTATED_REFRESH = "rt-lab-rotated-still-only-the-manager"; let instanceId = ""; let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; } async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { const { stdout } = await exec(instanceId, MACHINE, [ "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, ], timeoutMs); const marker = stdout.lastIndexOf("__exit="); if (marker < 0) return { out: stdout, ok: false }; return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" }; } async function must(command: string, timeoutMs?: number): Promise { const { out, ok } = await on(command, timeoutMs); if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`); return out; } async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); } // The manager's adopt/refresh runtime writes its outputs as root, mode 0600 (secret files). To hand // one to `mesh-controller` — whose process runs as a non-root user — the test relaxes the mode on the // anchor host (where `must` is root) and then copies it in: `docker cp` preserves the source mode, so // the file lands 0644 and mesh-controller (a distroless image with no `chmod` of its own) can read it. // What is staged this way is a sealed box or the access token, never a cleartext refresh token, so a // world-readable copy discloses nothing the control plane does not already hold. In production the // operator who ran adopt owns the file and this does not arise. async function stageIntoControl(hostPath: string, dest: string): Promise { await must(`chmod 0644 ${hostPath} && docker cp ${hostPath} mesh-controller:${dest}`); } async function meshTry(command: string): Promise<{ out: string; ok: boolean }> { return on(`docker exec mesh-controller /mesh-controller ${command}`); } /** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ function pinned(reference: string): string { return onTheMachine(reference, held); } function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); } function tokenFrom(said: string): string { const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); assert.ok(found, `no token in:\n${said}`); return found; } /** The local image id the host pulled for a runtime, so the test can drive it deterministically. */ async function imageId(substr: string): Promise { const out = (await must(`docker images --no-trunc --format '{{.ID}} {{.Repository}}' | grep ${quote(substr)} | head -1`)).trim(); const id = out.split(/\s+/)[0] ?? ""; assert.ok(id.startsWith("sha256:") || id.length > 0, `no local image matched ${substr}:\n${out}`); return id; } async function settled(withinMs = 600_000): Promise { const until = Date.now() + withinMs; let last = ""; while (Date.now() < until) { const asked = await meshTry(`status --json`); if (asked.ok) { try { const state = JSON.parse(asked.out) as { wrong: { node: string; outcome: string }[]; waiting: { node: string }[]; reported: { node: string; outcome: string; current: boolean }[]; }; const bad = state.wrong.find((w) => w.node === MACHINE); if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`); const word = state.reported.find((r) => r.node === MACHINE); if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return; last = asked.out; } catch (err) { if (err instanceof Error && err.message.includes("did not apply")) throw err; last = asked.out; } } await new Promise((r) => setTimeout(r, 5000)); } throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`); } before(async () => { if (skip) return; const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; held = raised.images; await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000); const up = await must(`docker ps --format '{{.Names}}'`); for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) { assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`); } await mesh(`node add ${MACHINE}`); const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`)); await must(`${HOST_PATH} enrol --token ${quote(token)}`); await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); }, { timeout: 1_800_000 }); after(async () => { if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 600_000 }); test("model access refreshes on the manager node and delivers only the access token, never the refresh token", { skip, timeout: 1_500_000, }, async () => { const managerImage = pinned("mesh-runtime-anthropic-manager"); const consumerImage = pinned("mesh-runtime-anthropic-consumer"); // --- the licence, and the manager as its holder ------------------------------------------------ // The manager module is a HOLDER, named the licence's manager. It is delivered the refresh token; // its bound facts carry the node's PUBLIC sealing key, which is what adoption seals to. The consumer // holder is added later — only once an access token exists to seal to it — because a holder with no // credential yet cannot have a declaration built for it. await mesh(`licence add anthropic personal --serves '{"model":"a-model"}'`); await mesh(`licence manager personal ${MACHINE} anthropic-manager`); await mesh(`licence use personal ${MACHINE} anthropic-manager`); // --- the manager module, deployed so the host delivers its bound facts -------------------------- // Inline manifest mirroring the committed module.json: model-access holder, refresh token bound as a // sealed secret, no node-key mount. The scheduled container installs as present state (ADR 0053); // the test drives adopt/refresh directly for a deterministic flow rather than waiting on cron. const managerManifest = JSON.stringify({ module: "anthropic-manager", version: "1", requires: ["model-access"], binds: { "model-access": "/var/lib/mesh/anthropic-manager/model.json" }, secrets: { "model-access": "/var/lib/mesh/anthropic-manager/refresh-token" }, "own-secrets": { broker: "/var/lib/mesh/anthropic-manager/broker" }, emits: ["module.anthropic-manager.usage.read"], resources: [ { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-manager", mode: "0700" }, { id: "out", type: "directory", path: "/var/lib/mesh/anthropic-manager/out", mode: "0700" }, { id: "refresh", type: "container", name: "mesh-anthropic-manager-refresh", image: managerImage, network: "host", schedule: "*/9 * * * *", args: ["run", "/app/modules/anthropic-manager/dist/refresh/index.js"], volumes: ["/var/lib/mesh/anthropic-manager:/run/state"], env: { MESH_ANTHROPIC_LICENCE: "personal", MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token", MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage", MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/refresh-token", MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json", MESH_ANTHROPIC_ACCESS_OUT: "/run/state/out/access-token", MESH_ANTHROPIC_GRANT_OUT: "/run/state/out/grant.json", MESH_ANTHROPIC_USAGE_OUT: "/run/state/out/usage.json", }, }, ], }); await must(`printf %s ${quote(managerManifest)} > /tmp/anthropic-manager.json && docker cp /tmp/anthropic-manager.json mesh-controller:/anthropic-manager.json`); await mesh(`module add /anthropic-manager.json`); await mesh(`module issue anthropic-manager --node ${MACHINE}`); await mesh(`assign ${MACHINE} anthropic-manager`); await mesh(`push ${MACHINE}`); await settled(); // The image the host pulled for the manager runtime is now local; drive it directly. const managerId = await imageId("anthropic-manager"); // The host delivered the manager's bound facts, carrying the node's PUBLIC sealing key. const facts = await must(`cat /var/lib/mesh/anthropic-manager/model.json`); assert.match(facts, /"manager_public_key"\s*:/, `the manager was not delivered its node public key:\n${facts}`); // --- the OAuth stub: a tiny node server run from the manager image itself ----------------------- const stub = [ "const http=require('http');", "http.createServer((req,res)=>{let b='';req.on('data',c=>b+=c);req.on('end',()=>{", " if(req.url.startsWith('/token')){res.setHeader('content-type','application/json');", ` res.end(JSON.stringify({access_token:${JSON.stringify(ACCESS_TOKEN)},refresh_token:${JSON.stringify(ROTATED_REFRESH)},expires_in:3600,refresh_token_expires_in:2592000,subscription_type:'pro'}));return;}`, " if(req.url.startsWith('/usage')){res.setHeader('content-type','application/json');", " res.end(JSON.stringify({five_hour:{utilization:12,resets_at:'2026-01-01T00:00:00Z'},seven_day:{utilization:3}}));return;}", " res.statusCode=404;res.end('no');});}).listen(9099,'127.0.0.1',()=>console.log('stub up'));", ].join("\n"); await must(`printf %s ${quote(stub)} > /var/lib/mesh/anthropic-manager/stub.js`); await must(`docker rm -f oauth-stub 2>/dev/null; docker run -d --name oauth-stub --network host --entrypoint node -v /var/lib/mesh/anthropic-manager/stub.js:/run/stub.js:ro ${managerId} /run/stub.js`); await must(`for i in $(seq 1 20); do curl -s -X POST http://127.0.0.1:9099/token >/dev/null && break; sleep 1; done`); // --- 1. adopt: seal the operator's refresh token to THIS node's public key, on the manager node -- // adopt reads the node public key from the delivered bound facts (never a private key), seals, and // hands out only the box. await must(`printf %s ${quote(REFRESH_TOKEN)} > /var/lib/mesh/anthropic-manager/adopt-token`); await must( `docker run --rm --network host -v /var/lib/mesh/anthropic-manager:/run/state ` + `-e MESH_ANTHROPIC_ADOPT_TOKEN_FILE=/run/state/adopt-token ` + `-e MESH_MODEL_ACCESS_BIND_FILE=/run/state/model.json ` + `-e MESH_ANTHROPIC_GRANT_OUT=/run/state/out/grant.json ` + `${managerId} run /app/modules/anthropic-manager/dist/adopt/index.js`, ); const sealedGrant = await must(`cat /var/lib/mesh/anthropic-manager/out/grant.json`); assert.doesNotMatch(sealedGrant, new RegExp(REFRESH_TOKEN), `the adopted box holds the refresh token in the clear:\n${sealedGrant}`); assert.match(sealedGrant, /"sealed"\s*:/, `the adopted grant is not a sealed box:\n${sealedGrant}`); // Store the sealed box in the control plane — which never sees the refresh token. await stageIntoControl(`/var/lib/mesh/anthropic-manager/out/grant.json`, `/grant.json`); await mesh(`licence set-grant personal --file /grant.json`); // --- 2. the host unseals: a push mounts the cleartext refresh token at the manager's secret path -- await mesh(`push ${MACHINE}`); await settled(); let mounted = false; for (let i = 0; i < 20 && !mounted; i++) { mounted = (await on(`test -s /var/lib/mesh/anthropic-manager/refresh-token`)).ok; if (!mounted) await new Promise((r) => setTimeout(r, 3000)); } assert.ok(mounted, "the host never unsealed and mounted the refresh token for the manager"); const mountedToken = (await must(`cat /var/lib/mesh/anthropic-manager/refresh-token`)).trim(); assert.equal(mountedToken, REFRESH_TOKEN, "the host mounted the wrong cleartext refresh token"); // --- 3. refresh: read the cleartext, call the stub, re-seal a rotated token, write out ---------- await must( `docker run --rm --network host -v /var/lib/mesh/anthropic-manager:/run/state ` + `-e MESH_ANTHROPIC_LICENCE=personal ` + `-e MESH_ANTHROPIC_TOKEN_ENDPOINT=http://127.0.0.1:9099/token ` + `-e MESH_ANTHROPIC_USAGE_ENDPOINT=http://127.0.0.1:9099/usage ` + `-e MESH_MODEL_ACCESS_SECRET_FILE=/run/state/refresh-token ` + `-e MESH_MODEL_ACCESS_BIND_FILE=/run/state/model.json ` + `-e MESH_ANTHROPIC_ACCESS_OUT=/run/state/out/access-token ` + `-e MESH_ANTHROPIC_GRANT_OUT=/run/state/out/new-grant.json ` + `-e MESH_ANTHROPIC_USAGE_OUT=/run/state/out/usage.json ` + `${managerId} run /app/modules/anthropic-manager/dist/refresh/index.js`, ); const producedAccess = (await must(`cat /var/lib/mesh/anthropic-manager/out/access-token`)).trim(); assert.equal(producedAccess, ACCESS_TOKEN, "the manager did not mint the stub's access token"); const newBox = await must(`cat /var/lib/mesh/anthropic-manager/out/new-grant.json`); assert.doesNotMatch(newBox, new RegExp(ROTATED_REFRESH), `the re-sealed box holds the rotated refresh token in the clear:\n${newBox}`); const usage = await must(`cat /var/lib/mesh/anthropic-manager/out/usage.json`); assert.match(usage, /"sessionPct":12/, `the licence-grain usage reading is wrong:\n${usage}`); // --- 4. submit: the control plane is handed only the access token + opaque box ------------------- // The consumer is put on the licence now — an access token exists to seal to it. await mesh(`licence use personal ${MACHINE} anthropic-consumer`); await stageIntoControl(`/var/lib/mesh/anthropic-manager/out/access-token`, `/access-token`); await stageIntoControl(`/var/lib/mesh/anthropic-manager/out/new-grant.json`, `/new-grant.json`); const submitted = await mesh(`licence submit-refresh personal --access-file /access-token --grant-file /new-grant.json`); assert.match(submitted, /sealed to 1 holder/, submitted); // --- 5. deliver: deploy the consumer and push; it gets the sealed access token ------------------- const consumerManifest = JSON.stringify({ module: "anthropic-consumer", version: "1", requires: ["model-access"], binds: { "model-access": "/var/lib/anthropic-consumer/model.json" }, secrets: { "model-access": "/var/lib/anthropic-consumer/access-token" }, "own-secrets": { broker: "/var/lib/mesh/anthropic-consumer/broker" }, emits: ["module.anthropic-consumer.usage.session"], resources: [ { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-consumer", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/anthropic-consumer", mode: "0700" }, { id: "claude-home", type: "directory", path: "/var/lib/anthropic-consumer/claude", mode: "0700" }, { id: "apply", type: "container", name: "mesh-anthropic-consumer-apply", image: consumerImage, network: "host", schedule: "*/9 * * * *", args: ["run", "/app/modules/anthropic-consumer/dist/apply/index.js"], volumes: ["/var/lib/anthropic-consumer:/run/state"], env: { MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/access-token", MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json", MESH_CLAUDE_CREDENTIALS_FILE: "/run/state/claude/.credentials.json", MESH_CLAUDE_IDENTITY_FILE: "/run/state/claude/.claude.json", }, }, ], }); await must(`printf %s ${quote(consumerManifest)} > /tmp/anthropic-consumer.json && docker cp /tmp/anthropic-consumer.json mesh-controller:/anthropic-consumer.json`); await mesh(`module add /anthropic-consumer.json`); await mesh(`module issue anthropic-consumer --node ${MACHINE}`); await mesh(`assign ${MACHINE} anthropic-consumer`); await mesh(`push ${MACHINE}`); await settled(); const consumerId = await imageId("anthropic-consumer"); let delivered = false; for (let i = 0; i < 20 && !delivered; i++) { delivered = (await on(`test -s /var/lib/anthropic-consumer/access-token`)).ok; if (!delivered) await new Promise((r) => setTimeout(r, 3000)); } assert.ok(delivered, "the sealed access token was never delivered to the consumer's secret path"); await must( `docker run --rm --network host -v /var/lib/anthropic-consumer:/run/state ` + `-e MESH_MODEL_ACCESS_SECRET_FILE=/run/state/access-token ` + `-e MESH_MODEL_ACCESS_BIND_FILE=/run/state/model.json ` + `-e MESH_CLAUDE_CREDENTIALS_FILE=/run/state/claude/.credentials.json ` + `-e MESH_CLAUDE_IDENTITY_FILE=/run/state/claude/.claude.json ` + `${consumerId} run /app/modules/anthropic-consumer/dist/apply/index.js`, ); // --- the invariant, asserted from every angle --------------------------------------------------- const creds = await must(`cat /var/lib/anthropic-consumer/claude/.credentials.json`); assert.match(creds, new RegExp(ACCESS_TOKEN), `the access token did not reach the credential file:\n${creds}`); const parsed = JSON.parse(creds) as { claudeAiOauth?: { accessToken?: string; refreshToken?: string } }; assert.equal(parsed.claudeAiOauth?.accessToken, ACCESS_TOKEN); assert.ok(!parsed.claudeAiOauth?.refreshToken, "the consumer was given a refresh token"); // The refresh token — original or rotated — is nowhere on the CONSUMING node's tree. (It IS on the // manager's, as cleartext the host mounted for it — that is the one node allowed to read it.) for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) { const found = await on(`grep -rq ${quote(secret)} /var/lib/anthropic-consumer`); assert.ok(!found.ok, `a refresh token is on the consuming node under /var/lib/anthropic-consumer`); } // The control plane's own database holds the refresh token only as ciphertext. const grantRow = await must( `docker exec mesh-store psql -U postgres -d licences -qAt -c "select sealed, manager_key from refresh_grant where licence='personal'"`, ); assert.ok(grantRow.trim().length > 0, "no refresh grant was stored"); for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) { assert.doesNotMatch(grantRow, new RegExp(secret), `the refresh token is in the control plane's database in the clear:\n${grantRow}`); } // And the CONSUMER holder's sealed column carries the access token's seal, never a refresh token. const holder = await must( `docker exec mesh-store psql -U postgres -d licences -qAt -c "select coalesce(sealed,'') from licence_holder where licence='personal' and module='anthropic-consumer'"`, ); assert.ok(holder.trim().length > 0, "nothing was sealed to the consumer holder"); for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) { assert.doesNotMatch(holder, new RegExp(secret), "a refresh token is in the consumer holder row"); } await must(`docker rm -f oauth-stub 2>/dev/null || true`); });