mesh-lab/test/integration/anthropic-bed.test.ts

/**
 * The mesh plays out the model-access refreshable-grant flow for Anthropic end to end, with the
 * vendor's OAuth endpoint STUBBED (novox/hq ADR 0050). It proves the one property the carve-out rests
 * on, and the reviewer will scrutinise it: the refresh token is delivered ONLY to the manager node —
 * as an ordinary sealed credential the HOST unseals — the control plane is handed only the ACCESS
 * token in the clear (plus an opaque re-sealed refresh box), and a consuming node writes an
 * access-token-only credential and is never delivered a refresh token — nowhere on the machine,
 * nowhere in the control plane's database.
 *
 * **What changed from the earlier cut, and why this is simpler.** The refresh token no longer rides a
 * bespoke at-rest envelope the module opens with a node private key the mesh must somehow place — a
 * module is never given a node's private key, so that path could not exist. It rides the ORDINARY
 * sealed-delivery path instead: mesh-control (via the manager module at adoption) seals it to the
 * manager node's PUBLIC key, and the HOST unseals it with that node's real private key and mounts the
 * cleartext at the manager module's bound secret path — exactly as a consumer's db password arrives.
 * So there is no fake node key pair mounted here any more; the host's own real sealing key does the
 * unsealing, and the manager module does no crypto beyond re-sealing a rotated token to the same
 * public key.
 *
 * The flow, driven deterministically (the runtime images are the real ones the host pulled; the OAuth
 * endpoint is a tiny node stub run from the same image, so no vendor is reached):
 *   1. deploy the manager and adopt: the manager holder is delivered its bound facts (carrying the
 *      node's PUBLIC sealing key). The manager runtime seals the operator's refresh token to that key
 *      and hands the box to `licence set-grant` — the control plane stores ciphertext it cannot open.
 *   2. the host unseals: a push delivers the sealed refresh token to the manager, and the host mounts
 *      the cleartext at the manager module's secret path — the one place a refresh token is readable.
 *   3. refresh: the manager runtime reads that cleartext, calls the stub token endpoint, re-seals a
 *      rotated refresh token to the node's public key, and writes out ONLY { access token, sealed box }.
 *   4. submit: `licence submit-refresh` hands the control plane those two things — never the refresh
 *      token in the clear — and it seals the access token to the consumer holder.
 *   5. deliver: a push delivers the sealed access token to the consumer; the consumer runtime writes
 *      ~/.claude/.credentials.json, access-token-only.
 *
 * STUBBED, and flagged in the report: (a) the vendor OAuth endpoint (a node stub); (b) the submit
 * transport (the test invokes `mesh-control licence submit-refresh` on the manager's output, standing
 * in for the authenticated cross-node call a manager node would make). The node-private-key stub of
 * the earlier cut is GONE — the host uses its own real key.
 *
 *   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
 *   Build both runtime images into the local daemon first:
 *     scripts/build-module-runtime.sh anthropic-manager  /tmp/anthropic-manager.tar
 *     scripts/build-module-runtime.sh anthropic-consumer  /tmp/anthropic-consumer.tar
 */

import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";

const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";

const skip = !capability.usable
  ? `lab not usable: ${capability.why}`
  : !binary || !existsSync(binary)
    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
    : !bundle || !existsSync(bundle)
      ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)"
      : false;

const SCENARIO = "anthropic-bed";
const MACHINE = "anchor";

// The fake tokens the flow moves. The whole test is: the second reaches the consumer, the first never
// does — except on the manager node, which is allowed to read it.
const REFRESH_TOKEN = "rt-lab-refresh-only-the-manager-may-read";
const ACCESS_TOKEN = "at-lab-access-token-minted-by-the-stub";
const ROTATED_REFRESH = "rt-lab-rotated-still-only-the-manager";

let instanceId = "";
let held: HeldImage[] = [];

function quote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
  const { stdout } = await exec(instanceId, MACHINE, [
    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
  ], timeoutMs);
  const marker = stdout.lastIndexOf("__exit=");
  if (marker < 0) return { out: stdout, ok: false };
  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}

async function must(command: string, timeoutMs?: number): Promise<string> {
  const { out, ok } = await on(command, timeoutMs);
  if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
  return out;
}

async function mesh(command: string, timeoutMs?: number): Promise<string> {
  return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}

// The manager's adopt/refresh runtime writes its outputs as root, mode 0600 (secret files). To hand
// one to `mesh-control` — whose process runs as a non-root user — the test relaxes the mode on the
// anchor host (where `must` is root) and then copies it in: `docker cp` preserves the source mode, so
// the file lands 0644 and mesh-control (a distroless image with no `chmod` of its own) can read it.
// What is staged this way is a sealed box or the access token, never a cleartext refresh token, so a
// world-readable copy discloses nothing the control plane does not already hold. In production the
// operator who ran adopt owns the file and this does not arise.
async function stageIntoControl(hostPath: string, dest: string): Promise<void> {
  await must(`chmod 0644 ${hostPath} && docker cp ${hostPath} mesh-control:${dest}`);
}

async function meshTry(command: string): Promise<{ out: string; ok: boolean }> {
  return on(`docker exec mesh-control /mesh-control ${command}`);
}

/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
  return onTheMachine(reference, held);
}

function bundleFor(images: HeldImage[]): string {
  return substrateBundle(bundle, images);
}

function tokenFrom(said: string): string {
  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
  assert.ok(found, `no token in:\n${said}`);
  return found;
}

/** The local image id the host pulled for a runtime, so the test can drive it deterministically. */
async function imageId(substr: string): Promise<string> {
  const out = (await must(`docker images --no-trunc --format '{{.ID}} {{.Repository}}' | grep ${quote(substr)} | head -1`)).trim();
  const id = out.split(/\s+/)[0] ?? "";
  assert.ok(id.startsWith("sha256:") || id.length > 0, `no local image matched ${substr}:\n${out}`);
  return id;
}

async function settled(withinMs = 600_000): Promise<void> {
  const until = Date.now() + withinMs;
  let last = "";
  while (Date.now() < until) {
    const asked = await meshTry(`status --json`);
    if (asked.ok) {
      try {
        const state = JSON.parse(asked.out) as {
          wrong: { node: string; outcome: string }[];
          waiting: { node: string }[];
          reported: { node: string; outcome: string; current: boolean }[];
        };
        const bad = state.wrong.find((w) => w.node === MACHINE);
        if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
        const word = state.reported.find((r) => r.node === MACHINE);
        if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
        last = asked.out;
      } catch (err) {
        if (err instanceof Error && err.message.includes("did not apply")) throw err;
        last = asked.out;
      }
    }
    await new Promise((r) => setTimeout(r, 5000));
  }
  throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}

before(async () => {
  if (skip) return;

  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
    onProgress: (m) => console.log(`raise: ${m}`),
  });
  instanceId = raised.instanceId;
  held = raised.images;

  await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
  await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
  const up = await must(`docker ps --format '{{.Names}}'`);
  for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) {
    assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`);
  }

  await mesh(`node add ${MACHINE}`);
  const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
  await must(`${HOST_PATH} enrol --token ${quote(token)}`);
  await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });

after(async () => {
  if (instanceId) await destroy(instanceId);
  await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });

test("model access refreshes on the manager node and delivers only the access token, never the refresh token", {
  skip, timeout: 1_500_000,
}, async () => {
  const managerImage = pinned("mesh-runtime-anthropic-manager");
  const consumerImage = pinned("mesh-runtime-anthropic-consumer");

  // --- the licence, and the manager as its holder ------------------------------------------------
  // The manager module is a HOLDER, named the licence's manager. It is delivered the refresh token;
  // its bound facts carry the node's PUBLIC sealing key, which is what adoption seals to. The consumer
  // holder is added later — only once an access token exists to seal to it — because a holder with no
  // credential yet cannot have a declaration built for it.
  await mesh(`licence add anthropic personal --serves '{"model":"a-model"}'`);
  await mesh(`licence manager personal ${MACHINE} anthropic-manager`);
  await mesh(`licence use personal ${MACHINE} anthropic-manager`);

  // --- the manager module, deployed so the host delivers its bound facts --------------------------
  // Inline manifest mirroring the committed module.json: model-access holder, refresh token bound as a
  // sealed secret, no node-key mount. The scheduled container installs as present state (ADR 0053);
  // the test drives adopt/refresh directly for a deterministic flow rather than waiting on cron.
  const managerManifest = JSON.stringify({
    module: "anthropic-manager",
    version: "1",
    requires: ["model-access"],
    binds: { "model-access": "/var/lib/mesh/anthropic-manager/model.json" },
    secrets: { "model-access": "/var/lib/mesh/anthropic-manager/refresh-token" },
    "own-secrets": { broker: "/var/lib/mesh/anthropic-manager/broker" },
    emits: ["module.anthropic-manager.usage.read"],
    resources: [
      { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-manager", mode: "0700" },
      { id: "out", type: "directory", path: "/var/lib/mesh/anthropic-manager/out", mode: "0700" },
      {
        id: "refresh", type: "container", name: "mesh-anthropic-manager-refresh",
        image: managerImage, network: "host", schedule: "*/9 * * * *",
        args: ["run", "/app/modules/anthropic-manager/dist/refresh/index.js"],
        volumes: ["/var/lib/mesh/anthropic-manager:/run/state"],
        env: {
          MESH_ANTHROPIC_LICENCE: "personal",
          MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token",
          MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage",
          MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/refresh-token",
          MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json",
          MESH_ANTHROPIC_ACCESS_OUT: "/run/state/out/access-token",
          MESH_ANTHROPIC_GRANT_OUT: "/run/state/out/grant.json",
          MESH_ANTHROPIC_USAGE_OUT: "/run/state/out/usage.json",
        },
      },
    ],
  });
  await must(`printf %s ${quote(managerManifest)} > /tmp/anthropic-manager.json && docker cp /tmp/anthropic-manager.json mesh-control:/anthropic-manager.json`);
  await mesh(`module add /anthropic-manager.json`);
  await mesh(`module issue anthropic-manager --node ${MACHINE}`);
  await mesh(`assign ${MACHINE} anthropic-manager`);
  await mesh(`push ${MACHINE}`);
  await settled();

  // The image the host pulled for the manager runtime is now local; drive it directly.
  const managerId = await imageId("anthropic-manager");

  // The host delivered the manager's bound facts, carrying the node's PUBLIC sealing key.
  const facts = await must(`cat /var/lib/mesh/anthropic-manager/model.json`);
  assert.match(facts, /"manager_public_key"\s*:/, `the manager was not delivered its node public key:\n${facts}`);

  // --- the OAuth stub: a tiny node server run from the manager image itself -----------------------
  const stub = [
    "const http=require('http');",
    "http.createServer((req,res)=>{let b='';req.on('data',c=>b+=c);req.on('end',()=>{",
    "  if(req.url.startsWith('/token')){res.setHeader('content-type','application/json');",
    `    res.end(JSON.stringify({access_token:${JSON.stringify(ACCESS_TOKEN)},refresh_token:${JSON.stringify(ROTATED_REFRESH)},expires_in:3600,refresh_token_expires_in:2592000,subscription_type:'pro'}));return;}`,
    "  if(req.url.startsWith('/usage')){res.setHeader('content-type','application/json');",
    "    res.end(JSON.stringify({five_hour:{utilization:12,resets_at:'2026-01-01T00:00:00Z'},seven_day:{utilization:3}}));return;}",
    "  res.statusCode=404;res.end('no');});}).listen(9099,'127.0.0.1',()=>console.log('stub up'));",
  ].join("\n");
  await must(`printf %s ${quote(stub)} > /var/lib/mesh/anthropic-manager/stub.js`);
  await must(`docker rm -f oauth-stub 2>/dev/null; docker run -d --name oauth-stub --network host --entrypoint node -v /var/lib/mesh/anthropic-manager/stub.js:/run/stub.js:ro ${managerId} /run/stub.js`);
  await must(`for i in $(seq 1 20); do curl -s -X POST http://127.0.0.1:9099/token >/dev/null && break; sleep 1; done`);

  // --- 1. adopt: seal the operator's refresh token to THIS node's public key, on the manager node --
  // adopt reads the node public key from the delivered bound facts (never a private key), seals, and
  // hands out only the box.
  await must(`printf %s ${quote(REFRESH_TOKEN)} > /var/lib/mesh/anthropic-manager/adopt-token`);
  await must(
    `docker run --rm --network host -v /var/lib/mesh/anthropic-manager:/run/state ` +
      `-e MESH_ANTHROPIC_ADOPT_TOKEN_FILE=/run/state/adopt-token ` +
      `-e MESH_MODEL_ACCESS_BIND_FILE=/run/state/model.json ` +
      `-e MESH_ANTHROPIC_GRANT_OUT=/run/state/out/grant.json ` +
      `${managerId} run /app/modules/anthropic-manager/dist/adopt/index.js`,
  );
  const sealedGrant = await must(`cat /var/lib/mesh/anthropic-manager/out/grant.json`);
  assert.doesNotMatch(sealedGrant, new RegExp(REFRESH_TOKEN),
    `the adopted box holds the refresh token in the clear:\n${sealedGrant}`);
  assert.match(sealedGrant, /"sealed"\s*:/, `the adopted grant is not a sealed box:\n${sealedGrant}`);

  // Store the sealed box in the control plane — which never sees the refresh token.
  await stageIntoControl(`/var/lib/mesh/anthropic-manager/out/grant.json`, `/grant.json`);
  await mesh(`licence set-grant personal --file /grant.json`);

  // --- 2. the host unseals: a push mounts the cleartext refresh token at the manager's secret path --
  await mesh(`push ${MACHINE}`);
  await settled();
  let mounted = false;
  for (let i = 0; i < 20 && !mounted; i++) {
    mounted = (await on(`test -s /var/lib/mesh/anthropic-manager/refresh-token`)).ok;
    if (!mounted) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(mounted, "the host never unsealed and mounted the refresh token for the manager");
  const mountedToken = (await must(`cat /var/lib/mesh/anthropic-manager/refresh-token`)).trim();
  assert.equal(mountedToken, REFRESH_TOKEN, "the host mounted the wrong cleartext refresh token");

  // --- 3. refresh: read the cleartext, call the stub, re-seal a rotated token, write out ----------
  await must(
    `docker run --rm --network host -v /var/lib/mesh/anthropic-manager:/run/state ` +
      `-e MESH_ANTHROPIC_LICENCE=personal ` +
      `-e MESH_ANTHROPIC_TOKEN_ENDPOINT=http://127.0.0.1:9099/token ` +
      `-e MESH_ANTHROPIC_USAGE_ENDPOINT=http://127.0.0.1:9099/usage ` +
      `-e MESH_MODEL_ACCESS_SECRET_FILE=/run/state/refresh-token ` +
      `-e MESH_MODEL_ACCESS_BIND_FILE=/run/state/model.json ` +
      `-e MESH_ANTHROPIC_ACCESS_OUT=/run/state/out/access-token ` +
      `-e MESH_ANTHROPIC_GRANT_OUT=/run/state/out/new-grant.json ` +
      `-e MESH_ANTHROPIC_USAGE_OUT=/run/state/out/usage.json ` +
      `${managerId} run /app/modules/anthropic-manager/dist/refresh/index.js`,
  );
  const producedAccess = (await must(`cat /var/lib/mesh/anthropic-manager/out/access-token`)).trim();
  assert.equal(producedAccess, ACCESS_TOKEN, "the manager did not mint the stub's access token");
  const newBox = await must(`cat /var/lib/mesh/anthropic-manager/out/new-grant.json`);
  assert.doesNotMatch(newBox, new RegExp(ROTATED_REFRESH),
    `the re-sealed box holds the rotated refresh token in the clear:\n${newBox}`);
  const usage = await must(`cat /var/lib/mesh/anthropic-manager/out/usage.json`);
  assert.match(usage, /"sessionPct":12/, `the licence-grain usage reading is wrong:\n${usage}`);

  // --- 4. submit: the control plane is handed only the access token + opaque box -------------------
  // The consumer is put on the licence now — an access token exists to seal to it.
  await mesh(`licence use personal ${MACHINE} anthropic-consumer`);
  await stageIntoControl(`/var/lib/mesh/anthropic-manager/out/access-token`, `/access-token`);
  await stageIntoControl(`/var/lib/mesh/anthropic-manager/out/new-grant.json`, `/new-grant.json`);
  const submitted = await mesh(`licence submit-refresh personal --access-file /access-token --grant-file /new-grant.json`);
  assert.match(submitted, /sealed to 1 holder/, submitted);

  // --- 5. deliver: deploy the consumer and push; it gets the sealed access token -------------------
  const consumerManifest = JSON.stringify({
    module: "anthropic-consumer",
    version: "1",
    requires: ["model-access"],
    binds: { "model-access": "/var/lib/anthropic-consumer/model.json" },
    secrets: { "model-access": "/var/lib/anthropic-consumer/access-token" },
    "own-secrets": { broker: "/var/lib/mesh/anthropic-consumer/broker" },
    emits: ["module.anthropic-consumer.usage.session"],
    resources: [
      { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-consumer", mode: "0700" },
      { id: "state", type: "directory", path: "/var/lib/anthropic-consumer", mode: "0700" },
      { id: "claude-home", type: "directory", path: "/var/lib/anthropic-consumer/claude", mode: "0700" },
      {
        id: "apply", type: "container", name: "mesh-anthropic-consumer-apply",
        image: consumerImage, network: "host", schedule: "*/9 * * * *",
        args: ["run", "/app/modules/anthropic-consumer/dist/apply/index.js"],
        volumes: ["/var/lib/anthropic-consumer:/run/state"],
        env: {
          MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/access-token",
          MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json",
          MESH_CLAUDE_CREDENTIALS_FILE: "/run/state/claude/.credentials.json",
          MESH_CLAUDE_IDENTITY_FILE: "/run/state/claude/.claude.json",
        },
      },
    ],
  });
  await must(`printf %s ${quote(consumerManifest)} > /tmp/anthropic-consumer.json && docker cp /tmp/anthropic-consumer.json mesh-control:/anthropic-consumer.json`);
  await mesh(`module add /anthropic-consumer.json`);
  await mesh(`module issue anthropic-consumer --node ${MACHINE}`);
  await mesh(`assign ${MACHINE} anthropic-consumer`);
  await mesh(`push ${MACHINE}`);
  await settled();

  const consumerId = await imageId("anthropic-consumer");

  let delivered = false;
  for (let i = 0; i < 20 && !delivered; i++) {
    delivered = (await on(`test -s /var/lib/anthropic-consumer/access-token`)).ok;
    if (!delivered) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(delivered, "the sealed access token was never delivered to the consumer's secret path");
  await must(
    `docker run --rm --network host -v /var/lib/anthropic-consumer:/run/state ` +
      `-e MESH_MODEL_ACCESS_SECRET_FILE=/run/state/access-token ` +
      `-e MESH_MODEL_ACCESS_BIND_FILE=/run/state/model.json ` +
      `-e MESH_CLAUDE_CREDENTIALS_FILE=/run/state/claude/.credentials.json ` +
      `-e MESH_CLAUDE_IDENTITY_FILE=/run/state/claude/.claude.json ` +
      `${consumerId} run /app/modules/anthropic-consumer/dist/apply/index.js`,
  );

  // --- the invariant, asserted from every angle ---------------------------------------------------
  const creds = await must(`cat /var/lib/anthropic-consumer/claude/.credentials.json`);
  assert.match(creds, new RegExp(ACCESS_TOKEN), `the access token did not reach the credential file:\n${creds}`);
  const parsed = JSON.parse(creds) as { claudeAiOauth?: { accessToken?: string; refreshToken?: string } };
  assert.equal(parsed.claudeAiOauth?.accessToken, ACCESS_TOKEN);
  assert.ok(!parsed.claudeAiOauth?.refreshToken, "the consumer was given a refresh token");

  // The refresh token — original or rotated — is nowhere on the CONSUMING node's tree. (It IS on the
  // manager's, as cleartext the host mounted for it — that is the one node allowed to read it.)
  for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) {
    const found = await on(`grep -rq ${quote(secret)} /var/lib/anthropic-consumer`);
    assert.ok(!found.ok, `a refresh token is on the consuming node under /var/lib/anthropic-consumer`);
  }

  // The control plane's own database holds the refresh token only as ciphertext.
  const grantRow = await must(
    `docker exec mesh-store psql -U postgres -d licences -qAt -c "select sealed, manager_key from refresh_grant where licence='personal'"`,
  );
  assert.ok(grantRow.trim().length > 0, "no refresh grant was stored");
  for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) {
    assert.doesNotMatch(grantRow, new RegExp(secret),
      `the refresh token is in the control plane's database in the clear:\n${grantRow}`);
  }
  // And the CONSUMER holder's sealed column carries the access token's seal, never a refresh token.
  const holder = await must(
    `docker exec mesh-store psql -U postgres -d licences -qAt -c "select coalesce(sealed,'') from licence_holder where licence='personal' and module='anthropic-consumer'"`,
  );
  assert.ok(holder.trim().length > 0, "nothing was sealed to the consumer holder");
  for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) {
    assert.doesNotMatch(holder, new RegExp(secret), "a refresh token is in the consumer holder row");
  }

  await must(`docker rm -f oauth-stub 2>/dev/null || true`);
});